diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/ServerFlagModule.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/ServerFlagModule.docx new file mode 100644 index 000000000..fdedaec12 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/ServerFlagModule.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/ServerFlagModule.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/ServerFlagModule.scala deleted file mode 100644 index d38bfb24a..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/ServerFlagModule.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.graph_feature_service.server.modules - -import com.twitter.inject.TwitterModule - -object ServerFlagNames { - final val NumWorkers = "service.num_workers" - final val ServiceRole = "service.role" - final val ServiceEnv = "service.env" - - final val MemCacheClientName = "service.mem_cache_client_name" - final val MemCachePath = "service.mem_cache_path" -} - -/** - * Initializes references to the flag values defined in the aurora.deploy file. - * To check what the flag values are initialized in runtime, search FlagsModule in stdout - */ -object ServerFlagsModule extends TwitterModule { - - import ServerFlagNames._ - - flag[Int](NumWorkers, "Num of workers") - - flag[String](ServiceRole, "Service Role") - - flag[String](ServiceEnv, "Service Env") - - flag[String](MemCacheClientName, "MemCache Client Name") - - flag[String](MemCachePath, "MemCache Path") -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/FeatureTypesEncoder.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/FeatureTypesEncoder.docx new file mode 100644 index 000000000..84276e205 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/FeatureTypesEncoder.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/FeatureTypesEncoder.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/FeatureTypesEncoder.scala deleted file mode 100644 index b8ad4d743..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/FeatureTypesEncoder.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.graph_feature_service.server.stores - -import com.twitter.graph_feature_service.common.Configs.RandomSeed -import com.twitter.graph_feature_service.thriftscala.FeatureType -import scala.util.hashing.MurmurHash3 - -object FeatureTypesEncoder { - - def apply(featureTypes: Seq[FeatureType]): String = { - val byteArray = featureTypes.flatMap { featureType => - Array(featureType.leftEdgeType.getValue.toByte, featureType.rightEdgeType.getValue.toByte) - }.toArray - (MurmurHash3.bytesHash(byteArray, RandomSeed) & 0x7fffffff).toString // keep positive - } - -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/GetIntersectionStore.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/GetIntersectionStore.docx new file mode 100644 index 000000000..1b4f90067 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/GetIntersectionStore.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/GetIntersectionStore.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/GetIntersectionStore.scala deleted file mode 100644 index 7824be511..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/GetIntersectionStore.scala +++ /dev/null @@ -1,181 +0,0 @@ -package com.twitter.graph_feature_service.server.stores - -import com.twitter.finagle.RequestTimeoutException -import com.twitter.finagle.stats.{Stat, StatsReceiver} -import com.twitter.graph_feature_service.server.handlers.ServerGetIntersectionHandler.GetIntersectionRequest -import com.twitter.graph_feature_service.server.modules.GraphFeatureServiceWorkerClients -import com.twitter.graph_feature_service.server.stores.GetIntersectionStore.GetIntersectionQuery -import com.twitter.graph_feature_service.thriftscala._ -import com.twitter.inject.Logging -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import javax.inject.Singleton -import scala.collection.mutable.ArrayBuffer - -@Singleton -case class GetIntersectionStore( - graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, - statsReceiver: StatsReceiver) - extends ReadableStore[GetIntersectionQuery, CachedIntersectionResult] - with Logging { - - import GetIntersectionStore._ - - private val stats = statsReceiver.scope("get_intersection_store") - private val requestCount = stats.counter(name = "request_count") - private val aggregatorLatency = stats.stat("aggregator_latency") - private val timeOutCounter = stats.counter("worker_timeouts") - private val unknownErrorCounter = stats.counter("unknown_errors") - - override def multiGet[K1 <: GetIntersectionQuery]( - ks: Set[K1] - ): Map[K1, Future[Option[CachedIntersectionResult]]] = { - if (ks.isEmpty) { - Map.empty - } else { - requestCount.incr() - - val head = ks.head - // We assume all the GetIntersectionQuery use the same userId and featureTypes - val userId = head.userId - val featureTypes = head.featureTypes - val presetFeatureTypes = head.presetFeatureTypes - val calculatedFeatureTypes = head.calculatedFeatureTypes - val intersectionIdLimit = head.intersectionIdLimit - - val request = WorkerIntersectionRequest( - userId, - ks.map(_.candidateId).toArray, - featureTypes, - presetFeatureTypes, - intersectionIdLimit - ) - - val resultFuture = Future - .collect( - graphFeatureServiceWorkerClients.workers.map { worker => - worker - .getIntersection(request) - .rescue { - case _: RequestTimeoutException => - timeOutCounter.incr() - Future.value(DefaultWorkerIntersectionResponse) - case e => - unknownErrorCounter.incr() - logger.error("Failure to load result.", e) - Future.value(DefaultWorkerIntersectionResponse) - } - } - ).map { responses => - Stat.time(aggregatorLatency) { - gfsIntersectionResponseAggregator( - responses, - calculatedFeatureTypes, - request.candidateUserIds, - intersectionIdLimit - ) - } - } - - ks.map { query => - query -> resultFuture.map(_.get(query.candidateId)) - }.toMap - } - } - - /** - * Function to merge GfsIntersectionResponse from workers into one result. - */ - private def gfsIntersectionResponseAggregator( - responseList: Seq[WorkerIntersectionResponse], - features: Seq[FeatureType], - candidates: Seq[Long], - intersectionIdLimit: Int - ): Map[Long, CachedIntersectionResult] = { - - // Map of (candidate -> features -> type -> value) - val cube = Array.fill[Int](candidates.length, features.length, 3)(0) - // Map of (candidate -> features -> intersectionIds) - val ids = Array.fill[Option[ArrayBuffer[Long]]](candidates.length, features.length)(None) - val notZero = intersectionIdLimit != 0 - - for { - response <- responseList - (features, candidateIndex) <- response.results.zipWithIndex - (workerValue, featureIndex) <- features.zipWithIndex - } { - cube(candidateIndex)(featureIndex)(CountIndex) += workerValue.count - cube(candidateIndex)(featureIndex)(LeftDegreeIndex) += workerValue.leftNodeDegree - cube(candidateIndex)(featureIndex)(RightDegreeIndex) += workerValue.rightNodeDegree - - if (notZero && workerValue.intersectionIds.nonEmpty) { - val arrayBuffer = ids(candidateIndex)(featureIndex) match { - case Some(buffer) => buffer - case None => - val buffer = ArrayBuffer[Long]() - ids(candidateIndex)(featureIndex) = Some(buffer) - buffer - } - val intersectionIds = workerValue.intersectionIds - - // Scan the intersectionId based on the Shard. The response order is consistent. - if (arrayBuffer.size < intersectionIdLimit) { - if (intersectionIds.size > intersectionIdLimit - arrayBuffer.size) { - arrayBuffer ++= intersectionIds.slice(0, intersectionIdLimit - arrayBuffer.size) - } else { - arrayBuffer ++= intersectionIds - } - } - } - } - - candidates.zipWithIndex.map { - case (candidate, candidateIndex) => - candidate -> CachedIntersectionResult(features.indices.map { featureIndex => - WorkerIntersectionValue( - cube(candidateIndex)(featureIndex)(CountIndex), - cube(candidateIndex)(featureIndex)(LeftDegreeIndex), - cube(candidateIndex)(featureIndex)(RightDegreeIndex), - ids(candidateIndex)(featureIndex).getOrElse(Nil) - ) - }) - }.toMap - } - -} - -object GetIntersectionStore { - - private[graph_feature_service] case class GetIntersectionQuery( - userId: Long, - candidateId: Long, - featureTypes: Seq[FeatureType], - presetFeatureTypes: PresetFeatureTypes, - featureTypesString: String, - calculatedFeatureTypes: Seq[FeatureType], - intersectionIdLimit: Int) - - private[graph_feature_service] object GetIntersectionQuery { - def buildQueries(request: GetIntersectionRequest): Set[GetIntersectionQuery] = { - request.candidateUserIds.toSet.map { candidateId: Long => - GetIntersectionQuery( - request.userId, - candidateId, - request.featureTypes, - request.presetFeatureTypes, - request.calculatedFeatureTypesString, - request.calculatedFeatureTypes, - request.intersectionIdLimit.getOrElse(DefaultIntersectionIdLimit) - ) - } - } - } - - // Don't return the intersectionId for better performance - private val DefaultIntersectionIdLimit = 0 - private val DefaultWorkerIntersectionResponse = WorkerIntersectionResponse() - - private val CountIndex = 0 - private val LeftDegreeIndex = 1 - private val RightDegreeIndex = 2 -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/BUILD b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/BUILD deleted file mode 100644 index 7aa2bc51c..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/BUILD +++ /dev/null @@ -1,7 +0,0 @@ -scala_library( - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", - ], -) diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/BUILD.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/BUILD.docx new file mode 100644 index 000000000..390ae1e76 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/BUILD.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/FeatureTypesCalculator.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/FeatureTypesCalculator.docx new file mode 100644 index 000000000..1efa90ba3 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/FeatureTypesCalculator.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/FeatureTypesCalculator.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/FeatureTypesCalculator.scala deleted file mode 100644 index 86caad2bf..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/FeatureTypesCalculator.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.graph_feature_service.util - -import com.twitter.graph_feature_service.thriftscala.EdgeType._ -import com.twitter.graph_feature_service.thriftscala.{FeatureType, PresetFeatureTypes} - -object FeatureTypesCalculator { - - final val DefaultTwoHop = Seq( - FeatureType(Following, FollowedBy), - FeatureType(Following, FavoritedBy), - FeatureType(Following, RetweetedBy), - FeatureType(Following, MentionedBy), - FeatureType(Following, MutualFollow), - FeatureType(Favorite, FollowedBy), - FeatureType(Favorite, FavoritedBy), - FeatureType(Favorite, RetweetedBy), - FeatureType(Favorite, MentionedBy), - FeatureType(Favorite, MutualFollow), - FeatureType(MutualFollow, FollowedBy), - FeatureType(MutualFollow, FavoritedBy), - FeatureType(MutualFollow, RetweetedBy), - FeatureType(MutualFollow, MentionedBy), - FeatureType(MutualFollow, MutualFollow) - ) - - final val SocialProofTwoHop = Seq(FeatureType(Following, FollowedBy)) - - final val HtlTwoHop = DefaultTwoHop - - final val WtfTwoHop = SocialProofTwoHop - - final val SqTwoHop = DefaultTwoHop - - final val RuxTwoHop = DefaultTwoHop - - final val MRTwoHop = DefaultTwoHop - - final val UserTypeaheadTwoHop = SocialProofTwoHop - - final val presetFeatureTypes = - (HtlTwoHop ++ WtfTwoHop ++ SqTwoHop ++ RuxTwoHop ++ MRTwoHop ++ UserTypeaheadTwoHop).toSet - - def getFeatureTypes( - presetFeatureTypes: PresetFeatureTypes, - featureTypes: Seq[FeatureType] - ): Seq[FeatureType] = { - presetFeatureTypes match { - case PresetFeatureTypes.HtlTwoHop => HtlTwoHop - case PresetFeatureTypes.WtfTwoHop => WtfTwoHop - case PresetFeatureTypes.SqTwoHop => SqTwoHop - case PresetFeatureTypes.RuxTwoHop => RuxTwoHop - case PresetFeatureTypes.MrTwoHop => MRTwoHop - case PresetFeatureTypes.UserTypeaheadTwoHop => UserTypeaheadTwoHop - case _ => featureTypes - } - } - -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/IntersectionValueCalculator.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/IntersectionValueCalculator.docx new file mode 100644 index 000000000..ac94ce0ab Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/IntersectionValueCalculator.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/IntersectionValueCalculator.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/IntersectionValueCalculator.scala deleted file mode 100644 index 4e1376cc4..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/IntersectionValueCalculator.scala +++ /dev/null @@ -1,242 +0,0 @@ -package com.twitter.graph_feature_service.util - -import com.twitter.graph_feature_service.thriftscala.{ - FeatureType, - IntersectionValue, - WorkerIntersectionValue -} -import java.nio.ByteBuffer -import scala.collection.mutable.ArrayBuffer - -/** - * Functions for computing feature values based on the values returned by constantDB. - */ -object IntersectionValueCalculator { - - /** - * Compute the size of the array in a ByteBuffer. - * Note that this function assumes the ByteBuffer is encoded using Injections.seqLong2ByteBuffer - */ - def computeArraySize(x: ByteBuffer): Int = { - x.remaining() >> 3 // divide 8 - } - - /** - * - */ - def apply(x: ByteBuffer, y: ByteBuffer, intersectionIdLimit: Int): WorkerIntersectionValue = { - - val xSize = computeArraySize(x) - val ySize = computeArraySize(y) - - val largerArray = if (xSize > ySize) x else y - val smallerArray = if (xSize > ySize) y else x - - if (intersectionIdLimit == 0) { - val result = computeIntersectionUsingBinarySearchOnLargerByteBuffer(smallerArray, largerArray) - WorkerIntersectionValue(result, xSize, ySize) - } else { - val (result, ids) = computeIntersectionWithIds(smallerArray, largerArray, intersectionIdLimit) - WorkerIntersectionValue(result, xSize, ySize, ids) - } - } - - /** - * Note that this function assumes the ByteBuffer is encoded using Injections.seqLong2ByteBuffer - * - */ - def computeIntersectionUsingBinarySearchOnLargerByteBuffer( - smallArray: ByteBuffer, - largeArray: ByteBuffer - ): Int = { - var res: Int = 0 - var i: Int = 0 - - while (i < smallArray.remaining()) { - if (binarySearch(largeArray, smallArray.getLong(i)) >= 0) { - res += 1 - } - i += 8 - } - res - } - - def computeIntersectionWithIds( - smallArray: ByteBuffer, - largeArray: ByteBuffer, - intersectionLimit: Int - ): (Int, Seq[Long]) = { - var res: Int = 0 - var i: Int = 0 - // Most of the intersectionLimit is smaller than default size: 16 - val idBuffer = ArrayBuffer[Long]() - - while (i < smallArray.remaining()) { - val value = smallArray.getLong(i) - if (binarySearch(largeArray, value) >= 0) { - res += 1 - // Always get the smaller ids - if (idBuffer.size < intersectionLimit) { - idBuffer += value - } - } - i += 8 - } - (res, idBuffer) - } - - /** - * Note that this function assumes the ByteBuffer is encoded using Injections.seqLong2ByteBuffer - * - */ - private[util] def binarySearch(arr: ByteBuffer, value: Long): Int = { - var start = 0 - var end = arr.remaining() - - while (start <= end && start < arr.remaining()) { - val mid = ((start + end) >> 1) & ~7 // take mid - mid % 8 - if (arr.getLong(mid) == value) { - return mid // return the index of the value - } else if (arr.getLong(mid) < value) { - start = mid + 8 - } else { - end = mid - 1 - } - } - // if not existed, return -1 - -1 - } - - /** - * TODO: for now it only computes intersection size. Will add more feature types (e.g., dot - * product, maximum value). - * - * NOTE that this function assumes both x and y are SORTED arrays. - * In graph feature service, the sorting is done in the offline Scalding job. - * - * @param x source user's array - * @param y candidate user's array - * @param featureType feature type - * @return - */ - def apply(x: Array[Long], y: Array[Long], featureType: FeatureType): IntersectionValue = { - - val xSize = x.length - val ySize = y.length - - val intersection = - if (xSize.min(ySize) * math.log(xSize.max(ySize)) < (xSize + ySize).toDouble) { - if (xSize < ySize) { - computeIntersectionUsingBinarySearchOnLargerArray(x, y) - } else { - computeIntersectionUsingBinarySearchOnLargerArray(y, x) - } - } else { - computeIntersectionUsingListMerging(x, y) - } - - IntersectionValue( - featureType, - Some(intersection.toInt), - None, // return None for now - Some(xSize), - Some(ySize) - ) - } - - /** - * Function for computing the intersections of two SORTED arrays by list merging. - * - * @param x one array - * @param y another array - * @param ordering ordering function for comparing values of T - * @tparam T type - * @return The intersection size and the list of intersected elements - */ - private[util] def computeIntersectionUsingListMerging[T]( - x: Array[T], - y: Array[T] - )( - implicit ordering: Ordering[T] - ): Int = { - - var res: Int = 0 - var i: Int = 0 - var j: Int = 0 - - while (i < x.length && j < y.length) { - val comp = ordering.compare(x(i), y(j)) - if (comp > 0) j += 1 - else if (comp < 0) i += 1 - else { - res += 1 - i += 1 - j += 1 - } - } - res - } - - /** - * Function for computing the intersections of two arrays by binary search on the larger array. - * Note that the larger array MUST be SORTED. - * - * @param smallArray smaller array - * @param largeArray larger array - * @param ordering ordering function for comparing values of T - * @tparam T type - * - * @return The intersection size and the list of intersected elements - */ - private[util] def computeIntersectionUsingBinarySearchOnLargerArray[T]( - smallArray: Array[T], - largeArray: Array[T] - )( - implicit ordering: Ordering[T] - ): Int = { - var res: Int = 0 - var i: Int = 0 - while (i < smallArray.length) { - val currentValue: T = smallArray(i) - if (binarySearch(largeArray, currentValue) >= 0) { - res += 1 - } - i += 1 - } - res - } - - /** - * Function for doing the binary search - * - * @param arr array - * @param value the target value for searching - * @param ordering ordering function - * @tparam T type - * @return the index of element in the larger array. - * If there is no such element in the array, return -1. - */ - private[util] def binarySearch[T]( - arr: Array[T], - value: T - )( - implicit ordering: Ordering[T] - ): Int = { - var start = 0 - var end = arr.length - 1 - - while (start <= end) { - val mid = (start + end) >> 1 - val comp = ordering.compare(arr(mid), value) - if (comp == 0) { - return mid // return the index of the value - } else if (comp < 0) { - start = mid + 1 - } else { - end = mid - 1 - } - } - // if not existed, return -1 - -1 - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/BUILD.bazel b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/BUILD.bazel deleted file mode 100644 index 7f0d975d7..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -scala_library( - sources = ["**/*.scala"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "discovery-common/src/main/scala/com/twitter/discovery/common/stats", - "finatra-internal/decider/src/main/scala", - "finatra-internal/gizmoduck/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "finatra/inject/inject-utils/src/main/scala", - "frigate/frigate-common:constdb_util", - "graph-feature-service/src/main/resources", - "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common", - "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util", - "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "servo/request/src/main/scala", - "twitter-server-internal/src/main/scala", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/BUILD.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/BUILD.docx new file mode 100644 index 000000000..d6efdef2a Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/BUILD.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/Main.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/Main.docx new file mode 100644 index 000000000..a5f3d5b04 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/Main.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/Main.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/Main.scala deleted file mode 100644 index 10e8ec0e2..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/Main.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.graph_feature_service.worker - -import com.google.inject.Module -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.gizmoduck.modules.TimerModule -import com.twitter.finatra.mtls.thriftmux.Mtls -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.finatra.thrift.filters.{ - LoggingMDCFilter, - StatsFilter, - ThriftMDCFilter, - TraceIdMDCFilter -} -import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule -import com.twitter.finatra.thrift.routing.ThriftRouter -import com.twitter.graph_feature_service.thriftscala -import com.twitter.graph_feature_service.worker.controllers.WorkerController -import com.twitter.graph_feature_service.worker.handlers.WorkerWarmupHandler -import com.twitter.graph_feature_service.worker.modules.{ - GraphContainerProviderModule, - WorkerFlagModule -} -import com.twitter.graph_feature_service.worker.util.GraphContainer -import com.twitter.inject.thrift.modules.ThriftClientIdModule -import com.twitter.util.Await - -object Main extends WorkerMain - -class WorkerMain extends ThriftServer with Mtls { - - override val name = "graph_feature_service-worker" - - override val modules: Seq[Module] = { - Seq( - WorkerFlagModule, - DeciderModule, - TimerModule, - ThriftClientIdModule, - GraphContainerProviderModule, - new MtlsThriftWebFormsModule[thriftscala.Worker.MethodPerEndpoint](this) - ) - } - - override def configureThrift(router: ThriftRouter): Unit = { - router - .filter[LoggingMDCFilter] - .filter[TraceIdMDCFilter] - .filter[ThriftMDCFilter] - .filter[StatsFilter] - .add[WorkerController] - } - - override protected def warmup(): Unit = { - val graphContainer = injector.instance[GraphContainer] - Await.result(graphContainer.warmup) - handle[WorkerWarmupHandler]() - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/controllers/WorkerController.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/controllers/WorkerController.docx new file mode 100644 index 000000000..8ae79e8b0 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/controllers/WorkerController.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/controllers/WorkerController.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/controllers/WorkerController.scala deleted file mode 100644 index f30305b2c..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/controllers/WorkerController.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.graph_feature_service.worker.controllers - -import com.twitter.discovery.common.stats.DiscoveryStatsFilter -import com.twitter.finagle.Service -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.thrift.Controller -import com.twitter.graph_feature_service.thriftscala -import com.twitter.graph_feature_service.thriftscala.Worker.GetIntersection -import com.twitter.graph_feature_service.thriftscala._ -import com.twitter.graph_feature_service.worker.handlers._ -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class WorkerController @Inject() ( - workerGetIntersectionHandler: WorkerGetIntersectionHandler -)( - implicit statsReceiver: StatsReceiver) - extends Controller(thriftscala.Worker) { - - // use DiscoveryStatsFilter to filter out exceptions out of our control - private val getIntersectionService: Service[ - WorkerIntersectionRequest, - WorkerIntersectionResponse - ] = - new DiscoveryStatsFilter[WorkerIntersectionRequest, WorkerIntersectionResponse]( - statsReceiver.scope("srv").scope("get_intersection") - ).andThen(Service.mk(workerGetIntersectionHandler)) - - val getIntersection: Service[GetIntersection.Args, WorkerIntersectionResponse] = { args => - getIntersectionService(args.request).onFailure { throwable => - logger.error(s"Failure to get intersection for request $args.", throwable) - } - } - - handle(GetIntersection) { getIntersection } - -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerGetIntersectionHandler.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerGetIntersectionHandler.docx new file mode 100644 index 000000000..0ddb54fea Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerGetIntersectionHandler.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerGetIntersectionHandler.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerGetIntersectionHandler.scala deleted file mode 100644 index 7acf8b1d3..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerGetIntersectionHandler.scala +++ /dev/null @@ -1,105 +0,0 @@ -package com.twitter.graph_feature_service.worker.handlers - -import com.twitter.finagle.stats.{Stat, StatsReceiver} -import com.twitter.graph_feature_service.thriftscala.{ - WorkerIntersectionRequest, - WorkerIntersectionResponse, - WorkerIntersectionValue -} -import com.twitter.graph_feature_service.util.{FeatureTypesCalculator, IntersectionValueCalculator} -import com.twitter.graph_feature_service.util.IntersectionValueCalculator._ -import com.twitter.graph_feature_service.worker.util.GraphContainer -import com.twitter.servo.request.RequestHandler -import com.twitter.util.Future -import java.nio.ByteBuffer -import javax.inject.{Inject, Singleton} - -@Singleton -class WorkerGetIntersectionHandler @Inject() ( - graphContainer: GraphContainer, - statsReceiver: StatsReceiver) - extends RequestHandler[WorkerIntersectionRequest, WorkerIntersectionResponse] { - - import WorkerGetIntersectionHandler._ - - private val stats: StatsReceiver = statsReceiver.scope("srv/get_intersection") - private val numCandidatesCount = stats.counter("total_num_candidates") - private val toPartialGraphQueryStat = stats.stat("to_partial_graph_query_latency") - private val fromPartialGraphQueryStat = stats.stat("from_partial_graph_query_latency") - private val intersectionCalculationStat = stats.stat("computation_latency") - - override def apply(request: WorkerIntersectionRequest): Future[WorkerIntersectionResponse] = { - - numCandidatesCount.incr(request.candidateUserIds.length) - - val userId = request.userId - - // NOTE: do not change the order of candidates - val candidateIds = request.candidateUserIds - - // NOTE: do not change the order of features - val featureTypes = - FeatureTypesCalculator.getFeatureTypes(request.presetFeatureTypes, request.featureTypes) - - val leftEdges = featureTypes.map(_.leftEdgeType).distinct - val rightEdges = featureTypes.map(_.rightEdgeType).distinct - - val rightEdgeMap = Stat.time(toPartialGraphQueryStat) { - rightEdges.map { rightEdge => - val map = graphContainer.toPartialMap.get(rightEdge) match { - case Some(graph) => - candidateIds.flatMap { candidateId => - graph.apply(candidateId).map(candidateId -> _) - }.toMap - case None => - Map.empty[Long, ByteBuffer] - } - rightEdge -> map - }.toMap - } - - val leftEdgeMap = Stat.time(fromPartialGraphQueryStat) { - leftEdges.flatMap { leftEdge => - graphContainer.toPartialMap.get(leftEdge).flatMap(_.apply(userId)).map(leftEdge -> _) - }.toMap - } - - val res = Stat.time(intersectionCalculationStat) { - WorkerIntersectionResponse( - // NOTE that candidate ordering is important - candidateIds.map { candidateId => - // NOTE that the featureTypes ordering is important - featureTypes.map { - featureType => - val leftNeighborsOpt = leftEdgeMap.get(featureType.leftEdgeType) - val rightNeighborsOpt = - rightEdgeMap.get(featureType.rightEdgeType).flatMap(_.get(candidateId)) - - if (leftNeighborsOpt.isEmpty && rightNeighborsOpt.isEmpty) { - EmptyWorkerIntersectionValue - } else if (rightNeighborsOpt.isEmpty) { - EmptyWorkerIntersectionValue.copy( - leftNodeDegree = computeArraySize(leftNeighborsOpt.get) - ) - } else if (leftNeighborsOpt.isEmpty) { - EmptyWorkerIntersectionValue.copy( - rightNodeDegree = computeArraySize(rightNeighborsOpt.get) - ) - } else { - IntersectionValueCalculator( - leftNeighborsOpt.get, - rightNeighborsOpt.get, - request.intersectionIdLimit) - } - } - } - ) - } - - Future.value(res) - } -} - -object WorkerGetIntersectionHandler { - val EmptyWorkerIntersectionValue: WorkerIntersectionValue = WorkerIntersectionValue(0, 0, 0, Nil) -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerWarmupHandler.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerWarmupHandler.docx new file mode 100644 index 000000000..4e7b7cf6e Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerWarmupHandler.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerWarmupHandler.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerWarmupHandler.scala deleted file mode 100644 index d89d215c1..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerWarmupHandler.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.graph_feature_service.worker.handlers - -import com.twitter.finatra.thrift.routing.ThriftWarmup -import com.twitter.inject.Logging -import com.twitter.inject.utils.Handler -import javax.inject.{Inject, Singleton} - -@Singleton -class WorkerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler with Logging { - - override def handle(): Unit = { - info("Warmup Done!") - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/GraphContainerProviderModule.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/GraphContainerProviderModule.docx new file mode 100644 index 000000000..0e9d6f494 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/GraphContainerProviderModule.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/GraphContainerProviderModule.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/GraphContainerProviderModule.scala deleted file mode 100644 index 6c66922d4..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/GraphContainerProviderModule.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.graph_feature_service.worker.modules - -import com.google.inject.Provides -import com.twitter.concurrent.AsyncSemaphore -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.graph_feature_service.common.Configs._ -import com.twitter.graph_feature_service.worker.util -import com.twitter.graph_feature_service.worker.util.AutoUpdatingGraph -import com.twitter.graph_feature_service.worker.util.FollowedByPartialValueGraph -import com.twitter.graph_feature_service.worker.util.FollowingPartialValueGraph -import com.twitter.graph_feature_service.worker.util.GraphContainer -import com.twitter.graph_feature_service.worker.util.GraphKey -import com.twitter.graph_feature_service.worker.util.MutualFollowPartialValueGraph -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.util.Timer -import javax.inject.Singleton - -object GraphContainerProviderModule extends TwitterModule { - - @Provides - @Singleton - def provideAutoUpdatingGraphs( - @Flag(WorkerFlagNames.HdfsCluster) hdfsCluster: String, - @Flag(WorkerFlagNames.HdfsClusterUrl) hdfsClusterUrl: String, - @Flag(WorkerFlagNames.ShardId) shardId: Int - )( - implicit statsReceiver: StatsReceiver, - timer: Timer - ): GraphContainer = { - - // NOTE that we do not load some the graphs for saving RAM at this moment. - val enabledGraphPaths: Map[GraphKey, String] = - Map( - FollowingPartialValueGraph -> FollowOutValPath, - FollowedByPartialValueGraph -> FollowInValPath - ) - - // Only allow one graph to update at the same time. - val sharedSemaphore = new AsyncSemaphore(1) - - val graphs: Map[GraphKey, AutoUpdatingGraph] = - enabledGraphPaths.map { - case (graphKey, path) => - graphKey -> AutoUpdatingGraph( - dataPath = getHdfsPath(path), - hdfsCluster = hdfsCluster, - hdfsClusterUrl = hdfsClusterUrl, - shard = shardId, - minimumSizeForCompleteGraph = 1e6.toLong, - sharedSemaphore = Some(sharedSemaphore) - )( - statsReceiver - .scope("graphs") - .scope(graphKey.getClass.getSimpleName), - timer - ) - } - - util.GraphContainer(graphs) - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/WorkerFlagModule.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/WorkerFlagModule.docx new file mode 100644 index 000000000..e79cc1f27 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/WorkerFlagModule.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/WorkerFlagModule.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/WorkerFlagModule.scala deleted file mode 100644 index 2188d169b..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/WorkerFlagModule.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.graph_feature_service.worker.modules - -import com.twitter.inject.TwitterModule - -object WorkerFlagNames { - final val ServiceRole = "service.role" - final val ServiceEnv = "service.env" - final val ShardId = "service.shardId" - final val NumShards = "service.numShards" - final val HdfsCluster = "service.hdfsCluster" - final val HdfsClusterUrl = "service.hdfsClusterUrl" -} - -/** - * Initializes references to the flag values defined in the aurora.deploy file. - * To check what the flag values are initialized in runtime, search FlagsModule in stdout - */ -object WorkerFlagModule extends TwitterModule { - - import WorkerFlagNames._ - - flag[Int](ShardId, "Shard Id") - - flag[Int](NumShards, "Num of Graph Shards") - - flag[String](ServiceRole, "Service Role") - - flag[String](ServiceEnv, "Service Env") - - flag[String](HdfsCluster, "Hdfs cluster to download graph files from") - - flag[String](HdfsClusterUrl, "Hdfs cluster url to download graph files from") -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/AutoUpdatingGraph.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/AutoUpdatingGraph.docx new file mode 100644 index 000000000..ebe131e3f Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/AutoUpdatingGraph.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/AutoUpdatingGraph.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/AutoUpdatingGraph.scala deleted file mode 100644 index e5c1c367f..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/AutoUpdatingGraph.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.graph_feature_service.worker.util - -import com.twitter.bijection.Injection -import com.twitter.concurrent.AsyncSemaphore -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.constdb_util.{ - AutoUpdatingReadOnlyGraph, - ConstDBImporter, - Injections -} -import com.twitter.graph_feature_service.common.Configs -import com.twitter.util.{Duration, Future, Timer} -import java.nio.ByteBuffer - -/** - * @param dataPath the path to the data on HDFS - * @param hdfsCluster cluster where we check for updates and download graph files from - * @param hdfsClusterUrl url to HDFS cluster - * @param shard The shard of the graph to download - * @param minimumSizeForCompleteGraph minimumSize for complete graph - otherwise we don't load it - * @param updateIntervalMin The interval after which the first update is tried and the interval between such updates - * @param updateIntervalMax the maximum time before an update is triggered - * @param deleteInterval The interval after which older data is deleted from disk - * @param sharedSemaphore The semaphore controls the number of graph loads at same time on the instance. - */ -case class AutoUpdatingGraph( - dataPath: String, - hdfsCluster: String, - hdfsClusterUrl: String, - shard: Int, - minimumSizeForCompleteGraph: Long, - updateIntervalMin: Duration = 1.hour, - updateIntervalMax: Duration = 12.hours, - deleteInterval: Duration = 2.seconds, - sharedSemaphore: Option[AsyncSemaphore] = None -)( - implicit statsReceiver: StatsReceiver, - timer: Timer) - extends AutoUpdatingReadOnlyGraph[Long, ByteBuffer]( - hdfsCluster, - hdfsClusterUrl, - shard, - minimumSizeForCompleteGraph, - updateIntervalMin, - updateIntervalMax, - deleteInterval, - sharedSemaphore - ) - with ConstDBImporter[Long, ByteBuffer] { - - override def numGraphShards: Int = Configs.NumGraphShards - - override def basePath: String = dataPath - - override val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint - - override val valueInj: Injection[ByteBuffer, ByteBuffer] = Injection.identity - - override def get(targetId: Long): Future[Option[ByteBuffer]] = - super - .get(targetId) - .map { res => - res.foreach(r => arraySizeStat.add(r.remaining())) - res - } - - private val arraySizeStat = stats.scope("get").stat("size") -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GfsQuery.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GfsQuery.docx new file mode 100644 index 000000000..a8fec3f33 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GfsQuery.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GfsQuery.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GfsQuery.scala deleted file mode 100644 index e5d822e2b..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GfsQuery.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.graph_feature_service.worker.util - -import com.twitter.graph_feature_service.thriftscala.EdgeType - -sealed trait GfsQuery { - def edgeType: EdgeType - def userId: Long -} - -/** - * Search for edges for any users to users in local partition. - */ -case class ToPartialQuery(edgeType: EdgeType, userId: Long) extends GfsQuery - diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphContainer.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphContainer.docx new file mode 100644 index 000000000..88f209437 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphContainer.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphContainer.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphContainer.scala deleted file mode 100644 index 9ac626bb9..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphContainer.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.graph_feature_service.worker.util - -import com.twitter.graph_feature_service.thriftscala.EdgeType -import com.twitter.util.Future - -case class GraphContainer( - graphs: Map[GraphKey, AutoUpdatingGraph]) { - - final val toPartialMap: Map[EdgeType, AutoUpdatingGraph] = - graphs.collect { - case (partialValueGraph: PartialValueGraph, graph) => - partialValueGraph.edgeType -> graph - } - - // load all the graphs from constantDB format to memory - def warmup: Future[Unit] = { - Future.collect(graphs.mapValues(_.warmup())).unit - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphKey.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphKey.docx new file mode 100644 index 000000000..3a363e200 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphKey.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphKey.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphKey.scala deleted file mode 100644 index 2b174eb8c..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphKey.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.graph_feature_service.worker.util - -import com.twitter.graph_feature_service.thriftscala.EdgeType -import com.twitter.graph_feature_service.thriftscala.EdgeType._ - -sealed trait GraphKey { - - def edgeType: EdgeType -} - -sealed trait PartialValueGraph extends GraphKey - -/** - * Follow Graphs - */ -object FollowingPartialValueGraph extends PartialValueGraph { - - override def edgeType: EdgeType = Following -} - -object FollowedByPartialValueGraph extends PartialValueGraph { - - override def edgeType: EdgeType = FollowedBy -} - -/** - * Mutual Follow Graphs - */ -object MutualFollowPartialValueGraph extends PartialValueGraph { - - override def edgeType: EdgeType = MutualFollow -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphType.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphType.docx new file mode 100644 index 000000000..41fbf0445 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphType.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphType.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphType.scala deleted file mode 100644 index a2a7634af..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphType.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.graph_feature_service.worker.util - -//These classes are to help the GraphContainer choose the right data structure to answer queries -sealed trait GraphType - -object FollowGraph extends GraphType - -object FavoriteGraph extends GraphType - -object RetweetGraph extends GraphType - -object ReplyGraph extends GraphType - -object MentionGraph extends GraphType - -object MutualFollowGraph extends GraphType diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/BUILD.bazel b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/BUILD.bazel deleted file mode 100644 index 66e27fb7f..000000000 --- a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/BUILD.bazel +++ /dev/null @@ -1,66 +0,0 @@ -scala_library( - platform = "java8", - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:core", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/constdb_util", - "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common", - "src/scala/com/twitter/interaction_graph/scio/agg_all:interaction_graph_history_aggregated_edge_snapshot-scala", - "src/scala/com/twitter/interaction_graph/scio/ml/scores:real_graph_in_scores-scala", - "src/scala/com/twitter/pluck/source/user_audits:user_audit_final-scala", - "src/scala/com/twitter/scalding_internal/dalv2", - "src/scala/com/twitter/scalding_internal/job", - "src/scala/com/twitter/scalding_internal/job/analytics_batch", - ], -) - -scalding_job( - name = "graph_feature_service_adhoc_job", - main = "com.twitter.graph_feature_service.scalding.GraphFeatureServiceAdhocApp", - args = [ - "--date 2022-10-24", - ], - config = [ - ("hadoop.map.jvm.total-memory", "3072m"), - ("hadoop.reduce.jvm.total-memory", "3072m"), - ("hadoop.submitter.jvm.total-memory", "5120m"), - ("submitter.tier", "preemptible"), - ], - contact = "recos-platform-alerts@twitter.com", - hadoop_cluster = "atla-proc", - hadoop_properties = [("mapreduce.job.hdfs-servers", "/atla/proc/user/cassowary")], - platform = "java8", - role = "cassowary", - runtime_platform = "java8", - tags = [ - "bazel-compatible:migrated", - "bazel-only", - ], - dependencies = [":scalding"], -) - -scalding_job( - name = "graph_feature_service_daily_job", - main = "com.twitter.graph_feature_service.scalding.GraphFeatureServiceScheduledApp", - config = [ - ("hadoop.map.jvm.total-memory", "3072m"), - ("hadoop.reduce.jvm.total-memory", "3072m"), - ("hadoop.submitter.jvm.total-memory", "5120m"), - ("submitter.tier", "preemptible"), - ], - contact = "recos-platform-alerts@twitter.com", - cron = "01,31 * * * *", - hadoop_cluster = "atla-proc", - hadoop_properties = [("mapreduce.job.hdfs-servers", "/atla/proc/user/cassowary")], - platform = "java8", - role = "cassowary", - runtime_platform = "java8", - tags = [ - "bazel-compatible:migrated", - "bazel-only", - ], - dependencies = [":scalding"], -) diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/BUILD.docx b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/BUILD.docx new file mode 100644 index 000000000..2a503817f Binary files /dev/null and b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/BUILD.docx differ diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/EdgeFeature.docx b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/EdgeFeature.docx new file mode 100644 index 000000000..f07e51761 Binary files /dev/null and b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/EdgeFeature.docx differ diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/EdgeFeature.scala b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/EdgeFeature.scala deleted file mode 100644 index 76005c3ae..000000000 --- a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/EdgeFeature.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.graph_feature_service.scalding - -case class EdgeFeature( - realGraphScore: Float, - followScore: Option[Float] = None, - mutualFollowScore: Option[Float] = None, - favoriteScore: Option[Float] = None, - retweetScore: Option[Float] = None, - mentionScore: Option[Float] = None) diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceAppBase.docx b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceAppBase.docx new file mode 100644 index 000000000..4c00ac2ec Binary files /dev/null and b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceAppBase.docx differ diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceAppBase.scala b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceAppBase.scala deleted file mode 100644 index 993dc5a39..000000000 --- a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceAppBase.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.graph_feature_service.scalding - -import com.twitter.scalding._ -import com.twitter.scalding_internal.job.TwitterExecutionApp -import com.twitter.scalding_internal.job.analytics_batch.{ - AnalyticsBatchExecution, - AnalyticsBatchExecutionArgs, - BatchDescription, - BatchFirstTime, - BatchIncrement, - TwitterScheduledExecutionApp -} -import java.util.TimeZone - -/** - * Each job only needs to implement this runOnDateRange() function. It makes it easier for testing. - */ -trait GraphFeatureServiceBaseJob { - implicit val timeZone: TimeZone = DateOps.UTC - implicit val dateParser: DateParser = DateParser.default - - def runOnDateRange( - enableValueGraphs: Option[Boolean] = None, - enableKeyGraphs: Option[Boolean] = None - )( - implicit dateRange: DateRange, - timeZone: TimeZone, - uniqueID: UniqueID - ): Execution[Unit] - - /** - * Print customized counters in the log - */ - def printerCounters[T](execution: Execution[T]): Execution[Unit] = { - execution.getCounters - .flatMap { - case (_, counters) => - counters.toMap.toSeq - .sortBy(e => (e._1.group, e._1.counter)) - .foreach { - case (statKey, value) => - println(s"${statKey.group}\t${statKey.counter}\t$value") - } - Execution.unit - } - } -} - -/** - * Trait that wraps things about adhoc jobs. - */ -trait GraphFeatureServiceAdhocBaseApp extends TwitterExecutionApp with GraphFeatureServiceBaseJob { - override def job: Execution[Unit] = Execution.withId { implicit uniqueId => - Execution.getArgs.flatMap { args: Args => - implicit val dateRange: DateRange = DateRange.parse(args.list("date"))(timeZone, dateParser) - printerCounters(runOnDateRange()) - } - } -} - -/** - * Trait that wraps things about scheduled jobs. - * - * A new daily app only needs to declare the starting date. - */ -trait GraphFeatureServiceScheduledBaseApp - extends TwitterScheduledExecutionApp - with GraphFeatureServiceBaseJob { - - def firstTime: RichDate // for example: RichDate("2018-02-21") - - def batchIncrement: Duration = Days(1) - - override def scheduledJob: Execution[Unit] = Execution.withId { implicit uniqueId => - val analyticsArgs = AnalyticsBatchExecutionArgs( - batchDesc = BatchDescription(getClass.getName), - firstTime = BatchFirstTime(firstTime), - batchIncrement = BatchIncrement(batchIncrement) - ) - - AnalyticsBatchExecution(analyticsArgs) { implicit dateRange => - printerCounters(runOnDateRange()) - } - } -} diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceApps.docx b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceApps.docx new file mode 100644 index 000000000..cda265ba1 Binary files /dev/null and b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceApps.docx differ diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceApps.scala b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceApps.scala deleted file mode 100644 index e6086526b..000000000 --- a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceApps.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.graph_feature_service.scalding - -import com.twitter.scalding.DateRange -import com.twitter.scalding.Execution -import com.twitter.scalding.RichDate -import com.twitter.scalding.UniqueID -import java.util.Calendar -import java.util.TimeZone -import sun.util.calendar.BaseCalendar - -/** - * To launch an adhoc run: - * - scalding remote run --target graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding:graph_feature_service_adhoc_job - */ -object GraphFeatureServiceAdhocApp - extends GraphFeatureServiceMainJob - with GraphFeatureServiceAdhocBaseApp {} - -/** - * To schedule the job, upload the workflows config (only required for the first time and subsequent config changes): - * scalding workflow upload --jobs graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding:graph_feature_service_daily_job --autoplay --build-cron-schedule "20 23 1 * *" - * You can then build from the UI by clicking "Build" and pasting in your remote branch, or leave it empty if you're redeploying from master. - * The workflows config above should automatically trigger once each month. - */ -object GraphFeatureServiceScheduledApp - extends GraphFeatureServiceMainJob - with GraphFeatureServiceScheduledBaseApp { - override def firstTime: RichDate = RichDate("2018-05-18") - - override def runOnDateRange( - enableValueGraphs: Option[Boolean], - enableKeyGraphs: Option[Boolean] - )( - implicit dateRange: DateRange, - timeZone: TimeZone, - uniqueID: UniqueID - ): Execution[Unit] = { - // Only run the value Graphs on Tuesday, Thursday, Saturday - val overrideEnableValueGraphs = { - val dayOfWeek = dateRange.start.toCalendar.get(Calendar.DAY_OF_WEEK) - dayOfWeek == BaseCalendar.TUESDAY | - dayOfWeek == BaseCalendar.THURSDAY | - dayOfWeek == BaseCalendar.SATURDAY - } - - super.runOnDateRange( - Some(true), - Some(false) // disable key Graphs since we are not using them in production - ) - } -} diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceMainJob.docx b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceMainJob.docx new file mode 100644 index 000000000..e6e9ae8a9 Binary files /dev/null and b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceMainJob.docx differ diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceMainJob.scala b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceMainJob.scala deleted file mode 100644 index f0446285c..000000000 --- a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceMainJob.scala +++ /dev/null @@ -1,297 +0,0 @@ -package com.twitter.graph_feature_service.scalding - -import com.twitter.bijection.Injection -import com.twitter.frigate.common.constdb_util.Injections -import com.twitter.frigate.common.constdb_util.ScaldingUtil -import com.twitter.graph_feature_service.common.Configs -import com.twitter.graph_feature_service.common.Configs._ -import com.twitter.interaction_graph.scio.agg_all.InteractionGraphHistoryAggregatedEdgeSnapshotScalaDataset -import com.twitter.interaction_graph.scio.ml.scores.RealGraphInScoresScalaDataset -import com.twitter.interaction_graph.thriftscala.FeatureName -import com.twitter.interaction_graph.thriftscala.{EdgeFeature => TEdgeFeature} -import com.twitter.pluck.source.user_audits.UserAuditFinalScalaDataset -import com.twitter.scalding.DateRange -import com.twitter.scalding.Days -import com.twitter.scalding.Execution -import com.twitter.scalding.Stat -import com.twitter.scalding.UniqueID -import com.twitter.scalding.typed.TypedPipe -import com.twitter.scalding_internal.dalv2.DAL -import com.twitter.scalding_internal.dalv2.remote_access.AllowCrossClusterSameDC -import com.twitter.scalding_internal.multiformat.format.keyval.KeyVal -import com.twitter.util.Time -import com.twitter.wtf.candidate.thriftscala.CandidateSeq -import java.nio.ByteBuffer -import java.util.TimeZone - -trait GraphFeatureServiceMainJob extends GraphFeatureServiceBaseJob { - - // keeping hdfsPath as a separate variable in order to override it in unit tests - protected val hdfsPath: String = BaseHdfsPath - - protected def getShardIdForUser(userId: Long): Int = shardForUser(userId) - - protected implicit val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint - - protected implicit val valueInj: Injection[Long, ByteBuffer] = Injections.long2ByteBuffer - - protected val bufferSize: Int = 1 << 26 - - protected val maxNumKeys: Int = 1 << 24 - - protected val numReducers: Int = NumGraphShards - - protected val outputStreamBufferSize: Int = 1 << 26 - - protected final val shardingByKey = { (k: Long, _: Long) => - getShardIdForUser(k) - } - - protected final val shardingByValue = { (_: Long, v: Long) => - getShardIdForUser(v) - } - - private def writeGraphToDB( - graph: TypedPipe[(Long, Long)], - shardingFunction: (Long, Long) => Int, - path: String - )( - implicit dateRange: DateRange - ): Execution[TypedPipe[(Int, Unit)]] = { - ScaldingUtil - .writeConstDB[Long, Long]( - graph.withDescription(s"sharding $path"), - shardingFunction, - shardId => - getTimedHdfsShardPath( - shardId, - getHdfsPath(path, Some(hdfsPath)), - Time.fromMilliseconds(dateRange.end.timestamp) - ), - Int.MaxValue, - bufferSize, - maxNumKeys, - numReducers, - outputStreamBufferSize - )( - keyInj, - valueInj, - Ordering[(Long, Long)] - ) - .forceToDiskExecution - } - - def extractFeature( - featureList: Seq[TEdgeFeature], - featureName: FeatureName - ): Option[Float] = { - featureList - .find(_.name == featureName) - .map(_.tss.ewma.toFloat) - .filter(_ > 0.0) - } - - /** - * Function to extract a subgraph (e.g., follow graph) from real graph and take top K by real graph - * weight. - * - * @param input input real graph - * @param edgeFilter filter function to only get the edges needed (e.g., only follow edges) - * @param counter counter - * @return a subgroup that contains topK, e.g., follow graph for each user. - */ - private def getSubGraph( - input: TypedPipe[(Long, Long, EdgeFeature)], - edgeFilter: EdgeFeature => Boolean, - counter: Stat - ): TypedPipe[(Long, Long)] = { - input - .filter(c => edgeFilter(c._3)) - .map { - case (srcId, destId, features) => - (srcId, (destId, features.realGraphScore)) - } - .group - // auto reducer estimation only allocates 15 reducers, so setting an explicit number here - .withReducers(2000) - .sortedReverseTake(TopKRealGraph)(Ordering.by(_._2)) - .flatMap { - case (srcId, topKNeighbors) => - counter.inc() - topKNeighbors.map { - case (destId, _) => - (srcId, destId) - } - } - } - - def getMauIds()(implicit dateRange: DateRange, uniqueID: UniqueID): TypedPipe[Long] = { - val numMAUs = Stat("NUM_MAUS") - val uniqueMAUs = Stat("UNIQUE_MAUS") - - DAL - .read(UserAuditFinalScalaDataset) - .withRemoteReadPolicy(AllowCrossClusterSameDC) - .toTypedPipe - .collect { - case user_audit if user_audit.isValid => - numMAUs.inc() - user_audit.userId - } - .distinct - .map { u => - uniqueMAUs.inc() - u - } - } - - def getRealGraphWithMAUOnly( - implicit dateRange: DateRange, - timeZone: TimeZone, - uniqueID: UniqueID - ): TypedPipe[(Long, Long, EdgeFeature)] = { - val numMAUs = Stat("NUM_MAUS") - val uniqueMAUs = Stat("UNIQUE_MAUS") - - val monthlyActiveUsers = DAL - .read(UserAuditFinalScalaDataset) - .withRemoteReadPolicy(AllowCrossClusterSameDC) - .toTypedPipe - .collect { - case user_audit if user_audit.isValid => - numMAUs.inc() - user_audit.userId - } - .distinct - .map { u => - uniqueMAUs.inc() - u - } - .asKeys - - val realGraphAggregates = DAL - .readMostRecentSnapshot( - InteractionGraphHistoryAggregatedEdgeSnapshotScalaDataset, - dateRange.embiggen(Days(5))) - .withRemoteReadPolicy(AllowCrossClusterSameDC) - .toTypedPipe - .map { edge => - val featureList = edge.features - val edgeFeature = EdgeFeature( - edge.weight.getOrElse(0.0).toFloat, - extractFeature(featureList, FeatureName.NumMutualFollows), - extractFeature(featureList, FeatureName.NumFavorites), - extractFeature(featureList, FeatureName.NumRetweets), - extractFeature(featureList, FeatureName.NumMentions) - ) - (edge.sourceId, (edge.destinationId, edgeFeature)) - } - .join(monthlyActiveUsers) - .map { - case (srcId, ((destId, feature), _)) => - (destId, (srcId, feature)) - } - .join(monthlyActiveUsers) - .map { - case (destId, ((srcId, feature), _)) => - (srcId, destId, feature) - } - realGraphAggregates - } - - def getTopKFollowGraph( - implicit dateRange: DateRange, - timeZone: TimeZone, - uniqueID: UniqueID - ): TypedPipe[(Long, Long)] = { - val followGraphMauStat = Stat("NumFollowEdges_MAU") - val mau: TypedPipe[Long] = getMauIds() - DAL - .readMostRecentSnapshot(RealGraphInScoresScalaDataset, dateRange.embiggen(Days(7))) - .withRemoteReadPolicy(AllowCrossClusterSameDC) - .toTypedPipe - .groupBy(_.key) - .join(mau.asKeys) - .withDescription("filtering srcId by mau") - .flatMap { - case (_, (KeyVal(srcId, CandidateSeq(candidates)), _)) => - followGraphMauStat.inc() - val topK = candidates.sortBy(-_.score).take(TopKRealGraph) - topK.map { c => (srcId, c.userId) } - } - } - - override def runOnDateRange( - enableValueGraphs: Option[Boolean], - enableKeyGraphs: Option[Boolean] - )( - implicit dateRange: DateRange, - timeZone: TimeZone, - uniqueID: UniqueID - ): Execution[Unit] = { - - val processValueGraphs = enableValueGraphs.getOrElse(Configs.EnableValueGraphs) - val processKeyGraphs = enableKeyGraphs.getOrElse(Configs.EnableKeyGraphs) - - if (!processKeyGraphs && !processValueGraphs) { - // Skip the batch job - Execution.unit - } else { - // val favoriteGraphStat = Stat("NumFavoriteEdges") - // val retweetGraphStat = Stat("NumRetweetEdges") - // val mentionGraphStat = Stat("NumMentionEdges") - - // val realGraphAggregates = getRealGraphWithMAUOnly - - val followGraph = getTopKFollowGraph - // val mutualFollowGraph = followGraph.asKeys.join(followGraph.swap.asKeys).keys - - // val favoriteGraph = - // getSubGraph(realGraphAggregates, _.favoriteScore.isDefined, favoriteGraphStat) - - // val retweetGraph = - // getSubGraph(realGraphAggregates, _.retweetScore.isDefined, retweetGraphStat) - - // val mentionGraph = - // getSubGraph(realGraphAggregates, _.mentionScore.isDefined, mentionGraphStat) - - val writeValDataSetExecutions = if (processValueGraphs) { - Seq( - (followGraph, shardingByValue, FollowOutValPath), - (followGraph.swap, shardingByValue, FollowInValPath) - // (mutualFollowGraph, shardingByValue, MutualFollowValPath), - // (favoriteGraph, shardingByValue, FavoriteOutValPath), - // (favoriteGraph.swap, shardingByValue, FavoriteInValPath), - // (retweetGraph, shardingByValue, RetweetOutValPath), - // (retweetGraph.swap, shardingByValue, RetweetInValPath), - // (mentionGraph, shardingByValue, MentionOutValPath), - // (mentionGraph.swap, shardingByValue, MentionInValPath) - ) - } else { - Seq.empty - } - - val writeKeyDataSetExecutions = if (processKeyGraphs) { - Seq( - (followGraph, shardingByKey, FollowOutKeyPath), - (followGraph.swap, shardingByKey, FollowInKeyPath) - // (favoriteGraph, shardingByKey, FavoriteOutKeyPath), - // (favoriteGraph.swap, shardingByKey, FavoriteInKeyPath), - // (retweetGraph, shardingByKey, RetweetOutKeyPath), - // (retweetGraph.swap, shardingByKey, RetweetInKeyPath), - // (mentionGraph, shardingByKey, MentionOutKeyPath), - // (mentionGraph.swap, shardingByKey, MentionInKeyPath), - // (mutualFollowGraph, shardingByKey, MutualFollowKeyPath) - ) - } else { - Seq.empty - } - - Execution - .sequence((writeValDataSetExecutions ++ writeKeyDataSetExecutions).map { - case (graph, shardingMethod, path) => - writeGraphToDB(graph, shardingMethod, path) - }).unit - } - } -} diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/BUILD.bazel b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/BUILD.bazel deleted file mode 100644 index 6378b0a83..000000000 --- a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/BUILD.bazel +++ /dev/null @@ -1,27 +0,0 @@ -scala_library( - platform = "java8", - tags = ["bazel-only"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:core", - "3rdparty/jvm/com/twitter/bijection:scrooge", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/constdb_util", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api:api-base", - "src/scala/com/twitter/scalding_internal/job", - "src/scala/com/twitter/scalding_internal/job/analytics_batch", - "src/thrift/com/twitter/ml/api:data-java", - ], -) - -hadoop_binary( - name = "gfs_random_request-adhoc", - main = "com.twitter.graph_feature_service.scalding.adhoc.RandomRequestGenerationApp", - platform = "java8", - runtime_platform = "java8", - tags = [ - "bazel-compatible", - "bazel-compatible:migrated", - "bazel-only", - ], - dependencies = [":adhoc"], -) diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/BUILD.docx b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/BUILD.docx new file mode 100644 index 000000000..d048faef5 Binary files /dev/null and b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/BUILD.docx differ diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/RandomRequestGenerationApp.docx b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/RandomRequestGenerationApp.docx new file mode 100644 index 000000000..53c1bccfc Binary files /dev/null and b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/RandomRequestGenerationApp.docx differ diff --git a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/RandomRequestGenerationApp.scala b/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/RandomRequestGenerationApp.scala deleted file mode 100644 index 7163a96ac..000000000 --- a/graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/RandomRequestGenerationApp.scala +++ /dev/null @@ -1,77 +0,0 @@ -package com.twitter.graph_feature_service.scalding.adhoc - -import com.twitter.bijection.Injection -import com.twitter.frigate.common.constdb_util.Injections -import com.twitter.ml.api.Feature.Discrete -import com.twitter.ml.api.{DailySuffixFeatureSource, DataSetPipe, RichDataRecord} -import com.twitter.scalding._ -import com.twitter.scalding_internal.job.TwitterExecutionApp -import java.nio.ByteBuffer -import java.util.TimeZone - -object RandomRequestGenerationJob { - implicit val timeZone: TimeZone = DateOps.UTC - implicit val dateParser: DateParser = DateParser.default - - val timelineRecapDataSetPath: String = - "/atla/proc2/user/timelines/processed/suggests/recap/data_records" - - val USER_ID = new Discrete("meta.user_id") - val AUTHOR_ID = new Discrete("meta.author_id") - - val timelineRecapOutPutPath: String = "/user/cassowary/gfs/adhoc/timeline_data" - - implicit val inj: Injection[Long, ByteBuffer] = Injections.long2Varint - - def run( - dataSetPath: String, - outPutPath: String, - numOfPairsToTake: Int - )( - implicit dateRange: DateRange, - uniqueID: UniqueID - ): Execution[Unit] = { - - val NumUserAuthorPairs = Stat("NumUserAuthorPairs") - - val dataSet: DataSetPipe = DailySuffixFeatureSource(dataSetPath).read - - val userAuthorPairs: TypedPipe[(Long, Long)] = dataSet.records.map { record => - val richRecord = new RichDataRecord(record, dataSet.featureContext) - - val userId = richRecord.getFeatureValue(USER_ID) - val authorId = richRecord.getFeatureValue(AUTHOR_ID) - NumUserAuthorPairs.inc() - (userId, authorId) - } - - userAuthorPairs - .limit(numOfPairsToTake) - .writeExecution( - TypedTsv[(Long, Long)](outPutPath) - ) - } -} - -/** - * ./bazel bundle graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc:all - * - * oscar hdfs --screen --user cassowary --tee gfs_log --bundle gfs_random_request-adhoc \ - --tool com.twitter.graph_feature_service.scalding.adhoc.RandomRequestGenerationApp \ - -- --date 2018-08-11 \ - --input /atla/proc2/user/timelines/processed/suggests/recap/data_records \ - --output /user/cassowary/gfs/adhoc/timeline_data - */ -object RandomRequestGenerationApp extends TwitterExecutionApp { - import RandomRequestGenerationJob._ - override def job: Execution[Unit] = Execution.withId { implicit uniqueId => - Execution.getArgs.flatMap { args: Args => - implicit val dateRange: DateRange = DateRange.parse(args.list("date"))(timeZone, dateParser) - run( - args.optional("input").getOrElse(timelineRecapDataSetPath), - args.optional("output").getOrElse(timelineRecapOutPutPath), - args.int("num_pairs", 3000) - ) - } - } -} diff --git a/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/BUILD b/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/BUILD deleted file mode 100644 index 72b7d516b..000000000 --- a/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -create_thrift_libraries( - base_name = "graph_feature_service_thrift", - sources = ["*.thrift"], - platform = "java8", - tags = ["bazel-compatible"], - generate_languages = [ - "java", - # ruby is added due to ruby dependees in timelines - "ruby", - "scala", - "strato", - ], - provides_java_name = "graph_feature_service_thrift_java", - provides_scala_name = "graph_feature_service_thrift_scala", -) diff --git a/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/BUILD.docx b/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/BUILD.docx new file mode 100644 index 000000000..d0b95f8bd Binary files /dev/null and b/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/BUILD.docx differ diff --git a/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/graph_feature_service.docx b/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/graph_feature_service.docx new file mode 100644 index 000000000..89253cfe9 Binary files /dev/null and b/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/graph_feature_service.docx differ diff --git a/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/graph_feature_service.thrift b/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/graph_feature_service.thrift deleted file mode 100644 index 232f8488d..000000000 --- a/graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/graph_feature_service.thrift +++ /dev/null @@ -1,123 +0,0 @@ -namespace java com.twitter.graph_feature_service.thriftjava -#@namespace scala com.twitter.graph_feature_service.thriftscala -#@namespace strato com.twitter.graph_feature_service.thriftscala - -// edge type to differentiate different types of graphs (we can also add a lot of other types of edges) -enum EdgeType { - FOLLOWING, - FOLLOWED_BY, - FAVORITE, - FAVORITED_BY, - RETWEET, - RETWEETED_BY, - REPLY, - REPLYED_BY, - MENTION, - MENTIONED_BY, - MUTUAL_FOLLOW, - SIMILAR_TO, // more edge types (like block, report, etc.) can be supported later. - RESERVED_12, - RESERVED_13, - RESERVED_14, - RESERVED_15, - RESERVED_16, - RESERVED_17, - RESERVED_18, - RESERVED_19, - RESERVED_20 -} - -enum PresetFeatureTypes { - EMPTY, - HTL_TWO_HOP, - WTF_TWO_HOP, - SQ_TWO_HOP, - RUX_TWO_HOP, - MR_TWO_HOP, - USER_TYPEAHEAD_TWO_HOP -} - -struct UserWithCount { - 1: required i64 userId(personalDataType = 'UserId') - 2: required i32 count -}(hasPersonalData = 'true') - -struct UserWithScore { - 1: required i64 userId(personalDataType = 'UserId') - 2: required double score -}(hasPersonalData = 'true') - -// Feature Type -// For example, to compute how many of source user's following's have favorited candidate user, -// we need to compute the intersection between source user's FOLLOWING edges, and candidate user's -// FAVORITED_BY edge. In this case, we should user FeatureType(FOLLOWING, FAVORITED_BY) -struct FeatureType { - 1: required EdgeType leftEdgeType // edge type from source user - 2: required EdgeType rightEdgeType // edge type from candidate user -}(persisted="true") - -struct IntersectionValue { - 1: required FeatureType featureType - 2: optional i32 count - 3: optional list intersectionIds(personalDataType = 'UserId') - 4: optional i32 leftNodeDegree - 5: optional i32 rightNodeDegree -}(persisted="true", hasPersonalData = 'true') - -struct GfsIntersectionResult { - 1: required i64 candidateUserId(personalDataType = 'UserId') - 2: required list intersectionValues -}(hasPersonalData = 'true') - -struct GfsIntersectionRequest { - 1: required i64 userId(personalDataType = 'UserId') - 2: required list candidateUserIds(personalDataType = 'UserId') - 3: required list featureTypes - 4: optional i32 intersectionIdLimit -} - -struct GfsPresetIntersectionRequest { - 1: required i64 userId(personalDataType = 'UserId') - 2: required list candidateUserIds(personalDataType = 'UserId') - 3: required PresetFeatureTypes presetFeatureTypes - 4: optional i32 intersectionIdLimit -}(hasPersonalData = 'true') - -struct GfsIntersectionResponse { - 1: required list results -} - -service Server { - GfsIntersectionResponse getIntersection(1: GfsIntersectionRequest request) - GfsIntersectionResponse getPresetIntersection(1: GfsPresetIntersectionRequest request) -} - -################################################################################################### -## For internal usage only -################################################################################################### -struct WorkerIntersectionRequest { - 1: required i64 userId(personalDataType = 'UserId') - 2: required list candidateUserIds(personalDataType = 'UserId') - 3: required list featureTypes - 4: required PresetFeatureTypes presetFeatureTypes - 5: required i32 intersectionIdLimit -}(hasPersonalData = 'true') - -struct WorkerIntersectionResponse { - 1: required list> results -} - -struct WorkerIntersectionValue { - 1: i32 count - 2: i32 leftNodeDegree - 3: i32 rightNodeDegree - 4: list intersectionIds(personalDataType = 'UserId') -}(hasPersonalData = 'true') - -struct CachedIntersectionResult { - 1: required list values -} - -service Worker { - WorkerIntersectionResponse getIntersection(1: WorkerIntersectionRequest request) -} diff --git a/home-mixer/BUILD.bazel b/home-mixer/BUILD.bazel deleted file mode 100644 index ab7b358e7..000000000 --- a/home-mixer/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -jvm_binary( - name = "bin", - basename = "home-mixer", - main = "com.twitter.home_mixer.HomeMixerServerMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "finagle/finagle-zipkin-scribe/src/main/scala", - "finatra/inject/inject-logback/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - ], -) - -# Aurora Workflows build phase convention requires a jvm_app named with home-mixer-app -jvm_app( - name = "home-mixer-app", - archive = "zip", - binary = ":bin", - bundles = [ - bundle( - fileset = ["config/**/*"], - owning_target = "home-mixer/config:files", - ), - ], - tags = ["bazel-compatible"], -) diff --git a/home-mixer/BUILD.docx b/home-mixer/BUILD.docx new file mode 100644 index 000000000..138653575 Binary files /dev/null and b/home-mixer/BUILD.docx differ diff --git a/home-mixer/README.docx b/home-mixer/README.docx new file mode 100644 index 000000000..db8ad8ff1 Binary files /dev/null and b/home-mixer/README.docx differ diff --git a/home-mixer/README.md b/home-mixer/README.md deleted file mode 100644 index 20861d7a0..000000000 --- a/home-mixer/README.md +++ /dev/null @@ -1,101 +0,0 @@ -Home Mixer -========== - -Home Mixer is the main service used to construct and serve Twitter's Home Timelines. It currently -powers: -- For you - best Tweets from people you follow + recommended out-of-network content -- Following - reverse chronological Tweets from people you follow -- Lists - reverse chronological Tweets from List members - -Home Mixer is built on Product Mixer, our custom Scala framework that facilitates building -feeds of content. - -## Overview - -The For You recommendation algorithm in Home Mixer involves the following stages: - -- Candidate Generation - fetch Tweets from various Candidate Sources. For example: - - Earlybird Search Index - - User Tweet Entity Graph - - Cr Mixer - - Follow Recommendations Service -- Feature Hydration - - Fetch the ~6000 features needed for ranking -- Scoring and Ranking using ML model -- Filters and Heuristics. For example: - - Author Diversity - - Content Balance (In network vs Out of Network) - - Feedback fatigue - - Deduplication / previously seen Tweets removal - - Visibility Filtering (blocked, muted authors/tweets, NSFW settings) -- Mixing - integrate Tweets with non-Tweet content - - Ads - - Who-to-follow modules - - Prompts -- Product Features and Serving - - Conversation Modules for replies - - Social Context - - Timeline Navigation - - Edited Tweets - - Feedback options - - Pagination and cursoring - - Observability and logging - - Client instructions and content marshalling - -## Pipeline Structure - -### General - -Product Mixer services like Home Mixer are structured around Pipelines that split the execution -into transparent and structured steps. - -Requests first go to Product Pipelines, which are used to select which Mixer Pipeline or -Recommendation Pipeline to run for a given request. Each Mixer or Recommendation -Pipeline may run multiple Candidate Pipelines to fetch candidates to include in the response. - -Mixer Pipelines combine the results of multiple heterogeneous Candidate Pipelines together -(e.g. ads, tweets, users) while Recommendation Pipelines are used to score (via Scoring Pipelines) -and rank the results of homogenous Candidate Pipelines so that the top ranked ones can be returned. -These pipelines also marshall candidates into a domain object and then into a transport object -to return to the caller. - -Candidate Pipelines fetch candidates from underlying Candidate Sources and perform some basic -operations on the Candidates, such as filtering out unwanted candidates, applying decorations, -and hydrating features. - -The sections below describe the high level pipeline structure (non-exhaustive) for the main Home -Timeline tabs powered by Home Mixer. - -### For You - -- ForYouProductPipelineConfig - - ForYouScoredTweetsMixerPipelineConfig (main orchestration layer - mixes Tweets with ads and users) - - ForYouScoredTweetsCandidatePipelineConfig (fetch Tweets) - - ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer) - - Fetch Tweet Candidates - - ScoredTweetsInNetworkCandidatePipelineConfig - - ScoredTweetsTweetMixerCandidatePipelineConfig - - ScoredTweetsUtegCandidatePipelineConfig - - ScoredTweetsFrsCandidatePipelineConfig - - Feature Hydration and Scoring - - ScoredTweetsScoringPipelineConfig - - ForYouConversationServiceCandidatePipelineConfig (backup reverse chron pipeline in case Scored Tweets fails) - - ForYouAdsCandidatePipelineConfig (fetch ads) - - ForYouWhoToFollowCandidatePipelineConfig (fetch users to recommend) - -### Following - -- FollowingProductPipelineConfig - - FollowingMixerPipelineConfig - - FollowingEarlybirdCandidatePipelineConfig (fetch tweets from Search Index) - - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules) - - FollowingAdsCandidatePipelineConfig (fetch ads) - - FollowingWhoToFollowCandidatePipelineConfig (fetch users to recommend) - -### Lists - -- ListTweetsProductPipelineConfig - - ListTweetsMixerPipelineConfig - - ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service) - - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules) - - ListTweetsAdsCandidatePipelineConfig (fetch ads) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel deleted file mode 100644 index 103e079da..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel +++ /dev/null @@ -1,51 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-core/src/main", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/http-core/src/main/java/com/twitter/finatra/http", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-utils/src/main/scala", - "home-mixer/server/src/main/resources", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/federated", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "strato/config/columns/auth-context:auth-context-strato-client", - "strato/config/columns/gizmoduck:gizmoduck-strato-client", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - "stringcenter/client", - "stringcenter/client/src/main/java", - "stringcenter/client/src/main/scala/com/twitter/stringcenter/client", - "thrift-web-forms/src/main/scala/com/twitter/thriftwebforms/view", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/features/app", - "twitter-server-internal", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.docx new file mode 100644 index 000000000..3fd6910ee Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.docx new file mode 100644 index 000000000..1fb992706 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala deleted file mode 100644 index e27133b23..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.home_mixer - -import com.twitter.finatra.http.routing.HttpWarmup -import com.twitter.finatra.httpclient.RequestBuilder._ -import com.twitter.util.logging.Logging -import com.twitter.inject.utils.Handler -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) extends Handler with Logging { - - override def handle(): Unit = { - Try(warmup.send(get("/admin/product-mixer/product-pipelines"), admin = true)()) - .onFailure(e => error(e.getMessage, e)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.docx new file mode 100644 index 000000000..7de0592a8 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala deleted file mode 100644 index e635c7a68..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala +++ /dev/null @@ -1,128 +0,0 @@ -package com.twitter.home_mixer - -import com.google.inject.Module -import com.twitter.finagle.Filter -import com.twitter.finatra.annotations.DarkTrafficFilterType -import com.twitter.finatra.http.HttpServer -import com.twitter.finatra.http.routing.HttpRouter -import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} -import com.twitter.finatra.mtls.thriftmux.Mtls -import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.finatra.thrift.filters._ -import com.twitter.finatra.thrift.routing.ThriftRouter -import com.twitter.home_mixer.controller.HomeThriftController -import com.twitter.home_mixer.federated.HomeMixerColumn -import com.twitter.home_mixer.module._ -import com.twitter.home_mixer.param.GlobalParamConfigModule -import com.twitter.home_mixer.product.HomeMixerProductModule -import com.twitter.home_mixer.{thriftscala => st} -import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule -import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule -import com.twitter.product_mixer.component_library.module.EarlybirdModule -import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule -import com.twitter.product_mixer.component_library.module.GizmoduckClientModule -import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule -import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule -import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule -import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule -import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule -import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule -import com.twitter.product_mixer.component_library.module.TweetMixerClientModule -import com.twitter.product_mixer.component_library.module.UserSessionStoreModule -import com.twitter.product_mixer.core.controllers.ProductMixerController -import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper -import com.twitter.product_mixer.core.module.ProductMixerModule -import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule -import com.twitter.strato.fed.StratoFed -import com.twitter.strato.fed.server.StratoFedServer - -object HomeMixerServerMain extends HomeMixerServer - -class HomeMixerServer - extends StratoFedServer - with ThriftServer - with Mtls - with HttpServer - with HttpMtls { - override val name = "home-mixer-server" - - override val modules: Seq[Module] = Seq( - AccountRecommendationsMixerModule, - AdvertiserBrandSafetySettingsStoreModule, - BlenderClientModule, - ClientSentImpressionsPublisherModule, - ConversationServiceModule, - EarlybirdModule, - ExploreRankerClientModule, - FeedbackHistoryClientModule, - GizmoduckClientModule, - GlobalParamConfigModule, - HomeAdsCandidateSourceModule, - HomeMixerFlagsModule, - HomeMixerProductModule, - HomeMixerResourcesModule, - ImpressionBloomFilterModule, - InjectionHistoryClientModule, - ManhattanClientsModule, - ManhattanFeatureRepositoryModule, - ManhattanTweetImpressionStoreModule, - MemcachedFeatureRepositoryModule, - NaviModelClientModule, - OnboardingTaskServiceModule, - OptimizedStratoClientModule, - PeopleDiscoveryServiceModule, - ProductMixerModule, - RealGraphInNetworkScoresModule, - RealtimeAggregateFeatureRepositoryModule, - ScoredTweetsMemcacheModule, - ScribeEventPublisherModule, - SimClustersRecentEngagementsClientModule, - SocialGraphServiceModule, - StaleTweetsCacheModule, - ThriftFeatureRepositoryModule, - TimelineRankerClientModule, - TimelineScorerClientModule, - TimelineServiceClientModule, - TimelinesPersistenceStoreClientModule, - TopicSocialProofClientModule, - TweetImpressionStoreModule, - TweetMixerClientModule, - TweetypieClientModule, - TweetypieStaticEntitiesCacheClientModule, - UserSessionStoreModule, - new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](), - new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this), - new ProductScopeStringCenterModule() - ) - - override def configureThrift(router: ThriftRouter): Unit = { - router - .filter[LoggingMDCFilter] - .filter[TraceIdMDCFilter] - .filter[ThriftMDCFilter] - .filter[StatsFilter] - .filter[AccessLoggingFilter] - .filter[ExceptionMappingFilter] - .filter[Filter.TypeAgnostic, DarkTrafficFilterType] - .exceptionMapper[LoggingThrowableExceptionMapper] - .exceptionMapper[PipelineFailureExceptionMapper] - .add[HomeThriftController] - } - - override def configureHttp(router: HttpRouter): Unit = - router.add( - ProductMixerController[st.HomeMixer.MethodPerEndpoint]( - this.injector, - st.HomeMixer.ExecutePipeline)) - - override val dest: String = "/s/home-mixer/home-mixer:strato" - - override val columns: Seq[Class[_ <: StratoFed.Column]] = - Seq(classOf[HomeMixerColumn]) - - override protected def warmup(): Unit = { - handle[HomeMixerThriftServerWarmupHandler]() - handle[HomeMixerHttpServerWarmupHandler]() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.docx new file mode 100644 index 000000000..c629dca4c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala deleted file mode 100644 index 982b77487..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.home_mixer - -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.thrift.routing.ThriftWarmup -import com.twitter.home_mixer.{thriftscala => st} -import com.twitter.util.logging.Logging -import com.twitter.inject.utils.Handler -import com.twitter.product_mixer.core.{thriftscala => pt} -import com.twitter.scrooge.Request -import com.twitter.scrooge.Response -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup) - extends Handler - with Logging { - - private val clientId = ClientId("thrift-warmup-client") - - def handle(): Unit = { - val testIds = Seq(1, 2, 3) - try { - clientId.asCurrent { - testIds.foreach { id => - val warmupReq = warmupQuery(id) - info(s"Sending warm-up request to service with query: $warmupReq") - warmup.sendRequest( - method = st.HomeMixer.GetUrtResponse, - req = Request(st.HomeMixer.GetUrtResponse.Args(warmupReq)))(assertWarmupResponse) - } - } - } catch { - case e: Throwable => error(e.getMessage, e) - } - info("Warm-up done.") - } - - private def warmupQuery(userId: Long): st.HomeMixerRequest = { - val clientContext = pt.ClientContext( - userId = Some(userId), - guestId = None, - appId = Some(12345L), - ipAddress = Some("0.0.0.0"), - userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), - countryCode = Some("US"), - languageCode = Some("en"), - isTwoffice = None, - userRoles = None, - deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") - ) - st.HomeMixerRequest( - clientContext = clientContext, - product = st.Product.Following, - productContext = Some(st.ProductContext.Following(st.Following())), - maxResults = Some(3) - ) - } - - private def assertWarmupResponse( - result: Try[Response[st.HomeMixer.GetUrtResponse.SuccessType]] - ): Unit = { - result match { - case Return(_) => // ok - case Throw(exception) => - warn("Error performing warm-up request.") - error(exception.getMessage, exception) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel deleted file mode 100644 index 3f738bcb6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.docx new file mode 100644 index 000000000..0f30b6888 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.docx new file mode 100644 index 000000000..5d97e2bbd Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala deleted file mode 100644 index d26843193..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala +++ /dev/null @@ -1,116 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.functional_component.transformer.DependentCandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig - -/** - * Candidate Pipeline Config that fetches tweets from the Conversation Service Candidate Source - */ -class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( - conversationServiceCandidateSource: ConversationServiceCandidateSource, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - namesFeatureHydrator: NamesFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - override val gates: Seq[BaseGate[Query]], - override val decorator: Option[CandidateDecorator[Query, TweetCandidate]]) - extends DependentCandidatePipelineConfig[ - Query, - ConversationServiceCandidateSourceRequest, - TweetWithConversationMetadata, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ConversationService") - - private val TweetypieHydratedFilterId = "TweetypieHydrated" - private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" - - override val candidateSource: BaseCandidateSource[ - ConversationServiceCandidateSourceRequest, - TweetWithConversationMetadata - ] = conversationServiceCandidateSource - - override val queryTransformer: DependentCandidatePipelineQueryTransformer[ - Query, - ConversationServiceCandidateSourceRequest - ] = { (_, candidates) => - val tweetsWithConversationMetadata = candidates.map { candidate => - TweetWithConversationMetadata( - tweetId = candidate.candidateIdLong, - userId = candidate.features.getOrElse(AuthorIdFeature, None), - sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), - sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), - inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), - conversationId = None, - ancestors = Seq.empty - ) - } - ConversationServiceCandidateSourceRequest(tweetsWithConversationMetadata) - } - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[TweetWithConversationMetadata] - ] = Seq(ConversationServiceResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - TweetWithConversationMetadata, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[Query, TweetCandidate, _] - ] = Seq( - tweetypieFeatureHydrator, - InNetworkFeatureHydrator, - ) - - override def filters: Seq[Filter[Query, TweetCandidate]] = Seq( - RetweetDeduplicationFilter, - FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(QuotedTweetDroppedFilterId), - shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } - ), - invalidSubscriptionTweetFilter, - InvalidConversationModuleFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[Query, TweetCandidate, _] - ] = Seq(namesFeatureHydrator) - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.docx new file mode 100644 index 000000000..88b70fd56 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala deleted file mode 100644 index bb55f85e3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() ( - conversationServiceCandidateSource: ConversationServiceCandidateSource, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - namesFeatureHydrator: NamesFeatureHydrator) { - - def build( - gates: Seq[BaseGate[Query]] = Seq.empty, - decorator: Option[CandidateDecorator[Query, TweetCandidate]] = None - ): ConversationServiceCandidatePipelineConfig[Query] = { - new ConversationServiceCandidatePipelineConfig( - conversationServiceCandidateSource, - tweetypieFeatureHydrator, - namesFeatureHydrator, - invalidSubscriptionTweetFilter, - gates, - decorator - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.docx new file mode 100644 index 000000000..34ce55970 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala deleted file mode 100644 index 154c080ad..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.thriftscala.SuggestType - -object ConversationServiceResponseFeatureTransformer - extends CandidateFeatureTransformer[TweetWithConversationMetadata] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ConversationServiceResponse") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - InReplyToTweetIdFeature, - IsRetweetFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - ConversationModuleFocalTweetIdFeature, - AncestorsFeature, - SuggestTypeFeature - ) - - override def transform(candidate: TweetWithConversationMetadata): FeatureMap = FeatureMapBuilder() - .add(AuthorIdFeature, candidate.userId) - .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) - .add(IsRetweetFeature, candidate.sourceTweetId.isDefined) - .add(SourceTweetIdFeature, candidate.sourceTweetId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .add(ConversationModuleFocalTweetIdFeature, candidate.conversationId) - .add(AncestorsFeature, candidate.ancestors) - .add(SuggestTypeFeature, Some(SuggestType.RankedOrganicTweet)) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.docx new file mode 100644 index 000000000..ce0795471 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala deleted file mode 100644 index d9bb73695..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.EmptyClientEventInfoBuilder -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineFocalTweetSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches edited tweets from the Stale Tweets Cache - */ -@Singleton -case class EditedTweetsCandidatePipelineConfig @Inject() ( - staleTweetsCacheCandidateSource: StaleTweetsCacheCandidateSource, - namesFeatureHydrator: NamesFeatureHydrator, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) - extends DependentCandidatePipelineConfig[ - PipelineQuery, - Seq[Long], - Long, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("EditedTweets") - - override val candidateSource: BaseCandidateSource[Seq[Long], Long] = - staleTweetsCacheCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - PipelineQuery, - Seq[Long] - ] = EditedTweetsCandidatePipelineQueryTransformer - - override val resultTransformer: CandidatePipelineResultsTransformer[ - Long, - TweetCandidate - ] = { candidate => TweetCandidate(id = candidate) } - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] - ] = Seq(namesFeatureHydrator) - - override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { - val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( - clientEventInfoBuilder = EmptyClientEventInfoBuilder, - entryIdToReplaceBuilder = Some((_, candidate, _) => - Some(s"${TweetItem.TweetEntryNamespace}-${candidate.id.toString}")), - contextualTweetRefBuilder = Some( - ContextualTweetRefBuilder( - TweetHydrationContext( - // Apply safety level that includes canonical VF treatments that apply regardless of context. - safetyLevelOverride = Some(TimelineFocalTweetSafetyLevel), - outerTweetContext = None - ) - ) - ), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) - ) - - Some(UrtItemCandidateDecorator(tweetItemBuilder)) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.5, 50, 60, 60) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.docx new file mode 100644 index 000000000..87b7ff1f0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.scala deleted file mode 100644 index e1a92b9ca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.functional_component.gate.RequestContextNotGate -import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.DurationParamBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.ShowAlertCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertColorConfigurationBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertDisplayLocationBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertIconDisplayInfoBuilder -import com.twitter.product_mixer.component_library.gate.FeatureGate -import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.StaticCandidateSource -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseDurationBuilder -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.NewTweets -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertColorConfiguration -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertIconDisplayInfo -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.Top -import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.UpArrow -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.TwitterBlueRosettaColor -import com.twitter.product_mixer.core.model.marshalling.response.urt.color.WhiteRosettaColor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that creates the New Tweets Pill - */ -@Singleton -class NewTweetsPillCandidatePipelineConfig[Query <: PipelineQuery with HasDeviceContext] @Inject() ( -) extends DependentCandidatePipelineConfig[ - Query, - Unit, - ShowAlertCandidate, - ShowAlertCandidate - ] { - import NewTweetsPillCandidatePipelineConfig._ - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("NewTweetsPill") - - override val gates: Seq[Gate[Query]] = Seq( - RequestContextNotGate(Seq(DeviceContext.RequestContext.PullToRefresh)), - FeatureGate.fromFeature(GetNewerFeature) - ) - - override val candidateSource: CandidateSource[Unit, ShowAlertCandidate] = - StaticCandidateSource( - CandidateSourceIdentifier(identifier.name), - Seq(ShowAlertCandidate(id = identifier.name, userIds = Seq.empty)) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[Query, Unit] = { _ => Unit } - - override val resultTransformer: CandidatePipelineResultsTransformer[ - ShowAlertCandidate, - ShowAlertCandidate - ] = { candidate => candidate } - - override val decorator: Option[CandidateDecorator[Query, ShowAlertCandidate]] = { - val triggerDelayBuilder = new BaseDurationBuilder[Query] { - override def apply( - query: Query, - candidate: ShowAlertCandidate, - features: FeatureMap - ): Option[Duration] = { - val delay = query.deviceContext.flatMap(_.requestContextValue) match { - case Some(DeviceContext.RequestContext.TweetSelfThread) => 0.millis - case Some(DeviceContext.RequestContext.ManualRefresh) => 0.millis - case _ => TriggerDelay - } - - Some(delay) - } - } - - val homeShowAlertCandidateBuilder = ShowAlertCandidateUrtItemBuilder( - alertType = NewTweets, - colorConfigBuilder = StaticShowAlertColorConfigurationBuilder(DefaultColorConfig), - displayLocationBuilder = StaticShowAlertDisplayLocationBuilder(Top), - triggerDelayBuilder = Some(triggerDelayBuilder), - displayDurationBuilder = Some(DurationParamBuilder(StaticParam(DisplayDuration))), - iconDisplayInfoBuilder = Some(StaticShowAlertIconDisplayInfoBuilder(DefaultIconDisplayInfo)) - ) - - Some(UrtItemCandidateDecorator(homeShowAlertCandidateBuilder)) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} - -object NewTweetsPillCandidatePipelineConfig { - val DefaultColorConfig: ShowAlertColorConfiguration = ShowAlertColorConfiguration( - background = TwitterBlueRosettaColor, - text = WhiteRosettaColor, - border = Some(WhiteRosettaColor) - ) - - val DefaultIconDisplayInfo: ShowAlertIconDisplayInfo = - ShowAlertIconDisplayInfo(icon = UpArrow, tint = WhiteRosettaColor) - - // Unlimited display time (until user takes action) - val DisplayDuration = -1.millisecond - val TriggerDelay = 4.minutes -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.docx new file mode 100644 index 000000000..06c182314 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.scala deleted file mode 100644 index 3f0a932c9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.{thriftscala => t} - -object TimelineServiceResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("TimelineServiceResponse") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - InReplyToTweetIdFeature, - IsRetweetFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - ) - - override def transform(candidate: t.Tweet): FeatureMap = FeatureMapBuilder() - .add(AuthorIdFeature, candidate.userId) - .add(InReplyToTweetIdFeature, candidate.inReplyToStatusId) - .add(IsRetweetFeature, candidate.sourceStatusId.isDefined) - .add(SourceTweetIdFeature, candidate.sourceStatusId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel deleted file mode 100644 index dfaff319d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt", - "snowflake/src/main/scala/com/twitter/snowflake/id", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.docx new file mode 100644 index 000000000..4b61c223f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.docx new file mode 100644 index 000000000..2061c9956 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala deleted file mode 100644 index fc1b7770e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.controller - -import com.twitter.finatra.thrift.Controller -import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.service.ScoredTweetsService -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.controllers.DebugTwitterContext -import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder -import com.twitter.product_mixer.core.service.debug_query.DebugQueryService -import com.twitter.product_mixer.core.service.urt.UrtService -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params -import javax.inject.Inject - -class HomeThriftController @Inject() ( - homeRequestUnmarshaller: HomeMixerRequestUnmarshaller, - urtService: UrtService, - scoredTweetsService: ScoredTweetsService, - paramsBuilder: ParamsBuilder) - extends Controller(t.HomeMixer) - with DebugTwitterContext { - - handle(t.HomeMixer.GetUrtResponse) { args: t.HomeMixer.GetUrtResponse.Args => - val request = homeRequestUnmarshaller(args.request) - val params = buildParams(request) - Stitch.run(urtService.getUrtResponse[HomeMixerRequest](request, params)) - } - - handle(t.HomeMixer.GetScoredTweetsResponse) { args: t.HomeMixer.GetScoredTweetsResponse.Args => - val request = homeRequestUnmarshaller(args.request) - val params = buildParams(request) - withDebugTwitterContext(request.clientContext) { - Stitch.run(scoredTweetsService.getScoredTweetsResponse[HomeMixerRequest](request, params)) - } - } - - private def buildParams(request: HomeMixerRequest): Params = { - val userAgeOpt = request.clientContext.userId.map { userId => - SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) - } - val fsCustomMapInput = userAgeOpt.map("account_age_in_days" -> _).toMap - paramsBuilder.build( - clientContext = request.clientContext, - product = request.product, - featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), - fsCustomMapInput = fsCustomMapInput - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel deleted file mode 100644 index 30ee81acc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "stitch/stitch-repo/src/main/scala", - "strato/config/columns/auth-context:auth-context-strato-client", - "strato/config/columns/gizmoduck:gizmoduck-strato-client", - "strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala", - "strato/src/main/scala/com/twitter/strato/callcontext", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.docx new file mode 100644 index 000000000..4a23fda24 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.docx new file mode 100644 index 000000000..d1ea822f2 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala deleted file mode 100644 index 0ef27b7a0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala +++ /dev/null @@ -1,217 +0,0 @@ -package com.twitter.home_mixer.federated - -import com.twitter.gizmoduck.{thriftscala => gd} -import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.{thriftscala => hm} -import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.product_mixer.core.{thriftscala => pm} -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch -import com.twitter.strato.callcontext.CallContext -import com.twitter.strato.catalog.OpMetadata -import com.twitter.strato.config._ -import com.twitter.strato.data._ -import com.twitter.strato.fed.StratoFed -import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn -import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn -import com.twitter.strato.graphql.timelines.{thriftscala => gql} -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.timelines.render.{thriftscala => tr} -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerColumn @Inject() ( - homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller, - compositeOnUserClientColumn: CompositeOnUserClientColumn, - auditIpClientColumn: AuditIpClientColumn, - paramsBuilder: ParamsBuilder, - productPipelineRegistry: ProductPipelineRegistry) - extends StratoFed.Column(HomeMixerColumn.Path) - with StratoFed.Fetch.Arrow { - - override val contactInfo: ContactInfo = ContactInfo( - contactEmail = "", - ldapGroup = "", - slackRoomId = "" - ) - - override val metadata: OpMetadata = - OpMetadata( - lifecycle = Some(Lifecycle.Production), - description = - Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer")) - ) - - private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess()) - private val finatraTestServiceIdentifiers: Seq[Policy] = Seq( - ServiceIdentifierPattern( - role = "", - service = "", - env = "", - zone = Seq("")) - ) - - override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers) - - override type Key = gql.TimelineKey - override type View = gql.HomeTimelineView - override type Value = tr.Timeline - - override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey] - override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView] - override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline] - - private def createHomeMixerRequestArrow( - compositeOnUserClientColumn: CompositeOnUserClientColumn, - auditIpClientColumn: AuditIpClientColumn - ): Arrow[(Key, View), hm.HomeMixerRequest] = { - - val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = { - val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) = - (gd.LookupContext(), Set(gd.QueryFields.Roles)) - - val populateUserRoles = Arrow - .flatMap[(Key, View), Option[Set[String]]] { _ => - Stitch.collect { - CallContext.twitterUserId.map { userId => - compositeOnUserClientColumn.fetcher - .callStack(HomeMixerColumn.FetchCallstack) - .fetch(userId, gizmoduckView).map(_.v) - .map { - _.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty) - } - } - } - } - - val populateIpAddress = Arrow - .flatMap[(Key, View), Option[String]](_ => - auditIpClientColumn.fetcher - .callStack(HomeMixerColumn.FetchCallstack) - .fetch((), ()).map(_.v)) - - Arrow.join( - populateUserRoles, - populateIpAddress - ) - } - - Arrow.zipWithArg(populateUserRolesAndIp).map { - case ((key, view), (roles, ipAddress)) => - val deviceContextOpt = Some( - hm.DeviceContext( - isPolling = CallContext.isPolling, - requestContext = view.requestContext, - latestControlAvailable = view.latestControlAvailable, - autoplayEnabled = view.autoplayEnabled - )) - val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty) - - val (product, productContext) = key match { - case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) => - ( - hm.Product.ForYou, - hm.ProductContext.ForYou( - hm.ForYou( - deviceContextOpt, - seenTweetIds, - view.dspClientContext, - view.pushToHomeTweetId - ) - )) - case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) => - ( - hm.Product.Following, - hm.ProductContext.Following( - hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext))) - case gql.TimelineKey.CreatorSubscriptionsTimeline(_) => - ( - hm.Product.Subscribed, - hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds))) - case _ => throw new UnsupportedOperationException(s"Unknown product: $key") - } - - val clientContext = pm.ClientContext( - userId = CallContext.twitterUserId, - guestId = CallContext.guestId, - guestIdAds = CallContext.guestIdAds, - guestIdMarketing = CallContext.guestIdMarketing, - appId = CallContext.clientApplicationId, - ipAddress = ipAddress, - userAgent = CallContext.userAgent, - countryCode = CallContext.requestCountryCode, - languageCode = CallContext.requestLanguageCode, - isTwoffice = CallContext.isInternalOrTwoffice, - userRoles = roles, - deviceId = CallContext.deviceId, - mobileDeviceId = CallContext.mobileDeviceId, - mobileDeviceAdId = CallContext.adId, - limitAdTracking = CallContext.limitAdTracking - ) - - hm.HomeMixerRequest( - clientContext = clientContext, - product = product, - productContext = Some(productContext), - maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount), - cursor = view.cursor.filter(_.nonEmpty) - ) - } - } - - override val fetch: Arrow[(Key, View), Result[Value]] = { - val transformThriftIntoPipelineRequest: Arrow[ - (Key, View), - ProductPipelineRequest[HomeMixerRequest] - ] = { - Arrow - .identity[(Key, View)] - .andThen { - createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn) - } - .map { - case thriftRequest => - val request = homeMixerRequestUnmarshaller(thriftRequest) - val params = paramsBuilder.build( - clientContext = request.clientContext, - product = request.product, - featureOverrides = - request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), - ) - ProductPipelineRequest(request, params) - } - } - - val underlyingProduct: Arrow[ - ProductPipelineRequest[HomeMixerRequest], - ProductPipelineResult[tr.TimelineResponse] - ] = Arrow - .identity[ProductPipelineRequest[HomeMixerRequest]] - .map { pipelineRequest => - val pipelineArrow = productPipelineRegistry - .getProductPipeline[HomeMixerRequest, tr.TimelineResponse]( - pipelineRequest.request.product) - .arrow - (pipelineArrow, pipelineRequest) - }.applyArrow - - transformThriftIntoPipelineRequest.andThen(underlyingProduct).map { - _.result match { - case Some(result) => found(result.timeline) - case _ => missing - } - } - } -} - -object HomeMixerColumn { - val Path = "home-mixer/homeMixer.Timeline" - private val FetchCallstack = s"$Path:fetch" - private val MaxCount: Option[Int] = Some(100) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel deleted file mode 100644 index a40bd4030..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/search:earlybird-scala", - "stitch/stitch-timelineservice/src/main/scala", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.docx new file mode 100644 index 000000000..9ea963cfe Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.docx new file mode 100644 index 000000000..17b4b4b2c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.scala deleted file mode 100644 index 2ddd05814..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.functional_component.candidate_source - -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.search.earlybird.{thriftscala => t} -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -case object EarlybirdResponseTruncatedFeature - extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Boolean] { - override val defaultValue: Boolean = false -} - -case object EarlybirdBottomTweetFeature - extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Option[Long]] { - override val defaultValue: Option[Long] = None -} - -@Singleton -case class EarlybirdCandidateSource @Inject() ( - earlybird: t.EarlybirdService.MethodPerEndpoint) - extends CandidateSourceWithExtractedFeatures[t.EarlybirdRequest, t.ThriftSearchResult] { - - override val identifier = CandidateSourceIdentifier("Earlybird") - - override def apply( - request: t.EarlybirdRequest - ): Stitch[CandidatesWithSourceFeatures[t.ThriftSearchResult]] = { - Stitch.callFuture(earlybird.search(request)).map { response => - val candidates = response.searchResults.map(_.results).getOrElse(Seq.empty) - - val features = FeatureMapBuilder() - .add(EarlybirdResponseTruncatedFeature, candidates.size == request.searchQuery.numResults) - .add(EarlybirdBottomTweetFeature, candidates.lastOption.map(_.id)) - .build() - - CandidatesWithSourceFeatures(candidates, features) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.docx new file mode 100644 index 000000000..cd36af753 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.scala deleted file mode 100644 index 9a346129d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.functional_component.candidate_source - -import com.google.inject.name.Named -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.home_mixer.param.HomeMixerInjectionNames.StaleTweetsCache -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class StaleTweetsCacheCandidateSource @Inject() ( - @Named(StaleTweetsCache) staleTweetsCache: MemcachedClient) - extends CandidateSource[Seq[Long], Long] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("StaleTweetsCache") - - private val StaleTweetsCacheKeyPrefix = "v1_" - - override def apply(request: Seq[Long]): Stitch[Seq[Long]] = { - val keys = request.map(StaleTweetsCacheKeyPrefix + _) - - Stitch.callFuture(staleTweetsCache.get(keys).map { tweets => - tweets.map { - case (k, _) => k.replaceFirst(StaleTweetsCacheKeyPrefix, "").toLong - }.toSeq - }) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel deleted file mode 100644 index 4df69c9f9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-core/src/main", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "stringcenter/client", - "stringcenter/client/src/main/java", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.docx new file mode 100644 index 000000000..bf689fc18 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.docx new file mode 100644 index 000000000..d164e3f2f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala deleted file mode 100644 index d145391d4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object HomeConversationServiceCandidateDecorator { - - private val ConversationModuleNamespace = EntryNamespace("home-conversation") - - def apply( - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder - ): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = { - val suggestType = st.SuggestType.RankedOrganicTweet - val component = InjectionScribeUtil.scribeComponent(suggestType).get - val clientEventInfoBuilder = ClientEventInfoBuilder(component) - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = clientEventInfoBuilder, - timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) - ) - - val moduleBuilder = TimelineModuleBuilder( - entryNamespace = ConversationModuleNamespace, - clientEventInfoBuilder = clientEventInfoBuilder, - displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), - metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) - ) - - Some( - UrtMultipleModulesDecorator( - urtItemCandidateDecorator = UrtItemCandidateDecorator(tweetItemBuilder), - moduleBuilder = moduleBuilder, - groupByKey = (_, _, candidateFeatures) => - candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) - )) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.docx new file mode 100644 index 000000000..bfe68da00 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.scala deleted file mode 100644 index a79543269..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap - -object HomeQueryTypePredicates { - private[this] val QueryPredicates: Seq[(String, FeatureMap => Boolean)] = Seq( - ("request", _ => true), - ("get_initial", _.getOrElse(GetInitialFeature, false)), - ("get_newer", _.getOrElse(GetNewerFeature, false)), - ("get_older", _.getOrElse(GetOlderFeature, false)), - ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)), - ("request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)), - ("request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)) - ) - - val PredicateMap = QueryPredicates.toMap -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel deleted file mode 100644 index cb59d1d73..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "finagle/finagle-core/src/main", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "joinkey/src/main/scala/com/twitter/joinkey/context", - "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.docx new file mode 100644 index 000000000..384041a76 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.docx new file mode 100644 index 000000000..62bbb5ef1 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala deleted file mode 100644 index eb072f135..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.finagle.tracing.Trace -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.suggests.controller_data.home_tweets.v1.{thriftscala => v1ht} -import com.twitter.suggests.controller_data.home_tweets.{thriftscala => ht} -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} - -case class HomeAdsClientEventDetailsBuilder(injectionType: Option[String]) - extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): Option[ClientEventDetails] = { - val homeTweetsControllerDataV1 = v1ht.HomeTweetsControllerData( - tweetTypesBitmap = 0L, - traceId = Some(Trace.id.traceId.toLong), - requestJoinId = None) - - val serializedControllerData = HomeClientEventDetailsBuilder.ControllerDataSerializer( - ControllerData.V2( - ControllerDataV2.HomeTweets(ht.HomeTweetsControllerData.V1(homeTweetsControllerDataV1)))) - - val clientEventDetails = ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some( - TimelinesDetails( - injectionType = injectionType, - controllerData = Some(serializedControllerData), - sourceData = None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - ) - - Some(clientEventDetails) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.docx new file mode 100644 index 000000000..c6a8d943d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala deleted file mode 100644 index 2e4c60d25..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.bijection.Base64String -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.{Injection => Serializer} -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.PositionFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.joinkey.context.RequestJoinKeyContext -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.suggests.controller_data.Home -import com.twitter.suggests.controller_data.TweetTypeGenerator -import com.twitter.suggests.controller_data.home_tweets.v1.{thriftscala => v1ht} -import com.twitter.suggests.controller_data.home_tweets.{thriftscala => ht} -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} - -object HomeClientEventDetailsBuilder { - implicit val ByteSerializer: Serializer[ControllerData, Array[Byte]] = - BinaryScalaCodec(ControllerData) - - val ControllerDataSerializer: Serializer[ControllerData, String] = - Serializer.connect[ControllerData, Array[Byte], Base64String, String] - - /** - * define getRequestJoinId as a method(def) rather than a val because each new request - * needs to call the context to update the id. - */ - private def getRequestJoinId(): Option[Long] = - RequestJoinKeyContext.current.flatMap(_.requestJoinId) -} - -case class HomeClientEventDetailsBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( -) extends BaseClientEventDetailsBuilder[Query, Candidate] - with TweetTypeGenerator[FeatureMap] { - - import HomeClientEventDetailsBuilder._ - - override def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap - ): Option[ClientEventDetails] = { - - val tweetTypesBitmaps = mkTweetTypesBitmaps( - Home.TweetTypeIdxMap, - HomeTweetTypePredicates.PredicateMap, - candidateFeatures) - - val tweetTypesListBytes = mkItemTypesBitmapsV2( - Home.TweetTypeIdxMap, - HomeTweetTypePredicates.PredicateMap, - candidateFeatures) - - val candidateSourceId = - candidateFeatures.getOrElse(CandidateSourceIdFeature, None).map(_.value.toByte) - - val homeTweetsControllerDataV1 = v1ht.HomeTweetsControllerData( - tweetTypesBitmap = tweetTypesBitmaps.getOrElse(0, 0L), - tweetTypesBitmapContinued1 = tweetTypesBitmaps.get(1), - candidateTweetSourceId = candidateSourceId, - traceId = Some(Trace.id.traceId.toLong), - injectedPosition = candidateFeatures.getOrElse(PositionFeature, None), - tweetTypesListBytes = Some(tweetTypesListBytes), - requestJoinId = getRequestJoinId(), - ) - - val serializedControllerData = ControllerDataSerializer( - ControllerData.V2( - ControllerDataV2.HomeTweets(ht.HomeTweetsControllerData.V1(homeTweetsControllerDataV1)))) - - val clientEventDetails = ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some( - TimelinesDetails( - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None).map(_.name), - controllerData = Some(serializedControllerData), - sourceData = None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - ) - - Some(clientEventDetails) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.docx new file mode 100644 index 000000000..c44601054 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala deleted file mode 100644 index f79b0d931..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.home_mixer.model.HomeFeatures.EntityTokenFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil - -/** - * Sets the [[ClientEventInfo]] with the `component` field set to the Suggest Type assigned to each candidate - */ -case class HomeClientEventInfoBuilder[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - detailsBuilder: Option[BaseClientEventDetailsBuilder[Query, Candidate]] = None) - extends BaseClientEventInfoBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidate: Candidate, - candidateFeatures: FeatureMap, - element: Option[String] - ): Option[ClientEventInfo] = { - val suggestType = candidateFeatures - .getOrElse(SuggestTypeFeature, None) - .getOrElse(throw new UnsupportedOperationException(s"No SuggestType was set")) - - Some( - ClientEventInfo( - component = InjectionScribeUtil.scribeComponent(suggestType), - element = element, - details = detailsBuilder.flatMap(_.apply(query, candidate, candidateFeatures)), - action = None, - /** - * A backend entity encoded by the Client Entities Encoding Library. - * Placeholder string for now - */ - entityToken = candidateFeatures.getOrElse(EntityTokenFeature, None) - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.docx new file mode 100644 index 000000000..2daad79e0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala deleted file mode 100644 index dc6d51327..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleMetadataBuilder -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleConversationMetadata -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleMetadata -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class HomeConversationModuleMetadataBuilder[ - -Query <: PipelineQuery, - -Candidate <: BaseTweetCandidate -]() extends BaseModuleMetadataBuilder[Query, Candidate] { - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): ModuleMetadata = ModuleMetadata( - adsMetadata = None, - conversationMetadata = Some( - ModuleConversationMetadata( - allTweetIds = Some((candidates.last.candidate.id +: - candidates.last.features.getOrElse(AncestorsFeature, Seq.empty).map(_.tweetId)).reverse), - socialContext = None, - enableDeduplication = Some(true) - )), - gridCarouselMetadata = None - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.docx new file mode 100644 index 000000000..044e69337 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala deleted file mode 100644 index bc934414c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.tweet.BaseTimelinesScoreInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TimelinesScoreInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object HomeTimelinesScoreInfoBuilder - extends BaseTimelinesScoreInfoBuilder[PipelineQuery, TweetCandidate] { - - private val UndefinedTweetScore = -1.0 - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[TimelinesScoreInfo] = { - if (query.params(EnableSendScoresToClient)) { - val score = candidateFeatures.getOrElse(ScoreFeature, None).getOrElse(UndefinedTweetScore) - Some(TimelinesScoreInfo(score)) - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.docx new file mode 100644 index 000000000..432d8595c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala deleted file mode 100644 index 7272c360d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala +++ /dev/null @@ -1,256 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures -import com.twitter.tweetypie.{thriftscala => tpt} - -object HomeTweetTypePredicates { - - /** - * IMPORTANT: Please avoid logging tweet types that are tied to sensitive - * internal author information / labels (e.g. blink labels, abuse labels, or geo-location). - */ - private[this] val CandidatePredicates: Seq[(String, FeatureMap => Boolean)] = Seq( - ("with_candidate", _ => true), - ("retweet", _.getOrElse(IsRetweetFeature, false)), - ("reply", _.getOrElse(InReplyToTweetIdFeature, None).nonEmpty), - ("image", _.getOrElse(EarlybirdFeature, None).exists(_.hasImage)), - ("video", _.getOrElse(EarlybirdFeature, None).exists(_.hasVideo)), - ("link", _.getOrElse(EarlybirdFeature, None).exists(_.hasVisibleLink)), - ("quote", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuote.contains(true))), - ("like_social_context", _.getOrElse(NonSelfFavoritedByUserIdsFeature, Seq.empty).nonEmpty), - ("protected", _.getOrElse(EarlybirdFeature, None).exists(_.isProtected)), - ( - "has_exclusive_conversation_author_id", - _.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty), - ("is_eligible_for_connect_boost", _ => false), - ("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)), - ("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)), - ("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)), - ("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)), - ("get_initial", _.getOrElse(GetInitialFeature, false)), - ("get_newer", _.getOrElse(GetNewerFeature, false)), - ("get_middle", _.getOrElse(GetMiddleFeature, false)), - ("get_older", _.getOrElse(GetOlderFeature, false)), - ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)), - ("polling", _.getOrElse(PollingFeature, false)), - ("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)), - ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)), - ("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)), - ( - "less_than_10_mins_since_lnpt", - _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)), - ("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)), - ("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)), - ("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)), - ( - "conversation_module_has_2_displayed_tweets", - _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)), - ("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)), - ("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)), - ( - "served_size_between_50_and_100", - _.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)), - ("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)), - ( - "is_self_thread_tweet", - _.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))), - ("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty), - ("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)), - ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)), - ("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)), - ("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)), - ( - "account_age_less_than_30_minutes", - _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)), - ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)), - ( - "directed_at_user_is_in_first_degree", - _.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))), - ( - "has_semantic_core_annotation", - _.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)), - ("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)), - ( - "account_age_less_than_1_day", - _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)), - ( - "account_age_less_than_7_days", - _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)), - ( - "part_of_utt", - _.getOrElse(EarlybirdFeature, None) - .exists(_.semanticCoreAnnotations.exists(_.exists(annotation => - annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))), - ( - "has_home_latest_request_past_week", - _.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)), - ("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)), - ("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)), - ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)), - ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)), - ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)), - ("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)), - ("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)), - ("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)), - ( - "has_ancestor_authored_by_viewer", - candidate => - candidate - .getOrElse(AncestorsFeature, Seq.empty).exists(ancestor => - candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)), - ("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)), - ( - "deep_reply", - candidate => - candidate.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && candidate - .getOrElse(AncestorsFeature, Seq.empty).size > 2), - ( - "has_simcluster_embeddings", - _.getOrElse( - SimclustersTweetTopKClustersWithScoresFeature, - Map.empty[String, Double]).nonEmpty), - ( - "tweet_age_less_than_15_seconds", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 15.seconds)), - ( - "less_than_1_hour_since_lnpt", - _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)), - ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))), - ( - "device_language_matches_tweet_language", - candidate => - candidate.getOrElse(TweetLanguageFeature, None) == - candidate.getOrElse(DeviceLanguageFeature, None)), - ( - "root_ancestor", - candidate => - candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate - .getOrElse(InReplyToTweetIdFeature, None).isEmpty), - ("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))), - ("in_network", _.getOrElse(InNetworkFeature, true)), - ( - "has_political_annotation", - _.getOrElse(EarlybirdFeature, None).exists( - _.semanticCoreAnnotations.exists( - _.exists(annotation => - SemanticCoreFeatures.PoliticalDomains.contains(annotation.domainId) || - (annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy && - annotation.entityId == SemanticCoreFeatures.UttPoliticsEntityId))))), - ( - "is_dont_at_me_by_invitation", - _.getOrElse(EarlybirdFeature, None).exists( - _.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.ByInvitation]))), - ( - "is_dont_at_me_community", - _.getOrElse(EarlybirdFeature, None) - .exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))), - ("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)), - ( - "is_followed_topic_tweet", - _.getOrElse(TopicContextFunctionalityTypeFeature, None) - .exists(_ == BasicTopicContextFunctionalityType)), - ( - "is_recommended_topic_tweet", - _.getOrElse(TopicContextFunctionalityTypeFeature, None) - .exists(_ == RecommendationTopicContextFunctionalityType)), - ("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))), - ("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))), - ( - "has_gte_10k_favs", - _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10000))), - ( - "has_gte_100k_favs", - _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))), - ("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)), - ("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)), - ( - "has_gte_10_retweets", - _.getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= 10))), - ( - "has_gte_100_retweets", - _.getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= 100))), - ( - "has_gte_1k_retweets", - _.getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= 1000))), - ( - "has_us_political_annotation", - _.getOrElse(EarlybirdFeature, None) - .exists(_.semanticCoreAnnotations.exists(_.exists(annotation => - annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy && - annotation.entityId == SemanticCoreFeatures.usPoliticalTweetEntityId && - annotation.groupId == SemanticCoreFeatures.UsPoliticalTweetAnnotationGroupIds.BalancedV0)))), - ( - "has_toxicity_score_above_threshold", - _.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))), - ("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined), - ( - "text_only", - candidate => - candidate.getOrElse(HasDisplayedTextFeature, false) && - !(candidate.getOrElse(EarlybirdFeature, None).exists(_.hasImage) || - candidate.getOrElse(EarlybirdFeature, None).exists(_.hasVideo) || - candidate.getOrElse(EarlybirdFeature, None).exists(_.hasCard))), - ( - "image_only", - candidate => - candidate.getOrElse(EarlybirdFeature, None).exists(_.hasImage) && - !candidate.getOrElse(HasDisplayedTextFeature, false)), - ("has_1_image", _.getOrElse(NumImagesFeature, None).exists(_ == 1)), - ("has_2_images", _.getOrElse(NumImagesFeature, None).exists(_ == 2)), - ("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)), - ("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)), - ("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)), - ("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)), - ( - "has_liked_by_social_context", - candidateFeatures => - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - .exists(candidateFeatures - .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)), - ( - "has_followed_by_social_context", - _.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty), - ( - "has_topic_social_context", - candidateFeatures => - candidateFeatures - .getOrElse(TopicIdSocialContextFeature, None) - .isDefined && - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined), - ("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)), - ( - "video_bt_10_60_sec", - _.getOrElse(VideoDurationMsFeature, None).exists(duration => - duration > 10000 && duration <= 60000)), - ("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)), - ( - "tweet_age_lte_30_minutes", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 30.minutes)), - ( - "tweet_age_lte_1_hour", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 1.hour)), - ( - "tweet_age_lte_6_hours", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 6.hours)), - ( - "tweet_age_lte_12_hours", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow <= 12.hours)), - ( - "tweet_age_gte_24_hours", - _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) - .exists(_.untilNow >= 24.hours)), - ) - - val PredicateMap = CandidatePredicates.toMap -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.docx new file mode 100644 index 000000000..94bfd7884 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala deleted file mode 100644 index cfffee0ca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.builder - -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelineservice.suggests.{thriftscala => st} - -case class ListClientEventDetailsBuilder(suggestType: st.SuggestType) - extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] { - - override def apply( - query: PipelineQuery, - candidate: UniversalNoun[Any], - candidateFeatures: FeatureMap - ): Option[ClientEventDetails] = { - val clientEventDetails = ClientEventDetails( - conversationDetails = None, - timelinesDetails = Some( - TimelinesDetails( - injectionType = Some(suggestType.name), - controllerData = None, - sourceData = None)), - articleDetails = None, - liveEventDetails = None, - commerceDetails = None - ) - - Some(clientEventDetails) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.docx new file mode 100644 index 000000000..39edfa58f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala deleted file mode 100644 index dc3080eb5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AlwaysInclude -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtInstructionBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.Cover -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlert -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case class AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder[Query <: PipelineQuery]( - override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) - extends UrtInstructionBuilder[Query, AddEntriesTimelineInstruction] { - - override def build( - query: Query, - entries: Seq[TimelineEntry] - ): Seq[AddEntriesTimelineInstruction] = { - if (includeInstruction(query, entries)) { - val entriesToAdd = entries - .filterNot(_.isInstanceOf[ShowAlert]) - .filterNot(_.isInstanceOf[Cover]) - .filter(_.entryIdToReplace.isEmpty) - if (entriesToAdd.nonEmpty) Seq(AddEntriesTimelineInstruction(entriesToAdd)) - else Seq.empty - } else - Seq.empty - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.docx new file mode 100644 index 000000000..fbcc1b864 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala deleted file mode 100644 index dee273f5f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.service.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class AuthorChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { - CandidatesUtil.getOriginalAuthorId(candidateFeatures).flatMap { authorId => - FeedbackUtil.buildUserSeeFewerChildFeedbackAction( - userId = authorId, - namesByUserId = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]), - promptExternalString = externalStrings.showFewerTweetsString, - confirmationExternalString = externalStrings.showFewerTweetsConfirmationString, - engagementType = t.FeedbackEngagementType.Tweet, - stringCenter = stringCenter, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel deleted file mode 100644 index 3a4f19592..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "src/thrift/com/twitter/timelines/service:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.docx new file mode 100644 index 000000000..139d85372 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.docx new file mode 100644 index 000000000..b8d5dd930 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala deleted file mode 100644 index a23b57dee..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class BlockUserChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { - val userIdOpt = - if (candidateFeatures.getOrElse(IsRetweetFeature, false)) - candidateFeatures.getOrElse(SourceUserIdFeature, None) - else candidateFeatures.getOrElse(AuthorIdFeature, None) - - userIdOpt.flatMap { userId => - val screenNamesMap = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]) - val userScreenNameOpt = screenNamesMap.get(userId) - userScreenNameOpt.map { userScreenName => - val prompt = stringCenter.prepare( - externalStrings.blockUserString, - Map("username" -> userScreenName) - ) - ChildFeedbackAction( - feedbackType = RichBehavior, - prompt = Some(prompt), - confirmation = None, - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = Some(BottomSheet), - clientEventInfo = None, - icon = Some(icon.No), - richBehavior = Some(RichFeedbackBehaviorBlockUser(userId)), - subprompt = None - ) - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.docx new file mode 100644 index 000000000..6e4ce91a9 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala deleted file mode 100644 index 9032b53d9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNahFeedbackInfoParam -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Frown -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DontLike -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.common.{thriftscala => tlc} -import com.twitter.timelineservice.model.FeedbackInfo -import com.twitter.timelineservice.model.FeedbackMetadata -import com.twitter.timelineservice.{thriftscala => tls} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class DontLikeFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings, - authorChildFeedbackActionBuilder: AuthorChildFeedbackActionBuilder, - retweeterChildFeedbackActionBuilder: RetweeterChildFeedbackActionBuilder, - notRelevantChildFeedbackActionBuilder: NotRelevantChildFeedbackActionBuilder, - unfollowUserChildFeedbackActionBuilder: UnfollowUserChildFeedbackActionBuilder, - muteUserChildFeedbackActionBuilder: MuteUserChildFeedbackActionBuilder, - blockUserChildFeedbackActionBuilder: BlockUserChildFeedbackActionBuilder, - reportTweetChildFeedbackActionBuilder: ReportTweetChildFeedbackActionBuilder) { - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[FeedbackAction] = { - CandidatesUtil.getOriginalAuthorId(candidateFeatures).map { authorId => - val feedbackEntities = Seq( - tlc.FeedbackEntity.TweetId(candidate.id), - tlc.FeedbackEntity.UserId(authorId) - ) - val feedbackMetadata = FeedbackMetadata( - engagementType = None, - entityIds = feedbackEntities, - ttl = Some(30.days) - ) - val feedbackUrl = FeedbackInfo.feedbackUrl( - feedbackType = tls.FeedbackType.DontLike, - feedbackMetadata = feedbackMetadata, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) - ) - val childFeedbackActions = if (query.params(EnableNahFeedbackInfoParam)) { - Seq( - unfollowUserChildFeedbackActionBuilder(candidateFeatures), - muteUserChildFeedbackActionBuilder(candidateFeatures), - blockUserChildFeedbackActionBuilder(candidateFeatures), - reportTweetChildFeedbackActionBuilder(candidate) - ).flatten - } else { - Seq( - authorChildFeedbackActionBuilder(candidateFeatures), - retweeterChildFeedbackActionBuilder(candidateFeatures), - notRelevantChildFeedbackActionBuilder(candidate, candidateFeatures) - ).flatten - } - - FeedbackAction( - feedbackType = DontLike, - prompt = Some(stringCenter.prepare(externalStrings.dontLikeString)), - confirmation = Some(stringCenter.prepare(externalStrings.dontLikeConfirmationString)), - childFeedbackActions = - if (childFeedbackActions.nonEmpty) Some(childFeedbackActions) else None, - feedbackUrl = Some(feedbackUrl), - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = Some(Frown), - richBehavior = None, - subprompt = None, - encodedFeedbackRequest = None - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.docx new file mode 100644 index 000000000..1840cebdd Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala deleted file mode 100644 index 950271e4a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stringcenter.client.StringCenter -import com.twitter.stringcenter.client.core.ExternalString - -private[decorator] case class SocialContextIdAndScreenName( - socialContextId: Long, - screenName: String) - -object EngagerSocialContextBuilder { - private val UserIdRequestParamName = "user_id" - private val DirectInjectionContentSourceRequestParamName = "dis" - private val DirectInjectionIdRequestParamName = "diid" - private val DirectInjectionContentSourceSocialProofUsers = "socialproofusers" - private val SocialProofUrl = "" -} - -case class EngagerSocialContextBuilder( - contextType: GeneralContextType, - stringCenter: StringCenter, - oneUserString: ExternalString, - twoUsersString: ExternalString, - moreUsersString: ExternalString, - timelineTitle: ExternalString) { - import EngagerSocialContextBuilder._ - - def apply( - socialContextIds: Seq[Long], - query: PipelineQuery, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - val realNames = candidateFeatures.getOrElse(RealNamesFeature, Map.empty[Long, String]) - val validSocialContextIdAndScreenNames = socialContextIds.flatMap { socialContextId => - realNames - .get(socialContextId).map(screenName => - SocialContextIdAndScreenName(socialContextId, screenName)) - } - - validSocialContextIdAndScreenNames match { - case Seq(user) => - val socialContextString = - stringCenter.prepare(oneUserString, Map("user" -> user.screenName)) - Some(mkOneUserSocialContext(socialContextString, user.socialContextId)) - case Seq(firstUser, secondUser) => - val socialContextString = - stringCenter - .prepare( - twoUsersString, - Map("user1" -> firstUser.screenName, "user2" -> secondUser.screenName)) - Some( - mkManyUserSocialContext( - socialContextString, - query.getRequiredUserId, - validSocialContextIdAndScreenNames.map(_.socialContextId))) - - case firstUser +: otherUsers => - val otherUsersCount = otherUsers.size - val socialContextString = - stringCenter - .prepare( - moreUsersString, - Map("user" -> firstUser.screenName, "count" -> otherUsersCount)) - Some( - mkManyUserSocialContext( - socialContextString, - query.getRequiredUserId, - validSocialContextIdAndScreenNames.map(_.socialContextId))) - case _ => None - } - } - - private def mkOneUserSocialContext(socialContextString: String, userId: Long): GeneralContext = { - GeneralContext( - contextType = contextType, - text = socialContextString, - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = DeepLink, - url = "", - urtEndpointOptions = None - ) - ) - ) - } - - private def mkManyUserSocialContext( - socialContextString: String, - viewerId: Long, - socialContextIds: Seq[Long] - ): GeneralContext = { - GeneralContext( - contextType = contextType, - text = socialContextString, - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = UrtEndpoint, - url = SocialProofUrl, - urtEndpointOptions = Some(UrtEndpointOptions( - requestParams = Some(Map( - UserIdRequestParamName -> viewerId.toString, - DirectInjectionContentSourceRequestParamName -> DirectInjectionContentSourceSocialProofUsers, - DirectInjectionIdRequestParamName -> socialContextIds.mkString(",") - )), - title = Some(stringCenter.prepare(timelineTitle)), - cacheId = None, - subtitle = None - )) - )) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.docx new file mode 100644 index 000000000..e26f1ba76 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala deleted file mode 100644 index 74576eae8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetRealNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -/** - * Use '@A replied' when the root tweet is out-of-network and the reply is in network. - * - * This function should only be called for the root Tweet of convo modules. This is enforced by - * [[HomeTweetSocialContextBuilder]]. - */ -@Singleton -case class ExtendedReplySocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val extendedReplyString = externalStrings.socialContextExtendedReply - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - - // If these values are missing default to not showing an extended reply banner - val inNetworkRoot = candidateFeatures.getOrElse(InNetworkFeature, true) - - val inNetworkFocalTweet = - candidateFeatures.getOrElse(FocalTweetInNetworkFeature, None).getOrElse(false) - - if (!inNetworkRoot && inNetworkFocalTweet) { - - val focalTweetAuthorIdOpt = candidateFeatures.getOrElse(FocalTweetAuthorIdFeature, None) - val focalTweetRealNames = - candidateFeatures - .getOrElse(FocalTweetRealNamesFeature, None).getOrElse(Map.empty[Long, String]) - val focalTweetAuthorNameOpt = focalTweetAuthorIdOpt.flatMap(focalTweetRealNames.get) - - (focalTweetAuthorIdOpt, focalTweetAuthorNameOpt) match { - case (Some(focalTweetAuthorId), Some(focalTweetAuthorName)) => - Some( - GeneralContext( - contextType = ConversationGeneralContextType, - text = stringCenter - .prepare(extendedReplyString, placeholders = Map("user1" -> focalTweetAuthorName)), - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = DeepLink, - url = "", - urtEndpointOptions = None - )) - )) - case _ => - None - } - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.docx new file mode 100644 index 000000000..bf9e9cd89 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala deleted file mode 100644 index 8a9c214aa..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.ExternalStringRegistry -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class FeedbackStrings @Inject() ( - @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { - private val externalStringRegistry = externalStringRegistryProvider.get() - - val seeLessOftenFeedbackString = - externalStringRegistry.createProdString("Feedback.seeLessOften") - val seeLessOftenConfirmationFeedbackString = - externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.docx new file mode 100644 index 000000000..41f67fcf8 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala deleted file mode 100644 index 08534f26e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.conversions.DurationOps._ -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SeeFewer -import com.twitter.stringcenter.client.StringCenter -import com.twitter.stringcenter.client.core.ExternalString -import com.twitter.timelines.common.{thriftscala => tlc} -import com.twitter.timelines.service.{thriftscala => t} -import com.twitter.timelineservice.model.FeedbackInfo -import com.twitter.timelineservice.model.FeedbackMetadata -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.timelineservice.{thriftscala => tlst} - -object FeedbackUtil { - - val FeedbackTtl = 30.days - - def buildUserSeeFewerChildFeedbackAction( - userId: Long, - namesByUserId: Map[Long, String], - promptExternalString: ExternalString, - confirmationExternalString: ExternalString, - engagementType: t.FeedbackEngagementType, - stringCenter: StringCenter, - injectionType: Option[st.SuggestType] - ): Option[ChildFeedbackAction] = { - namesByUserId.get(userId).map { userScreenName => - val prompt = stringCenter.prepare( - promptExternalString, - Map("user" -> userScreenName) - ) - val confirmation = stringCenter.prepare( - confirmationExternalString, - Map("user" -> userScreenName) - ) - val feedbackMetadata = FeedbackMetadata( - engagementType = Some(engagementType), - entityIds = Seq(tlc.FeedbackEntity.UserId(userId)), - ttl = Some(FeedbackTtl)) - val feedbackUrl = FeedbackInfo.feedbackUrl( - feedbackType = tlst.FeedbackType.SeeFewer, - feedbackMetadata = feedbackMetadata, - injectionType = injectionType - ) - - ChildFeedbackAction( - feedbackType = SeeFewer, - prompt = Some(prompt), - confirmation = Some(confirmation), - feedbackUrl = Some(feedbackUrl), - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = None, - richBehavior = None, - subprompt = None - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.docx new file mode 100644 index 000000000..2f7e7452a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala deleted file mode 100644 index 7554f7b36..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -case class FollowedBySocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - - private val engagerSocialContextBuilder = EngagerSocialContextBuilder( - contextType = FollowGeneralContextType, - stringCenter = stringCenter, - oneUserString = externalStrings.socialContextOneUserFollowsString, - twoUsersString = externalStrings.socialContextTwoUsersFollowString, - moreUsersString = externalStrings.socialContextMoreUsersFollowString, - timelineTitle = externalStrings.socialContextFollowedByTimelineTitle - ) - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - // Only apply followed-by social context for OON Tweets - val inNetwork = candidateFeatures.getOrElse(InNetworkFeature, true) - if (!inNetwork) { - val validFollowedByUserIds = - candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Nil) - engagerSocialContextBuilder( - socialContextIds = validFollowedByUserIds, - query = query, - candidateFeatures = candidateFeatures - ) - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.docx new file mode 100644 index 000000000..34304711f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala deleted file mode 100644 index c932f6362..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNahFeedbackInfoParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.service.{thriftscala => t} -import com.twitter.timelines.util.FeedbackMetadataSerializer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeFeedbackActionInfoBuilder @Inject() ( - notInterestedTopicFeedbackActionBuilder: NotInterestedTopicFeedbackActionBuilder, - dontLikeFeedbackActionBuilder: DontLikeFeedbackActionBuilder) - extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate] { - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[FeedbackActionInfo] = { - val supportedProduct = query.product match { - case FollowingProduct => query.params(EnableNahFeedbackInfoParam) - case ForYouProduct => true - case _ => false - } - val isAuthoredByViewer = CandidatesUtil.isAuthoredByViewer(query, candidateFeatures) - - if (supportedProduct && !isAuthoredByViewer) { - val feedbackActions = Seq( - notInterestedTopicFeedbackActionBuilder(candidateFeatures), - dontLikeFeedbackActionBuilder(query, candidate, candidateFeatures) - ).flatten - val feedbackMetadata = FeedbackMetadataSerializer.serialize( - t.FeedbackMetadata(injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None))) - - Some( - FeedbackActionInfo( - feedbackActions = feedbackActions, - feedbackMetadata = Some(feedbackMetadata), - displayContext = None, - clientEventInfo = None - )) - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.docx new file mode 100644 index 000000000..1488af408 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala deleted file mode 100644 index 5138f254a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableSocialContextParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class HomeTweetSocialContextBuilder @Inject() ( - likedBySocialContextBuilder: LikedBySocialContextBuilder, - listsSocialContextBuilder: ListsSocialContextBuilder, - followedBySocialContextBuilder: FollowedBySocialContextBuilder, - topicSocialContextBuilder: TopicSocialContextBuilder, - extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder, - receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder, - popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder, - popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - features: FeatureMap - ): Option[SocialContext] = { - if (query.params(EnableSocialContextParam)) { - features.getOrElse(ConversationModuleFocalTweetIdFeature, None) match { - case None => - likedBySocialContextBuilder(query, candidate, features) - .orElse(followedBySocialContextBuilder(query, candidate, features)) - .orElse(topicSocialContextBuilder(query, candidate, features)) - .orElse(popularVideoSocialContextBuilder(query, candidate, features)) - .orElse(listsSocialContextBuilder(query, candidate, features)) - .orElse(popularInYourAreaSocialContextBuilder(query, candidate, features)) - case Some(_) => - val conversationId = features.getOrElse(ConversationModuleIdFeature, None) - // Only hydrate the social context into the root tweet in a conversation module - if (conversationId.contains(candidate.id)) { - extendedReplySocialContextBuilder(query, candidate, features) - .orElse(receivedReplySocialContextBuilder(query, candidate, features)) - } else None - } - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.docx new file mode 100644 index 000000000..8c868f1c0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala deleted file mode 100644 index 49017e8cf..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.service.{thriftscala => tl} -import com.twitter.timelines.util.FeedbackRequestSerializer -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import com.twitter.timelineservice.thriftscala.FeedbackType - -object HomeWhoToFollowFeedbackActionInfoBuilder { - private val FeedbackMetadata = tl.FeedbackMetadata( - injectionType = Some(SuggestType.WhoToFollow), - engagementType = None, - entityIds = Seq.empty, - ttlMs = None - ) - private val FeedbackRequest = - tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata) - private val EncodedFeedbackRequest = - FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest)) -} - -@Singleton -case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() ( - feedbackStrings: FeedbackStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { - - private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( - seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, - seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString, - stringCenter = stringCenterProvider.get(), - encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest) - ) - - override def apply( - query: PipelineQuery, - candidate: UserCandidate, - candidateFeatures: FeatureMap - ): Option[FeedbackActionInfo] = - whoToFollowFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.docx new file mode 100644 index 000000000..4d57279dd Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala deleted file mode 100644 index 270ea66f6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.service.{thriftscala => tl} -import com.twitter.timelines.util.FeedbackRequestSerializer -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import com.twitter.timelineservice.thriftscala.FeedbackType - -object HomeWhoToSubscribeFeedbackActionInfoBuilder { - private val FeedbackMetadata = tl.FeedbackMetadata( - injectionType = Some(SuggestType.WhoToSubscribe), - engagementType = None, - entityIds = Seq.empty, - ttlMs = None - ) - private val FeedbackRequest = - tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata) - private val EncodedFeedbackRequest = - FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest)) -} - -@Singleton -case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() ( - feedbackStrings: FeedbackStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { - - private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( - seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, - seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString, - stringCenter = stringCenterProvider.get(), - encodedFeedbackRequest = - Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest) - ) - - override def apply( - query: PipelineQuery, - candidate: UserCandidate, - candidateFeatures: FeatureMap - ): Option[FeedbackActionInfo] = - whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.docx new file mode 100644 index 000000000..7f1a6e646 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala deleted file mode 100644 index fbc65be13..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.LikeGeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -case class LikedBySocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - - private val engagerSocialContextBuilder = EngagerSocialContextBuilder( - contextType = LikeGeneralContextType, - stringCenter = stringCenter, - oneUserString = externalStrings.socialContextOneUserLikedString, - twoUsersString = externalStrings.socialContextTwoUsersLikedString, - moreUsersString = externalStrings.socialContextMoreUsersLikedString, - timelineTitle = externalStrings.socialContextLikedByTimelineTitle - ) - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - - // Liked by users are valid only if they pass both the SGS and Perspective filters. - val validLikedByUserIds = - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Nil) - .filter( - candidateFeatures.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Nil).toSet.contains) - - engagerSocialContextBuilder( - socialContextIds = validLikedByUserIds, - query = query, - candidateFeatures = candidateFeatures - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.docx new file mode 100644 index 000000000..0961ac362 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala deleted file mode 100644 index e5e746223..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelineservice.suggests.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -/** - * "Your Lists" will be rendered for the context and a url link for your lists. - */ -@Singleton -case class ListsSocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val listString = externalStrings.ownedSubscribedListsModuleHeaderString - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - candidateFeatures.get(SuggestTypeFeature) match { - case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet => - val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None)) - Some( - GeneralContext( - contextType = ListGeneralContextType, - text = stringCenter.prepare(listString), - url = userName.map(name => ""), - contextImageUrls = None, - landingUrl = None - )) - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.docx new file mode 100644 index 000000000..56caf526b Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala deleted file mode 100644 index 542ed8a67..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class MuteUserChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply( - candidateFeatures: FeatureMap - ): Option[ChildFeedbackAction] = { - val userIdOpt = - if (candidateFeatures.getOrElse(IsRetweetFeature, false)) - candidateFeatures.getOrElse(SourceUserIdFeature, None) - else candidateFeatures.getOrElse(AuthorIdFeature, None) - - userIdOpt.flatMap { userId => - val screenNamesMap = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]) - val userScreenNameOpt = screenNamesMap.get(userId) - userScreenNameOpt.map { userScreenName => - val prompt = stringCenter.prepare( - externalStrings.muteUserString, - Map("username" -> userScreenName) - ) - ChildFeedbackAction( - feedbackType = RichBehavior, - prompt = Some(prompt), - confirmation = None, - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = Some(icon.SpeakerOff), - richBehavior = Some(RichFeedbackBehaviorToggleMuteUser(userId)), - subprompt = None - ) - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.docx new file mode 100644 index 000000000..d7232dbfd Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala deleted file mode 100644 index 2ec520e11..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class NotInterestedTopicFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply( - candidateFeatures: FeatureMap - ): Option[FeedbackAction] = { - val isOutOfNetwork = !candidateFeatures.getOrElse(InNetworkFeature, true) - val validFollowedByUserIds = - candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Nil) - val validLikedByUserIds = - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Nil) - .filter( - candidateFeatures.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Nil).toSet.contains) - - if (isOutOfNetwork && validLikedByUserIds.isEmpty && validFollowedByUserIds.isEmpty) { - val topicIdSocialContext = candidateFeatures.getOrElse(TopicIdSocialContextFeature, None) - val topicContextFunctionalityType = - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None) - - (topicIdSocialContext, topicContextFunctionalityType) match { - case (Some(topicId), Some(topicContextFunctionalityType)) - if topicContextFunctionalityType == RecommendationTopicContextFunctionalityType || - topicContextFunctionalityType == RecWithEducationTopicContextFunctionalityType => - Some( - FeedbackAction( - feedbackType = RichBehavior, - prompt = None, - confirmation = None, - childFeedbackActions = None, - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = None, - richBehavior = - Some(RichFeedbackBehaviorMarkNotInterestedTopic(topicId = topicId.toString)), - subprompt = None, - encodedFeedbackRequest = None - ) - ) - case _ => None - } - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.docx new file mode 100644 index 000000000..ead75ed4a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala deleted file mode 100644 index 39df781ce..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.NotRelevant -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.common.{thriftscala => tlc} -import com.twitter.timelineservice.model.FeedbackInfo -import com.twitter.timelineservice.model.FeedbackMetadata -import com.twitter.timelineservice.{thriftscala => tlst} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class NotRelevantChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply( - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[ChildFeedbackAction] = { - val prompt = stringCenter.prepare(externalStrings.notRelevantString) - val confirmation = stringCenter.prepare(externalStrings.notRelevantConfirmationString) - val feedbackMetadata = FeedbackMetadata( - engagementType = None, - entityIds = Seq(tlc.FeedbackEntity.TweetId(candidate.id)), - ttl = Some(FeedbackUtil.FeedbackTtl)) - val feedbackUrl = FeedbackInfo.feedbackUrl( - feedbackType = tlst.FeedbackType.NotRelevant, - feedbackMetadata = feedbackMetadata, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) - ) - - Some( - ChildFeedbackAction( - feedbackType = NotRelevant, - prompt = Some(prompt), - confirmation = Some(confirmation), - feedbackUrl = Some(feedbackUrl), - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = None, - richBehavior = None, - subprompt = None - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.docx new file mode 100644 index 000000000..937584b52 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala deleted file mode 100644 index 3782dd115..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -case class PopularInYourAreaSocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None) - if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) { - Some( - GeneralContext( - contextType = LocationGeneralContextType, - text = stringCenter.prepare(popularInYourAreaString), - url = None, - contextImageUrls = None, - landingUrl = None - )) - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.docx new file mode 100644 index 000000000..af6622b4d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala deleted file mode 100644 index 7eef7f180..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -case class PopularVideoSocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val popularVideoString = externalStrings.socialContextPopularVideoString - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None) - if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) { - Some( - GeneralContext( - contextType = SparkleGeneralContextType, - text = stringCenter.prepare(popularVideoString), - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = DeepLink, - url = "" - ) - ) - )) - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.docx new file mode 100644 index 000000000..6c084b0d2 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala deleted file mode 100644 index f1af07c98..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -/** - * Use '@A received a reply' as social context when the root Tweet is in network and the focal tweet is OON. - * - * This function should only be called for the root Tweet of convo modules. This is enforced by - * [[HomeTweetSocialContextBuilder]]. - */ -@Singleton -case class ReceivedReplySocialContextBuilder @Inject() ( - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter]) - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - private val stringCenter = stringCenterProvider.get() - private val receivedReplyString = externalStrings.socialContextReceivedReply - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - - // If these values are missing default to not showing a received a reply banner - val inNetwork = candidateFeatures.getOrElse(InNetworkFeature, false) - val inNetworkFocalTweet = - candidateFeatures.getOrElse(FocalTweetInNetworkFeature, None).getOrElse(true) - - if (inNetwork && !inNetworkFocalTweet) { - - val authorIdOpt = candidateFeatures.getOrElse(AuthorIdFeature, None) - val realNames = candidateFeatures.getOrElse(RealNamesFeature, Map.empty[Long, String]) - val authorNameOpt = authorIdOpt.flatMap(realNames.get) - - (authorIdOpt, authorNameOpt) match { - case (Some(authorId), Some(authorName)) => - Some( - GeneralContext( - contextType = ConversationGeneralContextType, - text = stringCenter - .prepare(receivedReplyString, placeholders = Map("user1" -> authorName)), - url = None, - contextImageUrls = None, - landingUrl = Some( - Url( - urlType = DeepLink, - url = "", - urtEndpointOptions = None - ) - ) - ) - ) - case _ => None - } - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.docx new file mode 100644 index 000000000..f7209cc90 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala deleted file mode 100644 index a4ef0cbb2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ReportTweetChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply( - candidate: TweetCandidate - ): Option[ChildFeedbackAction] = { - Some( - ChildFeedbackAction( - feedbackType = RichBehavior, - prompt = Some(stringCenter.prepare(externalStrings.reportTweetString)), - confirmation = None, - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = Some(icon.Flag), - richBehavior = Some(RichFeedbackBehaviorReportTweet(candidate.id)), - subprompt = None - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.docx new file mode 100644 index 000000000..54cfe4001 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala deleted file mode 100644 index 006e93b58..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.service.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class RetweeterChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { - val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false) - - if (isRetweet) { - candidateFeatures.getOrElse(AuthorIdFeature, None).flatMap { retweeterId => - FeedbackUtil.buildUserSeeFewerChildFeedbackAction( - userId = retweeterId, - namesByUserId = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]), - promptExternalString = externalStrings.showFewerRetweetsString, - confirmationExternalString = externalStrings.showFewerRetweetsConfirmationString, - engagementType = t.FeedbackEngagementType.Retweet, - stringCenter = stringCenter, - injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) - ) - } - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.docx new file mode 100644 index 000000000..0fe1e36f1 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala deleted file mode 100644 index 7d01382c3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class TopicSocialContextBuilder @Inject() () - extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { - - def apply( - query: PipelineQuery, - candidate: TweetCandidate, - candidateFeatures: FeatureMap - ): Option[SocialContext] = { - val inNetwork = candidateFeatures.getOrElse(InNetworkFeature, true) - if (!inNetwork) { - val topicIdSocialContextOpt = candidateFeatures.getOrElse(TopicIdSocialContextFeature, None) - val topicContextFunctionalityTypeOpt = - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None) - (topicIdSocialContextOpt, topicContextFunctionalityTypeOpt) match { - case (Some(topicId), Some(topicContextFunctionalityType)) => - Some( - TopicContext( - topicId = topicId.toString, - functionalityType = Some(topicContextFunctionalityType) - )) - case _ => None - } - } else { - None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.docx new file mode 100644 index 000000000..6846e6552 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala deleted file mode 100644 index 864ee06d9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.home_mixer.functional_component.decorator.urt.builder - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.response.urt.icon -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class UnfollowUserChildFeedbackActionBuilder @Inject() ( - @ProductScoped stringCenter: StringCenter, - externalStrings: HomeMixerExternalStrings) { - - def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { - val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, false) - val userIdOpt = candidateFeatures.getOrElse(AuthorIdFeature, None) - - if (isInNetwork) { - userIdOpt.flatMap { userId => - val screenNamesMap = - candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]) - val userScreenNameOpt = screenNamesMap.get(userId) - userScreenNameOpt.map { userScreenName => - val prompt = stringCenter.prepare( - externalStrings.unfollowUserString, - Map("username" -> userScreenName) - ) - val confirmation = stringCenter.prepare( - externalStrings.unfollowUserConfirmationString, - Map("username" -> userScreenName) - ) - ChildFeedbackAction( - feedbackType = RichBehavior, - prompt = Some(prompt), - confirmation = Some(confirmation), - feedbackUrl = None, - hasUndoAction = Some(true), - confirmationDisplayType = None, - clientEventInfo = None, - icon = Some(icon.Unfollow), - richBehavior = Some(RichFeedbackBehaviorToggleFollowUser(userId)), - subprompt = None - ) - } - } - } else None - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel deleted file mode 100644 index 60c104143..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel +++ /dev/null @@ -1,64 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "configapi/configapi-decider", - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", - "joinkey/src/main/scala/com/twitter/joinkey/context", - "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/search/common/util/lang", - "src/scala/com/twitter/timelines/prediction/adapters/request_context", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", - "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "src/thrift/com/twitter/user_session_store:thrift-java", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "stitch/stitch-core", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph", - "stitch/stitch-timelineservice", - "stitch/stitch-tweetypie", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "user_session_store/src/main/scala/com/twitter/user_session_store", - "util/util-core", - ], - exports = [ - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.docx new file mode 100644 index 000000000..56bd2aafa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.docx new file mode 100644 index 000000000..41e9fdfa3 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.scala deleted file mode 100644 index 976cd1e30..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.timelinemixer.clients.manhattan.InjectionHistoryClient -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.manhattan.DismissInfo -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import javax.inject.Inject -import javax.inject.Singleton - -object DismissInfoQueryFeatureHydrator { - val DismissInfoSuggestTypes = Seq(SuggestType.WhoToFollow) -} - -@Singleton -case class DismissInfoQueryFeatureHydrator @Inject() ( - dismissInfoClient: InjectionHistoryClient) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("DismissInfo") - - override val features: Set[Feature[_, _]] = Set(DismissInfoFeature) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = - Stitch.callFuture { - dismissInfoClient - .readDismissInfoEntries( - query.getRequiredUserId, - DismissInfoQueryFeatureHydrator.DismissInfoSuggestTypes).map { response => - val dismissInfoMap = response.mapValues(DismissInfo.fromThrift) - FeatureMapBuilder().add(DismissInfoFeature, dismissInfoMap).build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8, 50, 60, 60) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.docx new file mode 100644 index 000000000..6f8d4de95 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala deleted file mode 100644 index c0793be4b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClient -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class FeedbackHistoryQueryFeatureHydrator @Inject() ( - feedbackHistoryClient: FeedbackHistoryManhattanClient) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory") - - override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature) - - override def hydrate( - query: PipelineQuery - ): Stitch[FeatureMap] = - Stitch - .callFuture(feedbackHistoryClient.get(query.getRequiredUserId)) - .map { feedbackHistory => - FeatureMapBuilder().add(FeedbackHistoryFeature, feedbackHistory).build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.docx new file mode 100644 index 000000000..6362b3021 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala deleted file mode 100644 index 431711900..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.model.HomeFeatures.UserFollowingCountFeature -import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature -import com.twitter.home_mixer.model.HomeFeatures.UserTypeFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class GizmoduckUserQueryFeatureHydrator @Inject() (gizmoduck: Gizmoduck) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GizmoduckUser") - - override val features: Set[Feature[_, _]] = - Set(UserFollowingCountFeature, UserTypeFeature, UserScreenNameFeature) - - private val queryFields: Set[gt.QueryFields] = - Set(gt.QueryFields.Counts, gt.QueryFields.Safety, gt.QueryFields.Profile) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - gizmoduck - .getUserById( - userId = userId, - queryFields = queryFields, - context = gt.LookupContext(forUserId = Some(userId), includeSoftUsers = true)) - .map { user => - FeatureMapBuilder() - .add(UserFollowingCountFeature, user.counts.map(_.following.toInt)) - .add(UserTypeFeature, Some(user.userType)) - .add(UserScreenNameFeature, user.profile.map(_.screenName)) - .build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.docx new file mode 100644 index 000000000..78d1f598d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala deleted file mode 100644 index 6ece16ce0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient -import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} -import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ImpressionBloomFilterQueryFeatureHydrator[ - Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( - bloomFilterClient: ManhattanStoreClient[ - blm.ImpressionBloomFilterKey, - blm.ImpressionBloomFilterSeq - ]) extends QueryFeatureHydrator[Query] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( - "ImpressionBloomFilter") - - private val ImpressionBloomFilterTTL = 7.day - - override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature) - - private val SurfaceArea = blm.SurfaceArea.HomeTimeline - - override def hydrate(query: Query): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - bloomFilterClient - .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea)) - .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty))) - .map { bloomFilterSeq => - val updatedBloomFilterSeq = - if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq - else { - ImpressionBloomFilter.addSeenTweetIds( - surfaceArea = SurfaceArea, - tweetIds = query.seenTweetIds.get, - bloomFilterSeq = bloomFilterSeq, - timeToLive = ImpressionBloomFilterTTL, - falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam) - ) - } - FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.docx new file mode 100644 index 000000000..aeb604965 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala deleted file mode 100644 index 55f002b1f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object InNetworkFeatureHydrator - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork") - - override val features: Set[Feature[_, _]] = Set(InNetworkFeature) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val viewerId = query.getRequiredUserId - val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet - - val featureMaps = candidates.map { candidate => - // We use authorId and not sourceAuthorId here so that retweets are defined as in network - val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId => - // Users cannot follow themselves but this is in network by definition - val isSelfTweet = authorId == viewerId - isSelfTweet || followedUserIds.contains(authorId) - } - FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build() - } - Stitch.value(featureMaps) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.docx new file mode 100644 index 000000000..a79961a8d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.scala deleted file mode 100644 index 9f3049049..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.user_session_store.ReadRequest -import com.twitter.user_session_store.ReadWriteUserSessionStore -import com.twitter.user_session_store.UserSessionDataset -import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset -import com.twitter.util.Time - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class LastNonPollingTimeQueryFeatureHydrator @Inject() ( - userSessionStore: ReadWriteUserSessionStore) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("LastNonPollingTime") - - override val features: Set[Feature[_, _]] = Set( - FollowingLastNonPollingTimeFeature, - LastNonPollingTimeFeature, - NonPollingTimesFeature - ) - - private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - userSessionStore - .read(ReadRequest(query.getRequiredUserId, datasets)) - .map { userSession => - val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) - - val lastNonPollingTime = nonPollingTimestamps - .flatMap(_.nonPollingTimestampsMs.headOption) - .map(Time.fromMilliseconds) - - val followingLastNonPollingTime = nonPollingTimestamps - .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) - .map(Time.fromMilliseconds) - - val nonPollingTimes = nonPollingTimestamps - .map(_.nonPollingTimestampsMs) - .getOrElse(Seq.empty) - - FeatureMapBuilder() - .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) - .add(LastNonPollingTimeFeature, lastNonPollingTime) - .add(NonPollingTimesFeature, nonPollingTimes) - .build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.docx new file mode 100644 index 000000000..87385854c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala deleted file mode 100644 index ede075e5c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNahFeedbackInfoParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import com.twitter.util.Return -import javax.inject.Inject -import javax.inject.Singleton - -protected case class ProfileNames(screenName: String, realName: String) - -@Singleton -class NamesFeatureHydrator @Inject() (gizmoduck: Gizmoduck) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with Conditionally[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Names") - - override val features: Set[Feature[_, _]] = Set(ScreenNamesFeature, RealNamesFeature) - - override def onlyIf(query: PipelineQuery): Boolean = query.product match { - case FollowingProduct => query.params(EnableNahFeedbackInfoParam) - case _ => true - } - - private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Profile) - - /** - * The UI currently only ever displays the first 2 names in social context lines - * E.g. "User and 3 others like" or "UserA and UserB liked" - */ - private val MaxCountUsers = 2 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - - val candidateUserIdsMap = candidates.map { candidate => - candidate.candidate.id -> - (candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++ - candidate.features.getOrElse(FollowedByUserIdsFeature, Nil).take(MaxCountUsers) ++ - candidate.features.getOrElse(AuthorIdFeature, None) ++ - candidate.features.getOrElse(SourceUserIdFeature, None)).distinct - }.toMap - - val distinctUserIds = candidateUserIdsMap.values.flatten.toSeq.distinct - - Stitch - .collectToTry(distinctUserIds.map(userId => gizmoduck.getUserById(userId, queryFields))) - .map { allUsers => - val idToProfileNamesMap = allUsers.flatMap { - case Return(allUser) => - allUser.profile - .map(profile => allUser.id -> ProfileNames(profile.screenName, profile.name)) - case _ => None - }.toMap - - val validUserIds = idToProfileNamesMap.keySet - - candidates.map { candidate => - val combinedMap = candidateUserIdsMap - .getOrElse(candidate.candidate.id, Nil) - .flatMap { - case userId if validUserIds.contains(userId) => - idToProfileNamesMap.get(userId).map(profileNames => userId -> profileNames) - case _ => None - } - - val perCandidateRealNameMap = combinedMap.map { case (k, v) => k -> v.realName }.toMap - val perCandidateScreenNameMap = combinedMap.map { case (k, v) => k -> v.screenName }.toMap - - FeatureMapBuilder() - .add(ScreenNamesFeature, perCandidateScreenNameMap) - .add(RealNamesFeature, perCandidateRealNameMap) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.docx new file mode 100644 index 000000000..c1dc4b0bb Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala deleted file mode 100644 index b7aae811b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelines.util.client_info.ClientPlatform -import com.twitter.timelineservice.model.TimelineQuery -import com.twitter.timelineservice.model.core.TimelineKind -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class PersistenceStoreQueryFeatureHydrator @Inject() ( - timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3], - statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore") - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize") - - private val WhoToFollowExcludedUserIdsLimit = 1000 - private val ServedTweetIdsDuration = 10.minutes - private val ServedTweetIdsLimit = 100 - private val ServedTweetPreviewIdsDuration = 10.hours - private val ServedTweetPreviewIdsLimit = 10 - - override val features: Set[Feature[_, _]] = - Set( - ServedTweetIdsFeature, - ServedTweetPreviewIdsFeature, - PersistenceEntriesFeature, - WhoToFollowExcludedUserIdsFeature) - - private val supportedClients = Seq( - ClientPlatform.IPhone, - ClientPlatform.IPad, - ClientPlatform.Mac, - ClientPlatform.Android, - ClientPlatform.Web, - ClientPlatform.RWeb, - ClientPlatform.TweetDeckGryphon - ) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val timelineKind = query.product match { - case FollowingProduct => TimelineKind.homeLatest - case ForYouProduct => TimelineKind.home - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - val timelineQuery = TimelineQuery(id = query.getRequiredUserId, kind = timelineKind) - - Stitch.callFuture { - timelineResponseBatchesClient - .get(query = timelineQuery, clientPlatforms = supportedClients) - .map { timelineResponses => - // Note that the WTF entries are not being scoped by ClientPlatform - val whoToFollowUserIds = timelineResponses - .flatMap { timelineResponse => - timelineResponse.entries - .filter(_.entityIdType == EntityIdType.WhoToFollow) - .flatMap(_.itemIds.toSeq.flatMap(_.flatMap(_.userId))) - }.take(WhoToFollowExcludedUserIdsLimit) - - val clientPlatform = ClientPlatform.fromQueryOptions( - clientAppId = query.clientContext.appId, - userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString)) - - val servedTweetIds = timelineResponses - .filter(_.clientPlatform == clientPlatform) - .filter(_.servedTime >= Time.now - ServedTweetIdsDuration) - .sortBy(-_.servedTime.inMilliseconds) - .flatMap( - _.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit)) - - servedTweetIdsSizeStat.add(servedTweetIds.size) - - val servedTweetPreviewIds = timelineResponses - .filter(_.clientPlatform == clientPlatform) - .filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration) - .sortBy(-_.servedTime.inMilliseconds) - .flatMap(_.entries - .filter(_.entityIdType == EntityIdType.TweetPreview) - .flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit)) - - FeatureMapBuilder() - .add(ServedTweetIdsFeature, servedTweetIds) - .add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds) - .add(PersistenceEntriesFeature, timelineResponses) - .add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds) - .build() - } - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7, 50, 60, 60) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.docx new file mode 100644 index 000000000..48a599d5c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala deleted file mode 100644 index 602a8b2dc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService -import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives -import com.twitter.timelineservice.thriftscala.PerspectiveType -import com.twitter.timelineservice.thriftscala.PerspectiveType.Favorited -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Filter out unlike edges from liked-by tweets - * Useful if the likes come from a cache and because UTEG does not fully remove unlike edges. - */ -@Singleton -class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService: TimelineService) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("PerspectiveFilteredSocialContext") - - override val features: Set[Feature[_, _]] = Set(PerspectiveFilteredLikedByUserIdsFeature) - - private val MaxCountUsers = 10 - private val favoritePerspectiveSet: Set[PerspectiveType] = Set(Favorited) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { - val engagingUserIdtoTweetId = candidates.flatMap { candidate => - candidate.features - .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) - .map(favoritedBy => favoritedBy -> candidate.candidate.id) - } - - val queries = engagingUserIdtoTweetId.map { - case (userId, tweetId) => - GetPerspectives.Query(userId = userId, tweetId = tweetId, types = favoritePerspectiveSet) - } - - Stitch.collect(queries.map(timelineService.getPerspective)).map { perspectiveResults => - val validUserIdTweetIds: Set[(Long, Long)] = - queries - .zip(perspectiveResults) - .collect { case (query, perspective) if perspective.favorited => query } - .map(query => (query.userId, query.tweetId)) - .toSet - - candidates.map { candidate => - val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features - .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) - .filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) } - - FeatureMapBuilder() - .add(PerspectiveFilteredLikedByUserIdsFeature, perspectiveFilteredFavoritedByUserIds) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.docx new file mode 100644 index 000000000..1d46d80d1 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala deleted file mode 100644 index 92ab90f2c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScores -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore -import com.twitter.wtf.candidate.{thriftscala => wtf} -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() ( - @Named(RealGraphInNetworkScores) store: ReadableStore[Long, Seq[wtf.Candidate]]) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealGraphInNetworkScores") - - override val features: Set[Feature[_, _]] = Set(RealGraphInNetworkScoresFeature) - - private val RealGraphCandidateCount = 1000 - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - Stitch.callFuture(store.get(query.getRequiredUserId)).map { realGraphFollowedUsers => - val realGraphScoresFeatures = realGraphFollowedUsers - .getOrElse(Seq.empty) - .sortBy(-_.score) - .map(candidate => candidate.userId -> scaleScore(candidate.score)) - .take(RealGraphCandidateCount) - .toMap - - FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build() - } - } - - // Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97] - private def scaleScore(score: Double): Double = - if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.docx new file mode 100644 index 000000000..0bf996b51 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala deleted file mode 100644 index 3968c9532..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.finagle.tracing.Annotation.BinaryAnnotation -import com.twitter.finagle.tracing.ForwardAnnotation -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.joinkey.context.RequestJoinKeyContext -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.search.common.util.lang.ThriftLanguageUtil -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp -import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp -import java.util.UUID -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RequestQueryFeatureHydrator[ - Query <: PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext] @Inject() ( -) extends QueryFeatureHydrator[Query] { - - override val features: Set[Feature[_, _]] = Set( - AccountAgeFeature, - ClientIdFeature, - DeviceLanguageFeature, - GetInitialFeature, - GetMiddleFeature, - GetNewerFeature, - GetOlderFeature, - GuestIdFeature, - HasDarkRequestFeature, - IsForegroundRequestFeature, - IsLaunchRequestFeature, - PollingFeature, - PullToRefreshFeature, - RequestJoinIdFeature, - ServedRequestIdFeature, - TimestampFeature, - TimestampGMTDowFeature, - TimestampGMTHourFeature, - ViewerIdFeature - ) - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Request") - - private val DarkRequestAnnotation = "clnt/has_dark_request" - - // Convert Language code to ISO 639-3 format - private def getLanguageISOFormatByCode(languageCode: String): String = - ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguageUtil.getThriftLanguageOf(languageCode)) - - private def getRequestJoinId(servedRequestId: Long): Option[Long] = - Some(RequestJoinKeyContext.current.flatMap(_.requestJoinId).getOrElse(servedRequestId)) - - private def hasDarkRequest: Option[Boolean] = ForwardAnnotation.current - .getOrElse(Seq[BinaryAnnotation]()) - .find(_.key == DarkRequestAnnotation) - .map(_.value.asInstanceOf[Boolean]) - - override def hydrate(query: Query): Stitch[FeatureMap] = { - val requestContext = query.deviceContext.flatMap(_.requestContextValue) - val servedRequestId = UUID.randomUUID.getMostSignificantBits - val timestamp = query.queryTime.inMilliseconds - - val featureMap = FeatureMapBuilder() - .add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt)) - .add(ClientIdFeature, query.clientContext.appId) - .add(DeviceLanguageFeature, query.getLanguageCode.map(getLanguageISOFormatByCode)) - .add( - GetInitialFeature, - query.pipelineCursor.forall(cursor => cursor.id.isEmpty && cursor.gapBoundaryId.isEmpty)) - .add( - GetMiddleFeature, - query.pipelineCursor.exists(cursor => - cursor.id.isDefined && cursor.gapBoundaryId.isDefined && - cursor.cursorType.contains(GapCursor))) - .add( - GetNewerFeature, - query.pipelineCursor.exists(cursor => - cursor.id.isDefined && cursor.gapBoundaryId.isEmpty && - cursor.cursorType.contains(TopCursor))) - .add( - GetOlderFeature, - query.pipelineCursor.exists(cursor => - cursor.id.isDefined && cursor.gapBoundaryId.isEmpty && - cursor.cursorType.contains(BottomCursor))) - .add(GuestIdFeature, query.clientContext.guestId) - .add(IsForegroundRequestFeature, requestContext.contains(RequestContext.Foreground)) - .add(IsLaunchRequestFeature, requestContext.contains(RequestContext.Launch)) - .add(PollingFeature, query.deviceContext.exists(_.isPolling.contains(true))) - .add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh)) - .add(ServedRequestIdFeature, Some(servedRequestId)) - .add(RequestJoinIdFeature, getRequestJoinId(servedRequestId)) - .add(TimestampFeature, timestamp) - .add(TimestampGMTDowFeature, dowFromTimestamp(timestamp)) - .add(TimestampGMTHourFeature, hourFromTimestamp(timestamp)) - .add(HasDarkRequestFeature, hasDarkRequest) - .add( - ViewerIdFeature, - query.getOptionalUserId - .orElse(query.getGuestId).getOrElse( - throw PipelineFailure(BadRequest, "Missing viewer id"))) - .build() - - Stitch.value(featureMap) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.docx new file mode 100644 index 000000000..6e9c600ae Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala deleted file mode 100644 index 92c351ecf..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala +++ /dev/null @@ -1,105 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import javax.inject.Inject -import javax.inject.Singleton - -/** - * This hydrator takes liked-by and followed-by user ids and checks via SGS that the viewer is - * following the engager, that the viewer is not blocking the engager, that the engager is not - * blocking the viewer, and that the viewer has not muted the engager. - */ -@Singleton -class SGSValidSocialContextFeatureHydrator @Inject() ( - socialGraph: SocialGraph) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("SGSValidSocialContext") - - override val features: Set[Feature[_, _]] = Set( - SGSValidFollowedByUserIdsFeature, - SGSValidLikedByUserIdsFeature - ) - - private val MaxCountUsers = 10 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { - val allSocialContextUserIds = - candidates.flatMap { candidate => - candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++ - candidate.features.getOrElse(FollowedByUserIdsFeature, Nil).take(MaxCountUsers) - }.distinct - - getValidUserIds(query.getRequiredUserId, allSocialContextUserIds).map { validUserIds => - candidates.map { candidate => - val sgsFilteredLikedByUserIds = - candidate.features - .getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) - .filter(validUserIds.contains) - - val sgsFilteredFollowedByUserIds = - candidate.features - .getOrElse(FollowedByUserIdsFeature, Nil).take(MaxCountUsers) - .filter(validUserIds.contains) - - FeatureMapBuilder() - .add(SGSValidFollowedByUserIdsFeature, sgsFilteredFollowedByUserIds) - .add(SGSValidLikedByUserIdsFeature, sgsFilteredLikedByUserIds) - .build() - } - } - } - - private def getValidUserIds( - viewerId: Long, - socialProofUserIds: Seq[Long] - ): Stitch[Seq[Long]] = { - if (socialProofUserIds.nonEmpty) { - val request = sg.IdsRequest( - relationships = Seq( - sg.SrcRelationship( - viewerId, - sg.RelationshipType.Following, - targets = Some(socialProofUserIds), - hasRelationship = true), - sg.SrcRelationship( - viewerId, - sg.RelationshipType.Blocking, - targets = Some(socialProofUserIds), - hasRelationship = false), - sg.SrcRelationship( - viewerId, - sg.RelationshipType.BlockedBy, - targets = Some(socialProofUserIds), - hasRelationship = false), - sg.SrcRelationship( - viewerId, - sg.RelationshipType.Muting, - targets = Some(socialProofUserIds), - hasRelationship = false) - ), - pageRequest = Some(sg.PageRequest(selectAll = Some(true))) - ) - socialGraph.ids(request).map(_.ids) - } else Stitch.Nil - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.docx new file mode 100644 index 000000000..ce02e4681 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala deleted file mode 100644 index 0fc6fd1f5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.TweetImpressionsFeature -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.impression.{thriftscala => t} -import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient -import com.twitter.util.Duration -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class TweetImpressionsQueryFeatureHydrator[ - Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( - manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) - extends QueryFeatureHydrator[Query] { - - private val TweetImpressionTTL = 2.days - private val TweetImpressionCap = 5000 - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions") - - override val features: Set[Feature[_, _]] = Set(TweetImpressionsFeature) - - override def hydrate(query: Query): Stitch[FeatureMap] = { - manhattanTweetImpressionStoreClient.get(query.getRequiredUserId).map { entriesOpt => - val entries = entriesOpt.map(_.entries).toSeq.flatten - val updatedImpressions = - if (query.seenTweetIds.forall(_.isEmpty)) entries - else updateTweetImpressions(entries, query.seenTweetIds.get) - - FeatureMapBuilder().add(TweetImpressionsFeature, updatedImpressions).build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) - - /** - * 1) Check timestamps and remove expired tweets based on [[TweetImpressionTTL]] - * 2) Filter duplicates between current tweets and those in the impression store (remove older ones) - * 3) Prepend new (Timestamp, Seq[TweetIds]) to the tweets from the impression store - * 4) Truncate older tweets if sum of all tweets across timestamps >= [[TweetImpressionCap]], - */ - private[feature_hydrator] def updateTweetImpressions( - tweetImpressionsFromStore: Seq[t.TweetImpressionsEntry], - seenIdsFromClient: Seq[Long], - currentTime: Long = Time.now.inMilliseconds, - tweetImpressionTTL: Duration = TweetImpressionTTL, - tweetImpressionCap: Int = TweetImpressionCap, - ): Seq[t.TweetImpressionsEntry] = { - val seenIdsFromClientSet = seenIdsFromClient.toSet - val dedupedTweetImpressionsFromStore: Seq[t.TweetImpressionsEntry] = tweetImpressionsFromStore - .collect { - case t.TweetImpressionsEntry(ts, tweetIds) - if Time.fromMilliseconds(ts).untilNow < tweetImpressionTTL => - t.TweetImpressionsEntry(ts, tweetIds.filterNot(seenIdsFromClientSet.contains)) - }.filter { _.tweetIds.nonEmpty } - - val mergedTweetImpressionsEntries = - t.TweetImpressionsEntry(currentTime, seenIdsFromClient) +: dedupedTweetImpressionsFromStore - val initialTweetImpressionsWithCap = (Seq.empty[t.TweetImpressionsEntry], tweetImpressionCap) - - val (truncatedTweetImpressionsEntries: Seq[t.TweetImpressionsEntry], _) = - mergedTweetImpressionsEntries - .foldLeft(initialTweetImpressionsWithCap) { - case ( - (tweetImpressions: Seq[t.TweetImpressionsEntry], remainingCap), - t.TweetImpressionsEntry(ts, tweetIds)) if remainingCap > 0 => - ( - t.TweetImpressionsEntry(ts, tweetIds.take(remainingCap)) +: tweetImpressions, - remainingCap - tweetIds.size) - case (tweetImpressionsWithCap, _) => tweetImpressionsWithCap - } - truncatedTweetImpressionsEntries.reverse - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.docx new file mode 100644 index 000000000..6fd26eb56 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala deleted file mode 100644 index f2effa1b2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala +++ /dev/null @@ -1,179 +0,0 @@ -package com.twitter.home_mixer.functional_component.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.util.tweetypie.RequestFields -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.spam.rtf.{thriftscala => rtf} -import com.twitter.stitch.Stitch -import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} -import com.twitter.tweetypie.{thriftscala => tp} -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TweetypieFeatureHydrator @Inject() ( - tweetypieStitchClient: TweetypieStitchClient, - statsReceiver: StatsReceiver) - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with Logging { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - ExclusiveConversationAuthorIdFeature, - InReplyToTweetIdFeature, - IsHydratedFeature, - IsNsfw, - IsNsfwFeature, - IsRetweetFeature, - QuotedTweetDroppedFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - TweetTextFeature, - TweetLanguageFeature, - VisibilityReason - ) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(IsHydratedFeature, false) - .add(IsNsfw, None) - .add(IsNsfwFeature, false) - .add(QuotedTweetDroppedFeature, false) - .add(TweetTextFeature, None) - .add(VisibilityReason, None) - .build() - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - val safetyLevel = query.product match { - case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest - case ForYouProduct => - val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true) - if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations - case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome - case ListTweetsProduct => rtf.SafetyLevel.TimelineLists - case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed - case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown") - } - - val tweetFieldsOptions = tp.GetTweetFieldsOptions( - tweetIncludes = RequestFields.TweetTPHydrationFields, - includeRetweetedTweet = true, - includeQuotedTweet = true, - visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, - safetyLevel = Some(safetyLevel), - forUserId = query.getOptionalUserId - ) - - val exclusiveAuthorIdOpt = - existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None) - - tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map { - case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) => - val coreData = found.tweet.coreData - val isNsfwAdmin = coreData.exists(_.nsfwAdmin) - val isNsfwUser = coreData.exists(_.nsfwUser) - - val quotedTweetDropped = quoteOpt.exists { - case _: tp.TweetFieldsResultState.Filtered => true - case _: tp.TweetFieldsResultState.NotFound => true - case _ => false - } - val quotedTweetIsNsfw = quoteOpt.exists { - case quoteTweet: tp.TweetFieldsResultState.Found => - quoteTweet.found.tweet.coreData.exists(data => data.nsfwAdmin || data.nsfwUser) - case _ => false - } - - val sourceTweetIsNsfw = - found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)) - - val tweetText = coreData.map(_.text) - val tweetLanguage = found.tweet.language.map(_.language) - - val tweetAuthorId = coreData.map(_.userId) - val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)) - val retweetedTweetId = found.retweetedTweet.map(_.id) - val quotedTweetId = quoteOpt.flatMap { - case quoteTweet: tp.TweetFieldsResultState.Found => - Some(quoteTweet.found.tweet.id) - case _ => None - } - - val retweetedTweetUserId = found.retweetedTweet.flatMap(_.coreData).map(_.userId) - val quotedTweetUserId = quoteOpt.flatMap { - case quoteTweet: tp.TweetFieldsResultState.Found => - quoteTweet.found.tweet.coreData.map(_.userId) - case _ => None - } - - val isNsfw = isNsfwAdmin || isNsfwUser || sourceTweetIsNsfw || quotedTweetIsNsfw - - FeatureMapBuilder() - .add(AuthorIdFeature, tweetAuthorId) - .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) - .add(InReplyToTweetIdFeature, inReplyToTweetId) - .add(IsHydratedFeature, true) - .add(IsNsfw, Some(isNsfw)) - .add(IsNsfwFeature, isNsfw) - .add(IsRetweetFeature, retweetedTweetId.isDefined) - .add(QuotedTweetDroppedFeature, quotedTweetDropped) - .add(QuotedTweetIdFeature, quotedTweetId) - .add(QuotedUserIdFeature, quotedTweetUserId) - .add(SourceTweetIdFeature, retweetedTweetId) - .add(SourceUserIdFeature, retweetedTweetUserId) - .add(TweetLanguageFeature, tweetLanguage) - .add(TweetTextFeature, tweetText) - .add(VisibilityReason, found.suppressReason) - .build() - - // If no tweet result found, return default and pre-existing features - case _ => - DefaultFeatureMap ++ FeatureMapBuilder() - .add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) - .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) - .add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) - .add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) - .add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) - .add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) - .add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) - .add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None)) - .add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None)) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel deleted file mode 100644 index 59284224b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "stitch/stitch-socialgraph", - "stitch/stitch-tweetypie", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "util/util-slf4j-api/src/main/scala", - ], - exports = [ - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.docx new file mode 100644 index 000000000..f0316e2fa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.docx new file mode 100644 index 000000000..f73f0581d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.scala deleted file mode 100644 index 94943fc4a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.FSBoundedParam - -case class DropMaxCandidatesFilter[Candidate <: UniversalNoun[Any]]( - maxCandidatesParam: FSBoundedParam[Int]) - extends Filter[PipelineQuery, Candidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("DropMaxCandidates") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[FilterResult[Candidate]] = { - val maxCandidates = query.params(maxCandidatesParam) - val (kept, removed) = candidates.map(_.candidate).splitAt(maxCandidates) - - Stitch.value(FilterResult(kept, removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.docx new file mode 100644 index 000000000..1aaaba40f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala deleted file mode 100644 index 582583e7f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.common.thriftscala.FeedbackEntity -import com.twitter.timelineservice.model.FeedbackEntry -import com.twitter.timelineservice.{thriftscala => tls} - -object FeedbackFatigueFilter - extends Filter[PipelineQuery, TweetCandidate] - with Filter.Conditionally[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("FeedbackFatigue") - - override def onlyIf( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Boolean = - query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) - - private val DurationForFiltering = 14.days - - override def apply( - query: pipeline.PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val feedbackEntriesByEngagementType = - query.features - .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) - .filter { entry => - val timeSinceFeedback = query.queryTime.minus(entry.timestamp) - timeSinceFeedback < DurationForFiltering && - entry.feedbackType == tls.FeedbackType.SeeFewer - }.groupBy(_.engagementType) - - val authorsToFilter = - getUserIds( - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Tweet, Seq.empty)) - val likersToFilter = - getUserIds( - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Like, Seq.empty)) - val followersToFilter = - getUserIds( - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Follow, Seq.empty)) - val retweetersToFilter = - getUserIds( - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) - - val (removed, kept) = candidates.partition { candidate => - val originalAuthorId = CandidatesUtil.getOriginalAuthorId(candidate.features) - val authorId = candidate.features.getOrElse(AuthorIdFeature, None) - - val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - val eligibleLikers = likers.filterNot(likersToFilter.contains) - - val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) - val eligibleFollowers = followers.filterNot(followersToFilter.contains) - - originalAuthorId.exists(authorsToFilter.contains) || - (likers.nonEmpty && eligibleLikers.isEmpty) || - (followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) || - (candidate.features.getOrElse(IsRetweetFeature, false) && - authorId.exists(retweetersToFilter.contains)) - } - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } - - private def getUserIds( - feedbackEntries: Seq[FeedbackEntry], - ): Set[Long] = - feedbackEntries.collect { - case FeedbackEntry(_, _, FeedbackEntity.UserId(userId), _, _) => userId - }.toSet -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.docx new file mode 100644 index 000000000..3d0bde5cc Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.scala deleted file mode 100644 index 0d3f1ca1f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Exclude conversation modules where Tweets have been dropped by other filters - * - * Largest conversation modules have 3 Tweets, so if all 3 are present, module is valid. - * For 2 Tweet modules, check if the head is the root (not a reply) and the last item - * is actually replying to the root directly with no missing intermediate tweets - */ -object InvalidConversationModuleFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("InvalidConversationModule") - - val ValidThreeTweetModuleSize = 3 - val ValidTwoTweetModuleSize = 2 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val allowedTweetIds = candidates - .groupBy(_.features.getOrElse(ConversationModuleFocalTweetIdFeature, None)) - .map { case (id, candidates) => (id, candidates.sortBy(_.candidate.id)) } - .filter { - case (Some(_), conversation) if conversation.size == ValidThreeTweetModuleSize => true - case (Some(focalId), conversation) if conversation.size == ValidTwoTweetModuleSize => - conversation.head.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty && - conversation.last.candidate.id == focalId && - conversation.last.features - .getOrElse(InReplyToTweetIdFeature, None) - .contains(conversation.head.candidate.id) - case (None, _) => true - case _ => false - }.values.flatten.toSeq.map(_.candidate.id).toSet - - val (kept, removed) = - candidates.map(_.candidate).partition(candidate => allowedTweetIds.contains(candidate.id)) - Stitch.value(FilterResult(kept = kept, removed = removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.docx new file mode 100644 index 000000000..0c576a946 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala deleted file mode 100644 index 285eec49f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import com.twitter.util.logging.Logging - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author - * - * If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for - * subscription tweets, so we explicitly filter those tweets out. - */ -@Singleton -case class InvalidSubscriptionTweetFilter @Inject() ( - socialGraphClient: SocialGraph, - statsReceiver: StatsReceiver) - extends Filter[PipelineQuery, TweetCandidate] - with Logging { - - override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet") - - private val scopedStatsReceiver = statsReceiver.scope(identifier.toString) - private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet") - private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = Stitch - .traverse(candidates) { candidate => - val exclusiveAuthorId = - candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None) - - if (exclusiveAuthorId.isDefined) { - val request = sg.ExistsRequest( - source = query.getRequiredUserId, - target = exclusiveAuthorId.get, - relationships = - Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)), - ) - socialGraphClient.exists(request).map(_.exists).map { valid => - if (!valid) invalidCounter.incr() else validCounter.incr() - valid - } - } else Stitch.value(true) - }.map { validResults => - val (kept, removed) = candidates - .map(_.candidate) - .zip(validResults) - .partition { case (candidate, valid) => valid } - - val keptCandidates = kept.map { case (candidate, _) => candidate } - val removedCandidates = removed.map { case (candidate, _) => candidate } - - FilterResult(kept = keptCandidates, removed = removedCandidates) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.docx new file mode 100644 index 000000000..83df6e1fb Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.scala deleted file mode 100644 index b263a87fc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -trait FilterPredicate[-Query <: PipelineQuery] { - def apply(query: Query): Boolean -} - -/** - * A [[Filter]] with [[Conditionally]] based on a [[FilterPredicate]] - * - * @param predicate the predicate to turn this filter on and off - * @param filter the underlying filter to run when `predicate` is true - * @tparam Query The domain model for the query or request - * @tparam Candidate The type of the candidates - */ -case class PredicateGatedFilter[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( - predicate: FilterPredicate[Query], - filter: Filter[Query, Candidate]) - extends Filter[Query, Candidate] - with Filter.Conditionally[Query, Candidate] { - - override val identifier: FilterIdentifier = FilterIdentifier( - PredicateGatedFilter.IdentifierPrefix + filter.identifier.name) - - override val alerts: Seq[Alert] = filter.alerts - - override def onlyIf(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): Boolean = - Conditionally.and(Filter.Input(query, candidates), filter, predicate(query)) - - override def apply( - query: Query, - candidates: Seq[CandidateWithFeatures[Candidate]] - ): Stitch[FilterResult[Candidate]] = filter.apply(query, candidates) -} - -object PredicateGatedFilter { - val IdentifierPrefix = "PredicateGated" -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.docx new file mode 100644 index 000000000..154c5d081 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala deleted file mode 100644 index 047233a41..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.home_mixer.util.TweetImpressionsHelper -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Filter out users' previously seen tweets from 2 sources: - * 1. Heron Topology Impression Store in Memcache; - * 2. Manhattan Impression Store; - */ -object PreviouslySeenTweetsFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslySeenTweets") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val seenTweetIds = - query.features.map(TweetImpressionsHelper.tweetImpressions).getOrElse(Set.empty) - - val (removed, kept) = candidates.partition { candidate => - val tweetIdAndSourceId = CandidatesUtil.getTweetIdAndSourceId(candidate) - tweetIdAndSourceId.exists(seenTweetIds.contains) - } - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.docx new file mode 100644 index 000000000..1f7039ae9 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.scala deleted file mode 100644 index 42b0ad51a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent -import com.twitter.home_mixer.model.HomeFeatures.IsAncestorCandidateFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils -import com.twitter.timelines.util.client_info.ClientPlatform - -object PreviouslyServedAncestorsFilter - extends Filter[PipelineQuery, TweetCandidate] - with TimelinePersistenceUtils { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedAncestors") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val clientPlatform = ClientPlatform.fromQueryOptions( - clientAppId = query.clientContext.appId, - userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString)) - val entries = - query.features.map(_.getOrElse(PersistenceEntriesFeature, Seq.empty)).toSeq.flatten - val tweetIds = applicableResponses(clientPlatform, entries) - .flatMap(_.entries.flatMap(_.tweetIds(includeSourceTweets = true))).toSet - val ancestorIds = - candidates - .filter(_.features.getOrElse(IsAncestorCandidateFeature, false)).map(_.candidate.id).toSet - - val (removed, kept) = - candidates - .map(_.candidate).partition(candidate => - tweetIds.contains(candidate.id) && ancestorIds.contains(candidate.id)) - - Stitch.value(FilterResult(kept = kept, removed = removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.docx new file mode 100644 index 000000000..983d8c210 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala deleted file mode 100644 index 0ff820479..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val servedTweetPreviewIds = - query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet - - val (removed, kept) = candidates.partition { candidate => - servedTweetPreviewIds.contains(candidate.candidate.id) - } - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.docx new file mode 100644 index 000000000..2326f61df Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala deleted file mode 100644 index 56b6a1c0b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object PreviouslyServedTweetsFilter - extends Filter[PipelineQuery, TweetCandidate] - with Filter.Conditionally[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweets") - - override def onlyIf( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Boolean = { - query.features.exists(_.getOrElse(GetOlderFeature, false)) - } - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val servedTweetIds = - query.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)).toSeq.flatten.toSet - - val (removed, kept) = candidates.partition { candidate => - val tweetIdAndSourceId = CandidatesUtil.getTweetIdAndSourceId(candidate) - tweetIdAndSourceId.exists(servedTweetIds.contains) - } - - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.docx new file mode 100644 index 000000000..e4a65830e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.scala deleted file mode 100644 index c25d9ec2b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object RejectTweetFromViewerFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("RejectTweetFromViewer") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val (removed, kept) = candidates.partition(candidate => - CandidatesUtil.isAuthoredByViewer(query, candidate.features)) - Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.docx new file mode 100644 index 000000000..8b7bbb874 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala deleted file mode 100644 index a1b99df66..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] { - override val identifier: FilterIdentifier = FilterIdentifier("Reply") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val (kept, removed) = candidates - .partition { candidate => - candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty - } - - val filterResult = FilterResult( - kept = kept.map(_.candidate), - removed = removed.map(_.candidate) - ) - - Stitch.value(filterResult) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.docx new file mode 100644 index 000000000..4be35a775 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala deleted file mode 100644 index 1e1f7f03a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import scala.collection.mutable - -object RetweetDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("RetweetDeduplication") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - // If there are 2 retweets of the same native tweet, we will choose the first one - // The tweets are returned in descending score order, so we will choose the higher scored tweet - val dedupedTweetIdsSet = - candidates.partition(_.features.getOrElse(IsRetweetFeature, false)) match { - case (retweets, nativeTweets) => - val nativeTweetIds = nativeTweets.map(_.candidate.id) - val seenTweetIds = mutable.Set[Long]() ++ nativeTweetIds - val dedupedRetweets = retweets.filter { retweet => - val tweetIdAndSourceId = CandidatesUtil.getTweetIdAndSourceId(retweet) - val retweetIsUnique = tweetIdAndSourceId.forall(!seenTweetIds.contains(_)) - if (retweetIsUnique) { - seenTweetIds ++= tweetIdAndSourceId - } - retweetIsUnique - } - (nativeTweets ++ dedupedRetweets).map(_.candidate.id).toSet - } - - val (kept, removed) = - candidates - .map(_.candidate).partition(candidate => dedupedTweetIdsSet.contains(candidate.id)) - Stitch.value(FilterResult(kept = kept, removed = removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.docx new file mode 100644 index 000000000..6d61c0712 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala deleted file mode 100644 index 0ffc4b00c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.functional_component.filter - -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] { - override val identifier: FilterIdentifier = FilterIdentifier("Retweet") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val (kept, removed) = candidates - .partition { candidate => - !candidate.features.getOrElse(IsRetweetFeature, false) - } - - val filterResult = FilterResult( - kept = kept.map(_.candidate), - removed = removed.map(_.candidate) - ) - - Stitch.value(filterResult) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel deleted file mode 100644 index 6be06dee9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "stitch/stitch-socialgraph", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.docx new file mode 100644 index 000000000..14995550e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.docx new file mode 100644 index 000000000..f6706877f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala deleted file mode 100644 index b35a6cda5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.conversions.DurationOps._ -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.manhattan.DismissInfo -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import com.twitter.util.Duration - -object DismissFatigueGate { - // how long a dismiss action from user needs to be respected - val DefaultBaseDismissDuration = 7.days - val MaximumDismissalCountMultiplier = 4 -} - -case class DismissFatigueGate( - suggestType: SuggestType, - dismissInfoFeature: Feature[PipelineQuery, Map[SuggestType, Option[DismissInfo]]], - baseDismissDuration: Duration = DismissFatigueGate.DefaultBaseDismissDuration, -) extends Gate[PipelineQuery] { - - override val identifier: GateIdentifier = GateIdentifier("DismissFatigue") - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { - val dismissInfoMap = query.features.map( - _.getOrElse(dismissInfoFeature, Map.empty[SuggestType, Option[DismissInfo]])) - - val isVisible = dismissInfoMap - .flatMap(_.get(suggestType)) - .flatMap(_.map { info => - val currentDismissalDuration = query.queryTime.since(info.lastDismissed) - val targetDismissalDuration = dismissDurationForCount(info.count, baseDismissDuration) - - currentDismissalDuration > targetDismissalDuration - }).getOrElse(true) - Stitch.value(isVisible) - } - - private def dismissDurationForCount( - dismissCount: Int, - dismissDuration: Duration - ): Duration = - // limit to maximum dismissal duration - dismissDuration * Math.min(dismissCount, DismissFatigueGate.MaximumDismissalCountMultiplier) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.docx new file mode 100644 index 000000000..d8e394ecc Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.scala deleted file mode 100644 index a86159036..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.gizmoduck.{thriftscala => t} -import com.twitter.home_mixer.model.HomeFeatures.UserTypeFeature -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * A Soft User is a user who is in the gradual onboarding state. This gate can be - * used to turn off certain functionality like ads for these users. - */ -object ExcludeSoftUserGate extends Gate[PipelineQuery] { - - override val identifier: GateIdentifier = GateIdentifier("ExcludeSoftUser") - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { - val softUser = query.features - .exists(_.getOrElse(UserTypeFeature, None).exists(_ == t.UserType.Soft)) - Stitch.value(!softUser) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.docx new file mode 100644 index 000000000..ff1ec2ef6 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.scala deleted file mode 100644 index 0ffe6793e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Gate that fetches the request context from the device context and - * continues if the request context matches *any* of the specified ones. - */ -case class RequestContextGate(requestContexts: Seq[RequestContext.Value]) - extends Gate[PipelineQuery with HasDeviceContext] { - - override val identifier: GateIdentifier = GateIdentifier("RequestContext") - - override def shouldContinue(query: PipelineQuery with HasDeviceContext): Stitch[Boolean] = - Stitch.value( - requestContexts.exists(query.deviceContext.flatMap(_.requestContextValue).contains)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.docx new file mode 100644 index 000000000..32f55bf52 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.scala deleted file mode 100644 index c52f05f75..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Gate that fetches the request context from the device context and - * continues if the request context does not match any of the specified ones. - * - * If no input request context is specified, the gate continues - */ -case class RequestContextNotGate(requestContexts: Seq[RequestContext.Value]) - extends Gate[PipelineQuery with HasDeviceContext] { - - override val identifier: GateIdentifier = GateIdentifier("RequestContextNot") - - override def shouldContinue(query: PipelineQuery with HasDeviceContext): Stitch[Boolean] = - Stitch.value( - !requestContexts.exists(query.deviceContext.flatMap(_.requestContextValue).contains)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.docx new file mode 100644 index 000000000..75bb42615 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.scala deleted file mode 100644 index 53b681236..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object SupportedLanguagesGate extends Gate[PipelineQuery] { - - override val identifier: GateIdentifier = GateIdentifier("SupportedLanguages") - - // Production languages which have high translation coverage for strings used in Home Timeline. - private val supportedLanguages: Set[String] = Set( - "ar", // Arabic - "ar-x-fm", // Arabic (Female) - "bg", // Bulgarian - "bn", // Bengali - "ca", // Catalan - "cs", // Czech - "da", // Danish - "de", // German - "el", // Greek - "en", // English - "en-gb", // British English - "en-ss", // English Screen shot - "en-xx", // English Pseudo - "es", // Spanish - "eu", // Basque - "fa", // Farsi (Persian) - "fi", // Finnish - "fil", // Filipino - "fr", // French - "ga", // Irish - "gl", // Galician - "gu", // Gujarati - "he", // Hebrew - "hi", // Hindi - "hr", // Croatian - "hu", // Hungarian - "id", // Indonesian - "it", // Italian - "ja", // Japanese - "kn", // Kannada - "ko", // Korean - "mr", // Marathi - "msa", // Malay - "nl", // Dutch - "no", // Norwegian - "pl", // Polish - "pt", // Portuguese - "ro", // Romanian - "ru", // Russian - "sk", // Slovak - "sr", // Serbian - "sv", // Swedish - "ta", // Tamil - "th", // Thai - "tr", // Turkish - "uk", // Ukrainian - "ur", // Urdu - "vi", // Vietnamese - "zh-cn", // Simplified Chinese - "zh-tw" // Traditional Chinese - ) - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = - Stitch.value(query.getLanguageCode.forall(supportedLanguages.contains)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.docx new file mode 100644 index 000000000..a65559ea9 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala deleted file mode 100644 index 31fff2306..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.functional_component.gate - -import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils -import com.twitter.timelines.configapi.Param -import com.twitter.timelines.util.client_info.ClientPlatform -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.util.Duration -import com.twitter.util.Time - -/** - * Gate used to reduce the frequency of injections. Note that the actual interval between injections may be - * less than the specified minInjectionIntervalParam if data is unavailable or missing. For example, being deleted by - * the persistence store via a TTL or similar mechanism. - * - * @param minInjectionIntervalParam the desired minimum interval between injections - * @param persistenceEntriesFeature the feature for retrieving persisted timeline responses - */ -case class TimelinesPersistenceStoreLastInjectionGate( - minInjectionIntervalParam: Param[Duration], - persistenceEntriesFeature: Feature[PipelineQuery, Seq[TimelineResponseV3]], - entityIdType: EntityIdType.Value) - extends Gate[PipelineQuery] - with TimelinePersistenceUtils { - - override val identifier: GateIdentifier = GateIdentifier("TimelinesPersistenceStoreLastInjection") - - override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = - Stitch( - query.queryTime.since(getLastInjectionTime(query)) > query.params(minInjectionIntervalParam)) - - private def getLastInjectionTime(query: PipelineQuery) = query.features - .flatMap { featureMap => - val timelineResponses = featureMap.getOrElse(persistenceEntriesFeature, Seq.empty) - val clientPlatform = ClientPlatform.fromQueryOptions( - clientAppId = query.clientContext.appId, - userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString) - ) - val sortedResponses = responseByClient(clientPlatform, timelineResponses) - val latestResponseWithEntityIdTypeEntry = - sortedResponses.find(_.entries.exists(_.entityIdType == entityIdType)) - - latestResponseWithEntityIdTypeEntry.map(_.servedTime) - }.getOrElse(Time.Bottom) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.bazel deleted file mode 100644 index 67763235b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "common-internal/analytics/twitter-client-user-agent-parser/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelineservice/common:model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.docx new file mode 100644 index 000000000..d74ab95c6 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.docx new file mode 100644 index 000000000..27addb8fa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala deleted file mode 100644 index 8753f2f28..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.home_mixer.functional_component.query_transformer - -import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelinemixer.clients.persistence.EntryWithItemIds -import com.twitter.timelines.persistence.thriftscala.RequestType -import com.twitter.timelines.util.client_info.ClientPlatform -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.util.Time - -object EditedTweetsCandidatePipelineQueryTransformer - extends CandidatePipelineQueryTransformer[PipelineQuery, Seq[Long]] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets") - - // The time window for which a tweet remains editable after creation. - private val EditTimeWindow = 60.minutes - - override def transform(query: PipelineQuery): Seq[Long] = { - val applicableCandidates = getApplicableCandidates(query) - - if (applicableCandidates.nonEmpty) { - // Include the response corresponding with the Previous Timeline Load (PTL). - // Any tweets in it could have become stale since being served. - val previousTimelineLoadTime = applicableCandidates.head.servedTime - - // The time window for editing a tweet is 60 minutes, - // so we ignore responses older than (PTL Time - 60 mins). - val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates - .takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow) - - // Exclude the tweet IDs for which ReplaceEntry instructions have already been sent. - val (tweetsAlreadyReplaced, tweetsToCheck) = inWindowCandidates - .partition(_.entryWithItemIds.itemIds.exists(_.head.entryIdToReplace.nonEmpty)) - - val tweetIdFromEntry: PartialFunction[PersistenceStoreEntry, Long] = { - case entry if entry.tweetId.nonEmpty => entry.tweetId.get - } - - val tweetIdsAlreadyReplaced: Set[Long] = tweetsAlreadyReplaced.collect(tweetIdFromEntry).toSet - val tweetIdsToCheck: Seq[Long] = tweetsToCheck.collect(tweetIdFromEntry) - - tweetIdsToCheck.filterNot(tweetIdsAlreadyReplaced.contains).distinct - } else Seq.empty - } - - // The candidates here come from the Timelines Persistence Store, via a query feature - private def getApplicableCandidates(query: PipelineQuery): Seq[PersistenceStoreEntry] = { - val userAgent = UserAgent.fromString(query.clientContext.userAgent.getOrElse("")) - val clientPlatform = ClientPlatform.fromQueryOptions(query.clientContext.appId, userAgent) - - val sortedResponses = query.features - .getOrElse(FeatureMap.empty) - .getOrElse(PersistenceEntriesFeature, Seq.empty) - .filter(_.clientPlatform == clientPlatform) - .sortBy(-_.servedTime.inMilliseconds) - - val recentResponses = sortedResponses.indexWhere(_.requestType == RequestType.Initial) match { - case -1 => sortedResponses - case lastGetInitialIndex => sortedResponses.take(lastGetInitialIndex + 1) - } - - recentResponses.flatMap { r => - r.entries.collect { - case entry if entry.entityIdType == EntityIdType.Tweet => - PersistenceStoreEntry(entry, r.servedTime, r.clientPlatform, r.requestType) - } - }.distinct - } -} - -case class PersistenceStoreEntry( - entryWithItemIds: EntryWithItemIds, - servedTime: Time, - clientPlatform: ClientPlatform, - requestType: RequestType) { - - // Timelines Persistence Store currently includes 1 tweet ID per entryWithItemIds for tweets - val tweetId: Option[Long] = entryWithItemIds.itemIds.flatMap(_.head.tweetId) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel deleted file mode 100644 index 99605204c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/thrift/com/twitter/timelines/common:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "timelineservice/common:model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.docx new file mode 100644 index 000000000..1d695d280 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.docx new file mode 100644 index 000000000..9027d7904 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala deleted file mode 100644 index ceb71139e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.home_mixer.functional_component.scorer - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.common.{thriftscala => tl} -import com.twitter.timelineservice.model.FeedbackEntry -import com.twitter.timelineservice.{thriftscala => tls} -import com.twitter.util.Time -import scala.collection.mutable - -object FeedbackFatigueScorer - extends Scorer[PipelineQuery, TweetCandidate] - with Conditionally[PipelineQuery] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("FeedbackFatigue") - - override def features: Set[Feature[_, _]] = Set(ScoreFeature) - - override def onlyIf(query: PipelineQuery): Boolean = - query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) - - val DurationForFiltering = 14.days - val DurationForDiscounting = 140.days - private val ScoreMultiplierLowerBound = 0.2 - private val ScoreMultiplierUpperBound = 1.0 - private val ScoreMultiplierIncrementsCount = 4 - private val ScoreMultiplierIncrement = - (ScoreMultiplierUpperBound - ScoreMultiplierLowerBound) / ScoreMultiplierIncrementsCount - private val ScoreMultiplierIncrementDurationInDays = - DurationForDiscounting.inDays / ScoreMultiplierIncrementsCount.toDouble - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val feedbackEntriesByEngagementType = - query.features - .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) - .filter { entry => - val timeSinceFeedback = query.queryTime.minus(entry.timestamp) - timeSinceFeedback < DurationForFiltering + DurationForDiscounting && - entry.feedbackType == tls.FeedbackType.SeeFewer - }.groupBy(_.engagementType) - - val authorsToDiscount = - getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Tweet, Seq.empty)) - val likersToDiscount = - getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Like, Seq.empty)) - val followersToDiscount = - getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Follow, Seq.empty)) - val retweetersToDiscount = - getUserDiscounts( - query.queryTime, - feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) - - val featureMaps = candidates.map { candidate => - val multiplier = getScoreMultiplier( - candidate, - authorsToDiscount, - likersToDiscount, - followersToDiscount, - retweetersToDiscount - ) - val score = candidate.features.getOrElse(ScoreFeature, None) - FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build() - } - - Stitch.value(featureMaps) - } - - def getScoreMultiplier( - candidate: CandidateWithFeatures[TweetCandidate], - authorsToDiscount: Map[Long, Double], - likersToDiscount: Map[Long, Double], - followersToDiscount: Map[Long, Double], - retweetersToDiscount: Map[Long, Double], - ): Double = { - val originalAuthorId = - CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L) - val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0) - - val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - val likerMultipliers = likers.flatMap(likersToDiscount.get) - val likerMultiplier = - if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size) - likerMultipliers.max - else 1.0 - - val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) - val followerMultipliers = followers.flatMap(followersToDiscount.get) - val followerMultiplier = - if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size && - likers.isEmpty) - followerMultipliers.max - else 1.0 - - val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) - val retweeterMultiplier = - if (candidate.features.getOrElse(IsRetweetFeature, false)) - retweetersToDiscount.getOrElse(authorId, 1.0) - else 1.0 - - originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier - } - - def getUserDiscounts( - queryTime: Time, - feedbackEntries: Seq[FeedbackEntry], - ): Map[Long, Double] = { - val userDiscounts = mutable.Map[Long, Double]() - feedbackEntries - .collect { - case FeedbackEntry(_, _, tl.FeedbackEntity.UserId(userId), timestamp, _) => - val timeSinceFeedback = queryTime.minus(timestamp) - val timeSinceDiscounting = timeSinceFeedback - DurationForFiltering - val multiplier = ((timeSinceDiscounting.inDays / ScoreMultiplierIncrementDurationInDays) - * ScoreMultiplierIncrement + ScoreMultiplierLowerBound) - userDiscounts.update(userId, multiplier) - } - userDiscounts.toMap - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.docx new file mode 100644 index 000000000..be76299e5 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala deleted file mode 100644 index 4b1cec4a5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.functional_component.scorer - -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -/** - * Scales scores of each out-of-network tweet by the specified scale factor - */ -object OONTweetScalingScorer extends Scorer[PipelineQuery, TweetCandidate] { - - override val identifier: ScorerIdentifier = ScorerIdentifier("OONTweetScaling") - - override val features: Set[Feature[_, _]] = Set(ScoreFeature) - - private val ScaleFactor = 0.75 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - Stitch.value { - candidates.map { candidate => - val score = candidate.features.getOrElse(ScoreFeature, None) - val updatedScore = if (selector(candidate)) score.map(_ * ScaleFactor) else score - FeatureMapBuilder().add(ScoreFeature, updatedScore).build() - } - } - } - - /** - * We should only be applying this multiplier to Out-Of-Network tweets. - * In-Network Retweets of Out-Of-Network tweets should not have this multiplier applied - */ - private def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { - !candidate.features.getOrElse(InNetworkFeature, false) && - !candidate.features.getOrElse(IsRetweetFeature, false) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel deleted file mode 100644 index 3689de6ea..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/scala/com/twitter/suggests/controller_data", - "stringcenter/client", - "stringcenter/client/src/main/java", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.docx new file mode 100644 index 000000000..d5ba9f1de Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.docx new file mode 100644 index 000000000..5199a35ec Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.scala deleted file mode 100644 index 648174273..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.home_mixer.functional_component.selector - -import com.twitter.home_mixer.functional_component.selector.DebunchCandidates.TrailingTweetsMinSize -import com.twitter.home_mixer.functional_component.selector.DebunchCandidates.TrailingTweetsPortionToKeep -import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -trait MustDebunch { - def apply(candidate: CandidateWithDetails): Boolean -} - -object DebunchCandidates { - val TrailingTweetsMinSize = 5 - val TrailingTweetsPortionToKeep = 0.1 -} - -/** - * This selector rearranges the candidates to only allow bunches of size [[maxBunchSize]], where a - * bunch is a consecutive sequence of candidates that meet [[mustDebunch]]. - */ -case class DebunchCandidates( - override val pipelineScope: CandidateScope, - mustDebunch: MustDebunch, - maxBunchSize: Int) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val PartitionedCandidates(selectedCandidates, otherCandidates) = - pipelineScope.partition(remainingCandidates) - val mutableCandidates = collection.mutable.ListBuffer(selectedCandidates: _*) - - var candidatePointer = 0 - var nonDebunchPointer = 0 - var bunchSize = 0 - var finalNonDebunch = -1 - - while (candidatePointer < mutableCandidates.size) { - if (mustDebunch(mutableCandidates(candidatePointer))) bunchSize += 1 - else { - bunchSize = 0 - finalNonDebunch = candidatePointer - } - - if (bunchSize > maxBunchSize) { - nonDebunchPointer = Math.max(candidatePointer, nonDebunchPointer) - while (nonDebunchPointer < mutableCandidates.size && - mustDebunch(mutableCandidates(nonDebunchPointer))) { - nonDebunchPointer += 1 - } - if (nonDebunchPointer == mutableCandidates.size) - candidatePointer = mutableCandidates.size - else { - val nextNonDebunch = mutableCandidates(nonDebunchPointer) - mutableCandidates.remove(nonDebunchPointer) - mutableCandidates.insert(candidatePointer, nextNonDebunch) - bunchSize = 0 - finalNonDebunch = candidatePointer - } - } - - candidatePointer += 1 - } - - val debunchedCandidates = if (query.features.exists(_.getOrElse(GetNewerFeature, false))) { - val trailingTweetsSize = mutableCandidates.size - finalNonDebunch - 1 - val keepCandidates = finalNonDebunch + 1 + - Math.max(TrailingTweetsMinSize, TrailingTweetsPortionToKeep * trailingTweetsSize).toInt - mutableCandidates.toList.take(keepCandidates) - } else mutableCandidates.toList - - val updatedCandidates = otherCandidates ++ debunchedCandidates - SelectorResult(remainingCandidates = updatedCandidates, result = result) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.docx new file mode 100644 index 000000000..83f7483a0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.scala deleted file mode 100644 index bbde0ed0d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.home_mixer.functional_component.selector - -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * This selector updates the id of the conversation modules to be the head of the module's id. - */ -case class UpdateConversationModuleId( - override val pipelineScope: CandidateScope) - extends Selector[PipelineQuery] { - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val PartitionedCandidates(selectedCandidates, otherCandidates) = - pipelineScope.partition(remainingCandidates) - - val updatedCandidates = selectedCandidates.map { - case module @ ModuleCandidateWithDetails(candidates, presentationOpt, _) => - val updatedPresentation = presentationOpt.map { - case urtModule @ UrtModulePresentation(timelineModule) => - urtModule.copy(timelineModule = - timelineModule.copy(id = candidates.head.candidateIdLong)) - } - module.copy(presentation = updatedPresentation) - case candidate => candidate - } - - SelectorResult(remainingCandidates = updatedCandidates ++ otherCandidates, result = result) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.docx new file mode 100644 index 000000000..56dd90c1b Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala deleted file mode 100644 index c7944b506..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala +++ /dev/null @@ -1,137 +0,0 @@ -package com.twitter.home_mixer.functional_component.selector - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature -import com.twitter.home_mixer.model.HomeFeatures.HasRandomTweetFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetAboveFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature -import com.twitter.home_mixer.model.HomeFeatures.PositionFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedInConversationModuleFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedSizeFeature -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Builds serialized tweet type metrics controller data and updates Client Event Details - * and Candidate Presentations with this info. - * - * Currently only updates presentation of Item Candidates. This needs to be updated - * when modules are added. - * - * This is implemented as a Selector instead of a Decorator in the Candidate Pipeline - * because we need to add controller data that looks at the final timeline as a whole - * (e.g. served size, final candidate positions). - * - * @param candidatePipelines - only candidates from the specified pipeline will be updated - */ -case class UpdateHomeClientEventDetails(candidatePipelines: Set[CandidatePipelineIdentifier]) - extends Selector[PipelineQuery] { - - override val pipelineScope: CandidateScope = SpecificPipelines(candidatePipelines) - - private val detailsBuilder = HomeClientEventDetailsBuilder() - - override def apply( - query: PipelineQuery, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val selectedCandidates = result.filter(pipelineScope.contains) - - val randomTweetsByPosition = result - .map(_.features.getOrElse(IsRandomTweetFeature, false)) - .zipWithIndex.map(_.swap).toMap - - val resultFeatures = FeatureMapBuilder() - .add(ServedSizeFeature, Some(selectedCandidates.size)) - .add(HasRandomTweetFeature, randomTweetsByPosition.valuesIterator.contains(true)) - .build() - - val updatedResult = result.zipWithIndex.map { - case (item @ ItemCandidateWithDetails(candidate, _, _), position) - if pipelineScope.contains(item) => - val resultCandidateFeatures = FeatureMapBuilder() - .add(PositionFeature, Some(position)) - .add(IsRandomTweetAboveFeature, randomTweetsByPosition.getOrElse(position - 1, false)) - .build() - - updateItemPresentation(query, item, resultFeatures, resultCandidateFeatures) - - case (module @ ModuleCandidateWithDetails(candidates, presentation, features), position) - if pipelineScope.contains(module) => - val resultCandidateFeatures = FeatureMapBuilder() - .add(PositionFeature, Some(position)) - .add(IsRandomTweetAboveFeature, randomTweetsByPosition.getOrElse(position - 1, false)) - .add(ServedInConversationModuleFeature, true) - .add(ConversationModule2DisplayedTweetsFeature, module.candidates.size == 2) - .add( - ConversationModuleHasGapFeature, - module.candidates.last.features.getOrElse(AncestorsFeature, Seq.empty).size > 2) - .build() - - val updatedItemCandidates = - candidates.map(updateItemPresentation(query, _, resultFeatures, resultCandidateFeatures)) - - val updatedCandidateFeatures = features ++ resultFeatures ++ resultCandidateFeatures - - val updatedPresentation = presentation.map { - case urtModule @ UrtModulePresentation(timelineModule) => - val clientEventDetails = - detailsBuilder( - query, - candidates.last.candidate, - query.features.get ++ updatedCandidateFeatures) - val updatedClientEventInfo = - timelineModule.clientEventInfo.map(_.copy(details = clientEventDetails)) - val updatedTimelineModule = - timelineModule.copy(clientEventInfo = updatedClientEventInfo) - urtModule.copy(timelineModule = updatedTimelineModule) - } - - module.copy( - candidates = updatedItemCandidates, - presentation = updatedPresentation, - features = updatedCandidateFeatures - ) - - case (any, position) => any - } - - SelectorResult(remainingCandidates = remainingCandidates, result = updatedResult) - } - - private def updateItemPresentation( - query: PipelineQuery, - item: ItemCandidateWithDetails, - resultCandidateFeatures: FeatureMap, - resultFeatures: FeatureMap, - ): ItemCandidateWithDetails = { - val updatedItemCandidateFeatures = item.features ++ resultFeatures ++ resultCandidateFeatures - - val updatedPresentation = item.presentation.map { - case urtItem @ UrtItemPresentation(timelineItem: TweetItem, _) => - val clientEventDetails = - detailsBuilder(query, item.candidate, query.features.get ++ updatedItemCandidateFeatures) - val updatedClientEventInfo = - timelineItem.clientEventInfo.map(_.copy(details = clientEventDetails)) - val updatedTimelineItem = timelineItem.copy(clientEventInfo = updatedClientEventInfo) - urtItem.copy(timelineItem = updatedTimelineItem) - case any => any - } - item.copy(presentation = updatedPresentation, features = updatedItemCandidateFeatures) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.docx new file mode 100644 index 000000000..8e3736531 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.scala deleted file mode 100644 index a805ae1a7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.scala +++ /dev/null @@ -1,80 +0,0 @@ -package com.twitter.home_mixer.functional_component.selector - -import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration.NumAvatars -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.param.HomeGlobalParams.EnableNewTweetsPillAvatarsParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.selector.SelectorResult -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlert -import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stringcenter.client.StringCenter -import com.twitter.stringcenter.client.core.ExternalString - -object UpdateNewTweetsPillDecoration { - val NumAvatars = 3 -} - -case class UpdateNewTweetsPillDecoration[Query <: PipelineQuery with HasDeviceContext]( - override val pipelineScope: CandidateScope, - stringCenter: StringCenter, - seeNewTweetsString: ExternalString, - tweetedString: ExternalString) - extends Selector[Query] { - - override def apply( - query: Query, - remainingCandidates: Seq[CandidateWithDetails], - result: Seq[CandidateWithDetails] - ): SelectorResult = { - val (alerts, otherCandidates) = - remainingCandidates.partition(candidate => - candidate.isCandidateType[ShowAlertCandidate]() && pipelineScope.contains(candidate)) - val updatedCandidates = alerts - .collectFirst { - case newTweetsPill: ItemCandidateWithDetails => - val userIds = CandidatesUtil - .getItemCandidatesWithOnlyModuleLast(result) - .filter(candidate => - candidate.isCandidateType[TweetCandidate]() && pipelineScope.contains(candidate)) - .filterNot(_.features.getOrElse(IsRetweetFeature, false)) - .flatMap(_.features.getOrElse(AuthorIdFeature, None)) - .filterNot(_ == query.getRequiredUserId) - .distinct - - val updatedPresentation = newTweetsPill.presentation.map { - case presentation: UrtItemPresentation => - presentation.timelineItem match { - case alert: ShowAlert => - val text = if (useAvatars(query, userIds)) tweetedString else seeNewTweetsString - val richText = RichText( - text = stringCenter.prepare(text), - entities = List.empty, - rtl = None, - alignment = None) - - val updatedAlert = - alert.copy(userIds = Some(userIds.take(NumAvatars)), richText = Some(richText)) - presentation.copy(timelineItem = updatedAlert) - } - } - otherCandidates :+ newTweetsPill.copy(presentation = updatedPresentation) - }.getOrElse(remainingCandidates) - - SelectorResult(remainingCandidates = updatedCandidates, result = result) - } - - private def useAvatars(query: Query, userIds: Seq[Long]): Boolean = { - val enableAvatars = query.params(EnableNewTweetsPillAvatarsParam) - enableAvatars && userIds.size >= NumAvatars - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel deleted file mode 100644 index f352ce63a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel +++ /dev/null @@ -1,47 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "eventbus/client/src/main/scala/com/twitter/eventbus/client", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", - "src/thrift/com/twitter/timelines/impression_store:thrift-scala", - "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", - "src/thrift/com/twitter/user_session_store:thrift-scala", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelines/ml:kafka", - "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka", - "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", - "timelines/src/main/scala/com/twitter/timelines/clientconfig", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - "timelineservice/common:model", - "user_session_store/src/main/scala/com/twitter/user_session_store", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.docx new file mode 100644 index 000000000..c21615c26 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.docx new file mode 100644 index 000000000..c25e3d1a4 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala deleted file mode 100644 index a5cf739d3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala +++ /dev/null @@ -1,185 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates -import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates -import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil - -private[side_effect] sealed trait ClientEventsBuilder { - private val FollowingSection = Some("latest") - private val ForYouSection = Some("home") - private val ListTweetsSection = Some("list") - private val SubscribedSection = Some("subscribed") - - protected def section(query: PipelineQuery): Option[String] = { - query.product match { - case FollowingProduct => FollowingSection - case ForYouProduct => ForYouSection - case ListTweetsProduct => ListTweetsSection - case SubscribedProduct => SubscribedSection - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - } - - protected def count( - candidates: Seq[CandidateWithDetails], - predicate: FeatureMap => Boolean = _ => true, - queryFeatures: FeatureMap = FeatureMap.empty - ): Option[Long] = Some(candidates.view.count(item => predicate(item.features ++ queryFeatures))) - - protected def sum( - candidates: Seq[CandidateWithDetails], - predicate: FeatureMap => Option[Int], - queryFeatures: FeatureMap = FeatureMap.empty - ): Option[Long] = - Some(candidates.view.flatMap(item => predicate(item.features ++ queryFeatures)).sum) -} - -private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder { - - private val ServedTweetsAction = Some("served_tweets") - private val ServedUsersAction = Some("served_users") - private val InjectedComponent = Some("injected") - private val PromotedComponent = Some("promoted") - private val WhoToFollowComponent = Some("who_to_follow") - private val WhoToSubscribeComponent = Some("who_to_subscribe") - private val WithVideoDurationComponent = Some("with_video_duration") - private val VideoDurationSumElement = Some("video_duration_sum") - private val NumVideosElement = Some("num_videos") - - def build( - query: PipelineQuery, - injectedTweets: Seq[ItemCandidateWithDetails], - promotedTweets: Seq[ItemCandidateWithDetails], - whoToFollowUsers: Seq[ItemCandidateWithDetails], - whoToSubscribeUsers: Seq[ItemCandidateWithDetails] - ): Seq[ClientEvent] = { - val baseEventNamespace = EventNamespace( - section = section(query), - action = ServedTweetsAction - ) - val overallServedEvents = Seq( - ClientEvent(baseEventNamespace, eventValue = count(injectedTweets ++ promotedTweets)), - ClientEvent( - baseEventNamespace.copy(component = InjectedComponent), - eventValue = count(injectedTweets)), - ClientEvent( - baseEventNamespace.copy(component = PromotedComponent), - eventValue = count(promotedTweets)), - ClientEvent( - baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction), - eventValue = count(whoToFollowUsers)), - ClientEvent( - baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction), - eventValue = count(whoToSubscribeUsers)), - ) - - val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map { - case (tweetType, predicate) => - ClientEvent( - baseEventNamespace.copy(component = InjectedComponent, element = Some(tweetType)), - eventValue = count(injectedTweets, predicate, query.features.getOrElse(FeatureMap.empty)) - ) - }.toSeq - - val suggestTypeServedEvents = injectedTweets - .flatMap(_.features.getOrElse(SuggestTypeFeature, None)) - .map { - InjectionScribeUtil.scribeComponent - } - .groupBy(identity).map { - case (suggestType, group) => - ClientEvent( - baseEventNamespace.copy(component = suggestType), - eventValue = Some(group.size.toLong)) - }.toSeq - - // Video duration events - val numVideosEvent = ClientEvent( - baseEventNamespace.copy(component = WithVideoDurationComponent, element = NumVideosElement), - eventValue = count(injectedTweets, _.getOrElse(VideoDurationMsFeature, None).nonEmpty) - ) - val videoDurationSumEvent = ClientEvent( - baseEventNamespace - .copy(component = WithVideoDurationComponent, element = VideoDurationSumElement), - eventValue = sum(injectedTweets, _.getOrElse(VideoDurationMsFeature, None)) - ) - val videoEvents = Seq(numVideosEvent, videoDurationSumEvent) - - overallServedEvents ++ tweetTypeServedEvents ++ suggestTypeServedEvents ++ videoEvents - } -} - -private[side_effect] object EmptyTimelineEventsBuilder extends ClientEventsBuilder { - private val EmptyAction = Some("empty") - private val AccountAgeLessThan30MinutesComponent = Some("account_age_less_than_30_minutes") - private val ServedNonPromotedTweetElement = Some("served_non_promoted_tweet") - - def build( - query: PipelineQuery, - injectedTweets: Seq[ItemCandidateWithDetails] - ): Seq[ClientEvent] = { - val baseEventNamespace = EventNamespace( - section = section(query), - action = EmptyAction - ) - - // Empty timeline events - val accountAgeLessThan30Minutes = query.features - .flatMap(_.getOrElse(AccountAgeFeature, None)) - .exists(_.untilNow < 30.minutes) - val isEmptyTimeline = count(injectedTweets).contains(0L) - val predicates = Seq( - None -> isEmptyTimeline, - AccountAgeLessThan30MinutesComponent -> (isEmptyTimeline && accountAgeLessThan30Minutes) - ) - for { - (component, predicate) <- predicates - if predicate - } yield ClientEvent( - baseEventNamespace.copy(component = component, element = ServedNonPromotedTweetElement)) - } -} - -private[side_effect] object QueryEventsBuilder extends ClientEventsBuilder { - - private val ServedSizePredicateMap: Map[String, Int => Boolean] = Map( - ("size_is_empty", _ <= 0), - ("size_at_most_5", _ <= 5), - ("size_at_most_10", _ <= 10), - ("size_at_most_35", _ <= 35) - ) - - def build( - query: PipelineQuery, - injectedTweets: Seq[ItemCandidateWithDetails] - ): Seq[ClientEvent] = { - val baseEventNamespace = EventNamespace( - section = section(query) - ) - val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) - val servedSizeQueryEvents = - for { - (queryPredicateName, queryPredicate) <- HomeQueryTypePredicates.PredicateMap - if queryPredicate(queryFeatureMap) - (servedSizePredicateName, servedSizePredicate) <- ServedSizePredicateMap - if servedSizePredicate(injectedTweets.size) - } yield ClientEvent( - baseEventNamespace - .copy(component = Some(servedSizePredicateName), action = Some(queryPredicateName))) - servedSizeQueryEvents.toSeq - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.docx new file mode 100644 index 000000000..d43ce1491 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala deleted file mode 100644 index a8feef707..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Side effect that logs served tweet metrics to Scribe as client events. - */ -case class HomeScribeClientEventSideEffect( - enableScribeClientEvents: Boolean, - override val logPipelinePublisher: EventPublisher[LogEvent], - injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier], - adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, - whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, - whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None) - extends ScribeClientEventSideEffect[PipelineQuery, Timeline] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery, - Timeline - ] { - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent") - - override val page = "timelinemixer" - - override def onlyIf( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Boolean = enableScribeClientEvents - - override def buildClientEvents( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Seq[ScribeClientEventSideEffect.ClientEvent] = { - - val itemCandidates = CandidatesUtil.getItemCandidates(selectedCandidates) - val sources = itemCandidates.groupBy(_.source) - val injectedTweets = - injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty)) - val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten - - // WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline. - val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten - val whoToSubscribeUsers = - whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten - - val servedEvents = ServedEventsBuilder - .build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers) - - val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets) - - val queryEvents = QueryEventsBuilder.build(query, injectedTweets) - - (servedEvents ++ emptyTimelineEvents ++ queryEvents).filter(_.eventValue.forall(_ > 0)) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.docx new file mode 100644 index 000000000..f61ae8691 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala deleted file mode 100644 index 3f19dfc99..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala +++ /dev/null @@ -1,245 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller -import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller -import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller -import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature -import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature -import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature -import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature -import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature -import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag -import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator -import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.timeline_logging.{thriftscala => thrift} -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Side effect that logs home timeline served candidates to Scribe. - */ -@Singleton -class HomeScribeServedCandidatesSideEffect @Inject() ( - @Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean, - scribeEventPublisher: EventPublisher[thrift.ServedEntry]) - extends ScribeLogEventSideEffect[ - thrift.ServedEntry, - PipelineQuery with HasSeenTweetIds with HasDeviceContext, - Timeline - ] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery with HasSeenTweetIds with HasDeviceContext, - Timeline - ] { - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates") - - override def onlyIf( - query: PipelineQuery with HasSeenTweetIds with HasDeviceContext, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam) - - override def buildLogEvents( - query: PipelineQuery with HasSeenTweetIds with HasDeviceContext, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Seq[thrift.ServedEntry] = { - val timelineType = query.product match { - case FollowingProduct => thrift.TimelineType.HomeLatest - case ForYouProduct => thrift.TimelineType.Home - case SubscribedProduct => thrift.TimelineType.HomeSubscribed - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - val requestProvenance = query.deviceContext.map { deviceContext => - deviceContext.requestContextValue match { - case RequestContext.Foreground => thrift.RequestProvenance.Foreground - case RequestContext.Launch => thrift.RequestProvenance.Launch - case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr - case _ => thrift.RequestProvenance.Other - } - } - val queryType = query.features.map { featureMap => - if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder - else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer - else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle - else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial - else thrift.QueryType.Other - } - val requestInfo = thrift.RequestInfo( - requestTimeMs = query.queryTime.inMilliseconds, - traceId = Trace.id.traceId.toLong, - userId = query.getOptionalUserId, - clientAppId = query.clientContext.appId, - hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)), - parentId = Some(Trace.id.parentId.toLong), - spanId = Some(Trace.id.spanId.toLong), - timelineType = Some(timelineType), - ipAddress = query.clientContext.ipAddress, - userAgent = query.clientContext.userAgent, - queryType = queryType, - requestProvenance = requestProvenance, - languageCode = query.clientContext.languageCode, - countryCode = query.clientContext.countryCode, - requestEndTimeMs = Some(Time.now.inMilliseconds), - servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)), - requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)) - ) - - val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = - selectedCandidates.flatMap { - case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] => - Seq((item.candidateIdLong, item)) - case module: ModuleCandidateWithDetails - if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) => - module.candidates.map(item => (item.candidateIdLong, item)) - case _ => Seq.empty - }.toMap - - val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = - selectedCandidates.flatMap { - case module: ModuleCandidateWithDetails - if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) => - module.candidates.map { item => - (item.candidateIdLong, item) - } - case _ => Seq.empty - }.toMap - - response.instructions.zipWithIndex - .collect { - case (AddEntriesTimelineInstruction(entries), index) => - entries.collect { - case entry: TweetItem if entry.promotedMetadata.isDefined => - val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index) - Seq( - thrift.EntryInfo( - id = entry.id, - position = index.shortValue(), - entryId = entry.entryIdentifier, - entryType = thrift.EntryType.PromotedTweet, - sortIndex = entry.sortIndex, - verticalSize = Some(1), - displayType = Some(entry.displayType.toString), - details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails)) - ) - ) - case entry: TweetItem => - val candidate = tweetIdToItemCandidateMap(entry.id) - val tweetDetails = TweetDetailsMarshaller(entry, candidate) - Seq( - thrift.EntryInfo( - id = candidate.candidateIdLong, - position = index.shortValue(), - entryId = entry.entryIdentifier, - entryType = thrift.EntryType.Tweet, - sortIndex = entry.sortIndex, - verticalSize = Some(1), - score = candidate.features.getOrElse(ScoreFeature, None), - displayType = Some(entry.displayType.toString), - details = Some(thrift.ItemDetails.TweetDetails(tweetDetails)) - ) - ) - case module: TimelineModule - if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString => - module.items.collect { - case ModuleItem(entry: UserItem, _, _) => - val candidate = userIdToItemCandidateMap(entry.id) - val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate) - thrift.EntryInfo( - id = entry.id, - position = index.shortValue(), - entryId = module.entryIdentifier, - entryType = thrift.EntryType.WhoToFollowModule, - sortIndex = module.sortIndex, - score = candidate.features.getOrElse(ScoreFeature, None), - displayType = Some(entry.displayType.toString), - details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails)) - ) - } - case module: TimelineModule - if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString => - module.items.collect { - case ModuleItem(entry: UserItem, _, _) => - val candidate = userIdToItemCandidateMap(entry.id) - val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate) - thrift.EntryInfo( - id = entry.id, - position = index.shortValue(), - entryId = module.entryIdentifier, - entryType = thrift.EntryType.WhoToSubscribeModule, - sortIndex = module.sortIndex, - score = candidate.features.getOrElse(ScoreFeature, None), - displayType = Some(entry.displayType.toString), - details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails)) - ) - } - case module: TimelineModule - if module.sortIndex.isDefined && module.items.headOption.exists( - _.item.isInstanceOf[TweetItem]) => - module.items.collect { - case ModuleItem(entry: TweetItem, _, _) => - val candidate = tweetIdToItemCandidateMap(entry.id) - thrift.EntryInfo( - id = entry.id, - position = index.shortValue(), - entryId = module.entryIdentifier, - entryType = thrift.EntryType.ConversationModule, - sortIndex = module.sortIndex, - score = candidate.features.getOrElse(ScoreFeature, None), - displayType = Some(entry.displayType.toString) - ) - } - case _ => Seq.empty - }.flatten - // Other instructions - case _ => Seq.empty[thrift.EntryInfo] - }.flatten.map { entryInfo => - thrift.ServedEntry( - entry = Some(entryInfo), - request = requestInfo - ) - } - } - - override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] = - scribeEventPublisher - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.docx new file mode 100644 index 000000000..6d1bb45cd Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala deleted file mode 100644 index c0437767e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.eventbus.client.EventBusPublisher -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.impressionstore.thriftscala.Impression -import com.twitter.timelines.impressionstore.thriftscala.ImpressionList -import com.twitter.timelines.impressionstore.thriftscala.PublishedImpressionList -import com.twitter.timelines.impressionstore.thriftscala.SurfaceArea -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -object PublishClientSentImpressionsEventBusSideEffect { - val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline)) - val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline)) - val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed)) -} - -/** - * Side effect that publishes seen tweet IDs sent from clients. The seen tweet IDs are sent to a - * heron topology which writes to a memcache dataset. - */ -@Singleton -class PublishClientSentImpressionsEventBusSideEffect @Inject() ( - eventBusPublisher: EventBusPublisher[PublishedImpressionList]) - extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery with HasSeenTweetIds, - HasMarshalling - ] { - import PublishClientSentImpressionsEventBusSideEffect._ - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("PublishClientSentImpressionsEventBus") - - override def onlyIf( - query: PipelineQuery with HasSeenTweetIds, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: HasMarshalling - ): Boolean = query.seenTweetIds.exists(_.nonEmpty) - - def buildEvents( - query: PipelineQuery with HasSeenTweetIds, - currentTime: Long - ): Option[Seq[Impression]] = { - val surfaceArea = query.product match { - case ForYouProduct => HomeSurfaceArea - case FollowingProduct => HomeLatestSurfaceArea - case SubscribedProduct => HomeSubscribedSurfaceArea - case _ => None - } - query.seenTweetIds.map { seenTweetIds => - seenTweetIds.map { tweetId => - Impression( - tweetId = tweetId, - impressionTime = Some(currentTime), - surfaceAreas = surfaceArea - ) - } - } - } - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] - ): Stitch[Unit] = { - val currentTime = Time.now.inMilliseconds - val impressions = buildEvents(inputs.query, currentTime) - - Stitch.callFuture( - eventBusPublisher.publish( - PublishedImpressionList( - inputs.query.getRequiredUserId, - ImpressionList(impressions), - currentTime - ) - ) - ) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.4) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.docx new file mode 100644 index 000000000..e8a8976a8 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.scala deleted file mode 100644 index bf365f1a6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.TweetImpressionsFeature -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.impression.{thriftscala => t} -import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Side effect that updates the timelines tweet impression - * store (Manhattan) with seen tweet IDs sent from clients - */ -@Singleton -class PublishClientSentImpressionsManhattanSideEffect @Inject() ( - manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) - extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery with HasSeenTweetIds, - HasMarshalling - ] { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("PublishClientSentImpressionsManhattan") - - override def onlyIf( - query: PipelineQuery with HasSeenTweetIds, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: HasMarshalling - ): Boolean = query.seenTweetIds.exists(_.nonEmpty) - - def buildEvents(query: PipelineQuery): Option[(Long, t.TweetImpressionsEntries)] = { - query.features.flatMap { featureMap => - val impressions = featureMap.getOrElse(TweetImpressionsFeature, Seq.empty) - if (impressions.nonEmpty) - Some((query.getRequiredUserId, t.TweetImpressionsEntries(impressions))) - else None - } - } - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] - ): Stitch[Unit] = { - val events = buildEvents(inputs.query) - - Stitch - .traverse(events) { - case (key, value) => manhattanTweetImpressionStoreClient.write(key, value) - } - .unit - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.4) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.docx new file mode 100644 index 000000000..e7811ef79 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala deleted file mode 100644 index f965bafac..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient -import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PublishImpressionBloomFilterSideEffect @Inject() ( - bloomFilterClient: ManhattanStoreClient[ - blm.ImpressionBloomFilterKey, - blm.ImpressionBloomFilterSeq - ]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] - with PipelineResultSideEffect.Conditionally[ - PipelineQuery with HasSeenTweetIds, - HasMarshalling - ] { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("PublishImpressionBloomFilter") - - private val SurfaceArea = blm.SurfaceArea.HomeTimeline - - override def onlyIf( - query: PipelineQuery with HasSeenTweetIds, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: HasMarshalling - ): Boolean = - query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty) - - def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = { - query.features.flatMap { featureMap => - val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature) - if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq) - else None - } - } - - override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] - ): Stitch[Unit] = { - buildEvents(inputs.query) - .map { updatedBloomFilterSeq => - bloomFilterClient.write( - blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea), - updatedBloomFilterSeq) - }.getOrElse(Stitch.Unit) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.docx new file mode 100644 index 000000000..a47e0d1b2 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.scala deleted file mode 100644 index 63855d3b8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.param.HomeGlobalParams.TimelinesPersistenceStoreMaxEntriesPerClient -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelineservice.model.TimelineQuery -import com.twitter.timelineservice.model.core.TimelineKind -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Side effect that truncates entries in the Timelines Persistence store - * based on the number of entries per client. - */ -@Singleton -class TruncateTimelinesPersistenceStoreSideEffect @Inject() ( - timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) - extends PipelineResultSideEffect[PipelineQuery, Timeline] { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("TruncateTimelinesPersistenceStore") - - def getResponsesToDelete(query: PipelineQuery): Seq[TimelineResponseV3] = { - val responses = - query.features.map(_.getOrElse(PersistenceEntriesFeature, Seq.empty)).toSeq.flatten - val responsesByClient = responses.groupBy(_.clientPlatform).values.toSeq - val maxEntriesPerClient = query.params(TimelinesPersistenceStoreMaxEntriesPerClient) - - responsesByClient.flatMap { - _.sortBy(_.servedTime.inMilliseconds) - .foldRight((Seq.empty[TimelineResponseV3], maxEntriesPerClient)) { - case (response, (responsesToDelete, remainingCap)) => - if (remainingCap > 0) (responsesToDelete, remainingCap - response.entries.size) - else (response +: responsesToDelete, remainingCap) - } match { case (responsesToDelete, _) => responsesToDelete } - } - } - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] - ): Stitch[Unit] = { - val timelineKind = inputs.query.product match { - case FollowingProduct => TimelineKind.homeLatest - case ForYouProduct => TimelineKind.home - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - val timelineQuery = TimelineQuery(id = inputs.query.getRequiredUserId, kind = timelineKind) - - val responsesToDelete = getResponsesToDelete(inputs.query) - - if (responsesToDelete.nonEmpty) - Stitch.callFuture(timelineResponseBatchesClient.delete(timelineQuery, responsesToDelete)) - else Stitch.Unit - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.docx new file mode 100644 index 000000000..6973183fa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.scala deleted file mode 100644 index 24199a79c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature -import com.twitter.home_mixer.model.HomeFeatures.PollingFeature -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.side_effect.UserSessionStoreUpdateSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelineservice.model.util.FinagleRequestContext -import com.twitter.user_session_store.ReadWriteUserSessionStore -import com.twitter.user_session_store.WriteRequest -import com.twitter.user_session_store.thriftscala.NonPollingTimestamps -import com.twitter.user_session_store.thriftscala.UserSessionField -import com.twitter.util.Time - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Side effect that updates the User Session Store (Manhattan) with the timestamps of non polling requests. - */ -@Singleton -class UpdateLastNonPollingTimeSideEffect[ - Query <: PipelineQuery with HasDeviceContext, - ResponseType <: HasMarshalling] @Inject() ( - override val userSessionStore: ReadWriteUserSessionStore) - extends UserSessionStoreUpdateSideEffect[ - WriteRequest, - Query, - ResponseType - ] { - private val MaxNonPollingTimes = 10 - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("UpdateLastNonPollingTime") - - /** - * When the request is non polling and is not a background fetch request, update - * the list of non polling timestamps with the timestamp of the current request - */ - override def buildWriteRequest(query: Query): Option[WriteRequest] = { - val isBackgroundFetch = query.deviceContext - .exists(_.requestContextValue.contains(DeviceContext.RequestContext.BackgroundFetch)) - - if (!query.features.exists(_.getOrElse(PollingFeature, false)) && !isBackgroundFetch) { - val fields = Seq(UserSessionField.NonPollingTimestamps(makeLastNonPollingTimestamps(query))) - Some(WriteRequest(query.getRequiredUserId, fields)) - } else None - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.96) - ) - - private def makeLastNonPollingTimestamps(query: Query): NonPollingTimestamps = { - val priorNonPollingTimestamps = - query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty)).toSeq.flatten - - val lastNonPollingTimeMs = - FinagleRequestContext.default.requestStartTime.get.getOrElse(Time.now).inMillis - - val followingLastNonPollingTime = query.features - .flatMap(features => features.getOrElse(FollowingLastNonPollingTimeFeature, None)) - .map(_.inMillis) - - NonPollingTimestamps( - nonPollingTimestampsMs = - (lastNonPollingTimeMs +: priorNonPollingTimestamps).take(MaxNonPollingTimes), - mostRecentHomeLatestNonPollingTimestampMs = - if (query.product == FollowingProduct) Some(lastNonPollingTimeMs) - else followingLastNonPollingTime - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.docx new file mode 100644 index 000000000..89e74e844 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala deleted file mode 100644 index 5a1bb0f6b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala +++ /dev/null @@ -1,263 +0,0 @@ -package com.twitter.home_mixer.functional_component.side_effect - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.ReplaceEntryTimelineInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowCoverInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelinemixer.clients.persistence.EntryWithItemIds -import com.twitter.timelinemixer.clients.persistence.ItemIds -import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelines.persistence.thriftscala.TweetScoreV1 -import com.twitter.timelines.persistence.{thriftscala => persistence} -import com.twitter.timelineservice.model.TimelineQuery -import com.twitter.timelineservice.model.TimelineQueryOptions -import com.twitter.timelineservice.model.TweetScore -import com.twitter.timelineservice.model.core.TimelineKind -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.util.Time -import com.twitter.{timelineservice => tls} -import javax.inject.Inject -import javax.inject.Singleton - -object UpdateTimelinesPersistenceStoreSideEffect { - val EmptyItemIds = ItemIds( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None) -} - -/** - * Side effect that updates the Timelines Persistence Store (Manhattan) with the entries being returned. - */ -@Singleton -class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( - timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) - extends PipelineResultSideEffect[PipelineQuery, Timeline] { - - override val identifier: SideEffectIdentifier = - SideEffectIdentifier("UpdateTimelinesPersistenceStore") - - final override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] - ): Stitch[Unit] = { - if (inputs.response.instructions.nonEmpty) { - val timelineKind = inputs.query.product match { - case FollowingProduct => TimelineKind.homeLatest - case ForYouProduct => TimelineKind.home - case other => throw new UnsupportedOperationException(s"Unknown product: $other") - } - val timelineQuery = TimelineQuery( - id = inputs.query.getRequiredUserId, - kind = timelineKind, - options = TimelineQueryOptions( - contextualUserId = inputs.query.getOptionalUserId, - deviceContext = tls.DeviceContext.empty.copy( - userAgent = inputs.query.clientContext.userAgent, - clientAppId = inputs.query.clientContext.appId) - ) - ) - - val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = - inputs.selectedCandidates.flatMap { - case item: ItemCandidateWithDetails if item.candidate.id.isInstanceOf[Long] => - Seq((item.candidateIdLong, item)) - case module: ModuleCandidateWithDetails - if module.candidates.headOption.exists(_.candidate.id.isInstanceOf[Long]) => - module.candidates.map(item => (item.candidateIdLong, item)) - case _ => Seq.empty - }.toMap - - val entries = inputs.response.instructions.collect { - case AddEntriesTimelineInstruction(entries) => - entries.collect { - // includes tweets, tweet previews, and promoted tweets - case entry: TweetItem if entry.sortIndex.isDefined => { - Seq( - buildTweetEntryWithItemIds( - tweetIdToItemCandidateMap(entry.id), - entry.sortIndex.get - )) - } - // tweet conversation modules are flattened to individual tweets in the persistence store - case module: TimelineModule - if module.sortIndex.isDefined && module.items.headOption.exists( - _.item.isInstanceOf[TweetItem]) => - module.items.map { item => - buildTweetEntryWithItemIds( - tweetIdToItemCandidateMap(item.item.id.asInstanceOf[Long]), - module.sortIndex.get) - } - case module: TimelineModule - if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString => - val userIds = module.items - .map(item => - UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId = - Some(item.item.id.asInstanceOf[Long]))) - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.WhoToFollow, - sortIndex = module.sortIndex.get, - size = module.items.size.toShort, - itemIds = Some(userIds) - )) - case module: TimelineModule - if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString => - val userIds = module.items - .map(item => - UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId = - Some(item.item.id.asInstanceOf[Long]))) - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.WhoToSubscribe, - sortIndex = module.sortIndex.get, - size = module.items.size.toShort, - itemIds = Some(userIds) - )) - }.flatten - case ShowCoverInstruction(cover) => - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.Prompt, - sortIndex = cover.sortIndex.get, - size = 1, - itemIds = None - ) - ) - case ReplaceEntryTimelineInstruction(entry) => - val namespaceLength = TweetItem.TweetEntryNamespace.toString.length - Seq( - EntryWithItemIds( - entityIdType = EntityIdType.Tweet, - sortIndex = entry.sortIndex.get, - size = 1, - itemIds = Some( - Seq( - ItemIds( - tweetId = - entry.entryIdToReplace.map(e => e.substring(namespaceLength + 1).toLong), - sourceTweetId = None, - quoteTweetId = None, - sourceAuthorId = None, - quoteAuthorId = None, - inReplyToTweetId = None, - inReplyToAuthorId = None, - semanticCoreId = None, - articleId = None, - hasRelevancePrompt = None, - promptData = None, - tweetScore = None, - entryIdToReplace = entry.entryIdToReplace, - tweetReactiveData = None, - userId = None - ) - )) - ) - ) - - }.flatten - - val response = TimelineResponseV3( - clientPlatform = timelineQuery.clientPlatform, - servedTime = Time.now, - requestType = requestTypeFromQuery(inputs.query), - entries = entries) - - Stitch.callFuture(timelineResponseBatchesClient.insertResponse(timelineQuery, response)) - } else Stitch.Unit - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) - ) - - private def buildTweetEntryWithItemIds( - candidate: ItemCandidateWithDetails, - sortIndex: Long - ): EntryWithItemIds = { - val features = candidate.features - val sourceAuthorId = - if (features.getOrElse(IsRetweetFeature, false)) features.getOrElse(SourceUserIdFeature, None) - else features.getOrElse(AuthorIdFeature, None) - val quoteAuthorId = - if (features.getOrElse(QuotedTweetIdFeature, None).nonEmpty) - features.getOrElse(SourceUserIdFeature, None) - else None - val tweetScore = features.getOrElse(ScoreFeature, None).map { score => - TweetScore.fromThrift(persistence.TweetScore.TweetScoreV1(TweetScoreV1(score))) - } - - val itemIds = ItemIds( - tweetId = Some(candidate.candidateIdLong), - sourceTweetId = features.getOrElse(SourceTweetIdFeature, None), - quoteTweetId = features.getOrElse(QuotedTweetIdFeature, None), - sourceAuthorId = sourceAuthorId, - quoteAuthorId = quoteAuthorId, - inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None), - inReplyToAuthorId = features.getOrElse(DirectedAtUserIdFeature, None), - semanticCoreId = features.getOrElse(SemanticCoreIdFeature, None), - articleId = None, - hasRelevancePrompt = None, - promptData = None, - tweetScore = tweetScore, - entryIdToReplace = None, - tweetReactiveData = None, - userId = None - ) - - val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false) - val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet - - EntryWithItemIds( - entityIdType = entityType, - sortIndex = sortIndex, - size = 1.toShort, - itemIds = Some(Seq(itemIds)) - ) - } - - private def requestTypeFromQuery(query: PipelineQuery): persistence.RequestType = { - val features = query.features.getOrElse(FeatureMap.empty) - - val featureToRequestType = Seq( - (PollingFeature, persistence.RequestType.Polling), - (GetInitialFeature, persistence.RequestType.Initial), - (GetNewerFeature, persistence.RequestType.Newer), - (GetMiddleFeature, persistence.RequestType.Middle), - (GetOlderFeature, persistence.RequestType.Older) - ) - - featureToRequestType - .collectFirst { - case (feature, requestType) if features.getOrElse(feature, false) => requestType - }.getOrElse(persistence.RequestType.Other) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel deleted file mode 100644 index fd35daeeb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - ], - exports = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.docx new file mode 100644 index 000000000..e210962c4 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.docx new file mode 100644 index 000000000..973529727 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.scala deleted file mode 100644 index b2c6e6347..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class DeviceContextUnmarshaller @Inject() () { - - def apply(deviceContext: t.DeviceContext): DeviceContext = { - DeviceContext( - isPolling = deviceContext.isPolling, - requestContext = deviceContext.requestContext, - latestControlAvailable = deviceContext.latestControlAvailable, - autoplayEnabled = deviceContext.autoplayEnabled - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.docx new file mode 100644 index 000000000..5a6558da6 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala deleted file mode 100644 index 81a9abf2b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.HomeMixerDebugOptions -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.marshaller.request.FeatureValueUnmarshaller -import com.twitter.product_mixer.core.model.marshalling.request.DebugParams -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerDebugParamsUnmarshaller @Inject() ( - featureValueUnmarshaller: FeatureValueUnmarshaller) { - - def apply(debugParams: t.DebugParams): DebugParams = { - DebugParams( - featureOverrides = debugParams.featureOverrides.map { map => - map.mapValues(featureValueUnmarshaller(_)).toMap - }, - debugOptions = debugParams.debugOptions.map { options => - HomeMixerDebugOptions( - requestTimeOverride = options.requestTimeOverrideMillis.map(Time.fromMilliseconds) - ) - } - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.docx new file mode 100644 index 000000000..1438f23aa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala deleted file mode 100644 index ec3a183b9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.FollowingProductContext -import com.twitter.home_mixer.model.request.ForYouProductContext -import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext -import com.twitter.home_mixer.model.request.ListTweetsProductContext -import com.twitter.home_mixer.model.request.ScoredTweetsProductContext -import com.twitter.home_mixer.model.request.SubscribedProductContext -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.model.marshalling.request.ProductContext -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerProductContextUnmarshaller @Inject() ( - deviceContextUnmarshaller: DeviceContextUnmarshaller) { - - def apply(productContext: t.ProductContext): ProductContext = productContext match { - case t.ProductContext.Following(p) => - FollowingProductContext( - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - seenTweetIds = p.seenTweetIds, - dspClientContext = p.dspClientContext - ) - case t.ProductContext.ForYou(p) => - ForYouProductContext( - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - seenTweetIds = p.seenTweetIds, - dspClientContext = p.dspClientContext, - pushToHomeTweetId = p.pushToHomeTweetId - ) - case t.ProductContext.ListManagement(p) => - throw new UnsupportedOperationException(s"This product is no longer used") - case t.ProductContext.ScoredTweets(p) => - ScoredTweetsProductContext( - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - seenTweetIds = p.seenTweetIds, - servedTweetIds = p.servedTweetIds, - backfillTweetIds = p.backfillTweetIds - ) - case t.ProductContext.ListTweets(p) => - ListTweetsProductContext( - listId = p.listId, - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - dspClientContext = p.dspClientContext - ) - case t.ProductContext.ListRecommendedUsers(p) => - ListRecommendedUsersProductContext( - listId = p.listId, - selectedUserIds = p.selectedUserIds, - excludedUserIds = p.excludedUserIds, - listName = p.listName - ) - case t.ProductContext.Subscribed(p) => - SubscribedProductContext( - deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), - seenTweetIds = p.seenTweetIds, - ) - case t.ProductContext.UnknownUnionField(field) => - throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}") - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.docx new file mode 100644 index 000000000..65773b707 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala deleted file mode 100644 index f5d0d002b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.model.marshalling.request.Product -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerProductUnmarshaller @Inject() () { - - def apply(product: t.Product): Product = product match { - case t.Product.Following => FollowingProduct - case t.Product.ForYou => ForYouProduct - case t.Product.ListManagement => - throw new UnsupportedOperationException(s"This product is no longer used") - case t.Product.ScoredTweets => ScoredTweetsProduct - case t.Product.ListTweets => ListTweetsProduct - case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct - case t.Product.Subscribed => SubscribedProduct - case t.Product.EnumUnknownProduct(value) => - throw new UnsupportedOperationException(s"Unknown product: $value") - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.docx new file mode 100644 index 000000000..35dba327c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.scala deleted file mode 100644 index b8894c8b0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.home_mixer.marshaller.request - -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextUnmarshaller -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeMixerRequestUnmarshaller @Inject() ( - clientContextUnmarshaller: ClientContextUnmarshaller, - homeProductUnmarshaller: HomeMixerProductUnmarshaller, - homeProductContextUnmarshaller: HomeMixerProductContextUnmarshaller, - homeDebugParamsUnmarshaller: HomeMixerDebugParamsUnmarshaller) { - - def apply(homeRequest: t.HomeMixerRequest): HomeMixerRequest = { - HomeMixerRequest( - clientContext = clientContextUnmarshaller(homeRequest.clientContext), - product = homeProductUnmarshaller(homeRequest.product), - productContext = homeRequest.productContext.map(homeProductContextUnmarshaller(_)), - // Avoid de-serializing cursors in the request unmarshaller. The unmarshaller should never - // fail, which is often a possibility when trying to de-serialize a cursor. Cursors can also - // be product-specific and more appropriately handled in individual product pipelines. - serializedRequestCursor = homeRequest.cursor, - maxResults = homeRequest.maxResults, - debugParams = homeRequest.debugParams.map(homeDebugParamsUnmarshaller(_)), - homeRequestParam = false - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel deleted file mode 100644 index efcad840b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.docx new file mode 100644 index 000000000..cd00fb1bd Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.docx new file mode 100644 index 000000000..b4c4d5c34 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala deleted file mode 100644 index d913f8d64..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.home_mixer.marshaller.timeline_logging - -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} - -object PromotedTweetDetailsMarshaller { - - def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetDetails = { - thriftlog.PromotedTweetDetails( - advertiserId = Some(entry.promotedMetadata.map(_.advertiserId).getOrElse(0L)), - insertPosition = Some(position), - impressionId = entry.promotedMetadata.flatMap(_.impressionString) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.docx new file mode 100644 index 000000000..8dc19d4bc Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala deleted file mode 100644 index 8e1c475d5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.home_mixer.marshaller.timeline_logging - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation -import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ConversationGeneralContextType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext -import com.twitter.timelines.service.{thriftscala => tst} -import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} - -object TweetDetailsMarshaller { - - private val generalContextTypeMarshaller = new GeneralContextTypeMarshaller() - - def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetDetails = { - val socialContext = candidate.presentation.flatMap { - case _ @UrtItemPresentation(timelineItem: TweetItem, _) => timelineItem.socialContext - case _ @UrtModulePresentation(timelineModule) => - timelineModule.items.head.item match { - case timelineItem: TweetItem => timelineItem.socialContext - case _ => Some(ConversationGeneralContextType) - } - } - - val socialContextType = socialContext match { - case Some(GeneralContext(contextType, _, _, _, _)) => - Some(generalContextTypeMarshaller(contextType).value.toShort) - case Some(TopicContext(_, _)) => Some(tst.ContextType.Topic.value.toShort) - case _ => None - } - - thriftlog.TweetDetails( - sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), - socialContextType = socialContextType, - suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name), - authorId = candidate.features.getOrElse(AuthorIdFeature, None), - sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.docx new file mode 100644 index 000000000..1acee4f7a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala deleted file mode 100644 index 4c0b4dd1b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.home_mixer.marshaller.timeline_logging - -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem -import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} - -object WhoToFollowDetailsMarshaller { - - def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowDetails = - thriftlog.WhoToFollowDetails( - enableReactiveBlending = entry.enableReactiveBlending, - impressionId = entry.promotedMetadata.flatMap(_.impressionString), - advertiserId = entry.promotedMetadata.map(_.advertiserId) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel deleted file mode 100644 index ebf305ead..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "src/thrift/com/twitter/timelineservice:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.docx new file mode 100644 index 000000000..92ed23ec6 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.docx new file mode 100644 index 000000000..551f0ec05 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.scala deleted file mode 100644 index 40e25a8ab..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.timelines.service.{thriftscala => t} - -object ChronologicalCursorMarshaller { - - def apply(cursor: UrtOrderedCursor): Option[t.ChronologicalCursor] = { - cursor.cursorType match { - case Some(TopCursor) => Some(t.ChronologicalCursor(bottom = cursor.id)) - case Some(BottomCursor) => Some(t.ChronologicalCursor(top = cursor.id)) - case Some(GapCursor) => - Some(t.ChronologicalCursor(top = cursor.id, bottom = cursor.gapBoundaryId)) - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.docx new file mode 100644 index 000000000..47cb99a5e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.scala deleted file mode 100644 index 739c490ea..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.timelines.service.{thriftscala => t} - -object ChronologicalCursorUnmarshaller { - - def apply(requestCursor: t.RequestCursor): Option[UrtOrderedCursor] = { - requestCursor match { - case t.RequestCursor.ChronologicalCursor(cursor) => - (cursor.top, cursor.bottom) match { - case (Some(top), None) => - Some(UrtOrderedCursor(top, cursor.top, Some(BottomCursor))) - case (None, Some(bottom)) => - Some(UrtOrderedCursor(bottom, cursor.bottom, Some(TopCursor))) - case (Some(top), Some(bottom)) => - Some(UrtOrderedCursor(top, cursor.top, Some(GapCursor), cursor.bottom)) - case _ => None - } - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.docx new file mode 100644 index 000000000..e0cd03b52 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.scala deleted file mode 100644 index 097d8dea4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.timelineservice.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class DeviceContextMarshaller @Inject() () { - - def apply(deviceContext: DeviceContext, clientContext: ClientContext): t.DeviceContext = { - t.DeviceContext( - countryCode = clientContext.countryCode, - languageCode = clientContext.languageCode, - clientAppId = clientContext.appId, - ipAddress = clientContext.ipAddress, - guestId = clientContext.guestId, - userAgent = clientContext.userAgent, - deviceId = clientContext.deviceId, - isPolling = deviceContext.isPolling, - requestContext = deviceContext.requestContext, - referrer = None, - tfeAuthHeader = None, - mobileDeviceId = clientContext.mobileDeviceId, - isSessionStart = None, - latestControlAvailable = deviceContext.latestControlAvailable, - guestIdMarketing = clientContext.guestIdMarketing, - isInternalOrTwoffice = clientContext.isTwoffice, - guestIdAds = clientContext.guestIdAds, - isUrtRequest = Some(true) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.docx new file mode 100644 index 000000000..1b9973043 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.scala deleted file mode 100644 index 693684558..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor -import com.twitter.timelines.service.{thriftscala => t} -import com.twitter.util.Time - -object RecommendedUsersCursorUnmarshaller { - - def apply(requestCursor: t.RequestCursor): Option[UrtUnorderedExcludeIdsCursor] = { - requestCursor match { - case t.RequestCursor.RecommendedUsersCursor(cursor) => - Some( - UrtUnorderedExcludeIdsCursor( - initialSortIndex = cursor.minSortIndex.getOrElse(Time.now.inMilliseconds), - excludedIds = cursor.previouslyRecommendedUserIds - )) - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.docx new file mode 100644 index 000000000..cfcb762fc Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.scala deleted file mode 100644 index cbc956ad4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.timelineservice.{thriftscala => t} - -object TimelineServiceCursorMarshaller { - - def apply(cursor: UrtOrderedCursor): Option[t.Cursor2] = { - val id = cursor.id.map(_.toString) - val gapBoundaryId = cursor.gapBoundaryId.map(_.toString) - cursor.cursorType match { - case Some(TopCursor) => Some(t.Cursor2(bottom = id)) - case Some(BottomCursor) => Some(t.Cursor2(top = id)) - case Some(GapCursor) => Some(t.Cursor2(top = id, bottom = gapBoundaryId)) - case _ => None - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.docx new file mode 100644 index 000000000..b72efd81c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.scala deleted file mode 100644 index f542cd535..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.marshaller.timelines - -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType -import com.twitter.timelines.render.{thriftscala => urt} - -object TopicContextFunctionalityTypeUnmarshaller { - - def apply( - topicContextFunctionalityType: urt.TopicContextFunctionalityType - ): TopicContextFunctionalityType = topicContextFunctionalityType match { - case urt.TopicContextFunctionalityType.Basic => BasicTopicContextFunctionalityType - case urt.TopicContextFunctionalityType.Recommendation => - RecommendationTopicContextFunctionalityType - case urt.TopicContextFunctionalityType.RecWithEducation => - RecWithEducationTopicContextFunctionalityType - case urt.TopicContextFunctionalityType.EnumUnknownTopicContextFunctionalityType(field) => - throw new UnsupportedOperationException(s"Unknown topic context functionality type: $field") - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel deleted file mode 100644 index 65ece62a3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel +++ /dev/null @@ -1,42 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api:api-base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/recap", - "src/scala/com/twitter/timelines/prediction/features/request_context", - "src/thrift/com/twitter/escherbird:tweet-annotation-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "topic-social-proof/server/src/main/thrift:thrift-scala", - "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/timelines/impression:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.docx new file mode 100644 index 000000000..5f4382942 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.docx new file mode 100644 index 000000000..febded24c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala deleted file mode 100644 index 85154e55b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.home_mixer.model.request.DeviceContext.RequestContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -/** - * Include a clear cache timeline instruction when we satisfy these criteria: - * - Request Provenance is "pull to refresh" - * - Atleast N non-ad tweet entries in the response - * - * This is to ensure that we have sufficient new content to justify jumping users to the - * top of the new timelines response and don't add unnecessary load to backend systems - */ -case class ClearCacheIncludeInstruction( - enableParam: FSParam[Boolean], - minEntriesParam: FSBoundedParam[Int]) - extends IncludeInstruction[PipelineQuery with HasDeviceContext] { - - override def apply( - query: PipelineQuery with HasDeviceContext, - entries: Seq[TimelineEntry] - ): Boolean = { - val enabled = query.params(enableParam) - - val ptr = - query.deviceContext.flatMap(_.requestContextValue).contains(RequestContext.PullToRefresh) - - val minTweets = query.params(minEntriesParam) <= entries.collect { - case item: TweetItem if item.promotedMetadata.isEmpty => 1 - case module: TimelineModule if module.items.head.item.isInstanceOf[TweetItem] => - module.items.size - }.sum - - enabled && ptr && minTweets - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.docx new file mode 100644 index 000000000..b7f25af6b Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala deleted file mode 100644 index f141d67d1..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.escherbird.{thriftscala => esb} -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.tweetypie.{thriftscala => tp} - -object ContentFeatures { - val Empty: ContentFeatures = ContentFeatures( - 0.toShort, - false, - 0.toShort, - 0.toShort, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None - ) - - def fromThrift(ebFeatures: sc.ThriftTweetFeatures): ContentFeatures = - ContentFeatures( - length = ebFeatures.tweetLength.getOrElse(0).toShort, - hasQuestion = ebFeatures.hasQuestion.getOrElse(false), - numCaps = ebFeatures.numCaps.getOrElse(0).toShort, - numWhiteSpaces = ebFeatures.numWhitespaces.getOrElse(0).toShort, - numNewlines = ebFeatures.numNewlines, - videoDurationMs = ebFeatures.videoDurationMs, - bitRate = ebFeatures.bitRate, - aspectRatioNum = ebFeatures.aspectRatioNum, - aspectRatioDen = ebFeatures.aspectRatioDen, - widths = ebFeatures.widths.map(_.map(_.toShort)), - heights = ebFeatures.heights.map(_.map(_.toShort)), - resizeMethods = ebFeatures.resizeMethods.map(_.map(_.toShort)), - numMediaTags = ebFeatures.numMediaTags.map(_.toShort), - mediaTagScreenNames = ebFeatures.mediaTagScreenNames, - emojiTokens = ebFeatures.emojiTokens.map(_.toSet), - emoticonTokens = ebFeatures.emoticonTokens.map(_.toSet), - faceAreas = ebFeatures.faceAreas, - dominantColorRed = ebFeatures.dominantColorRed, - dominantColorBlue = ebFeatures.dominantColorBlue, - dominantColorGreen = ebFeatures.dominantColorGreen, - numColors = ebFeatures.numColors.map(_.toShort), - stickerIds = ebFeatures.stickerIds, - mediaOriginProviders = ebFeatures.mediaOriginProviders, - isManaged = ebFeatures.isManaged, - is360 = ebFeatures.is360, - viewCount = ebFeatures.viewCount, - isMonetizable = ebFeatures.isMonetizable, - isEmbeddable = ebFeatures.isEmbeddable, - hasSelectedPreviewImage = ebFeatures.hasSelectedPreviewImage, - hasTitle = ebFeatures.hasTitle, - hasDescription = ebFeatures.hasDescription, - hasVisitSiteCallToAction = ebFeatures.hasVisitSiteCallToAction, - hasAppInstallCallToAction = ebFeatures.hasAppInstallCallToAction, - hasWatchNowCallToAction = ebFeatures.hasWatchNowCallToAction, - dominantColorPercentage = ebFeatures.dominantColorPercentage, - posUnigrams = ebFeatures.posUnigrams.map(_.toSet), - posBigrams = ebFeatures.posBigrams.map(_.toSet), - semanticCoreAnnotations = ebFeatures.semanticCoreAnnotations, - tokens = ebFeatures.textTokens.map(_.toSeq), - conversationControl = ebFeatures.conversationControl, - // media and selfThreadMetadata not carried by ThriftTweetFeatures - media = None, - selfThreadMetadata = None - ) -} - -case class ContentFeatures( - length: Short, - hasQuestion: Boolean, - numCaps: Short, - numWhiteSpaces: Short, - numNewlines: Option[Short], - videoDurationMs: Option[Int], - bitRate: Option[Int], - aspectRatioNum: Option[Short], - aspectRatioDen: Option[Short], - widths: Option[Seq[Short]], - heights: Option[Seq[Short]], - resizeMethods: Option[Seq[Short]], - numMediaTags: Option[Short], - mediaTagScreenNames: Option[Seq[String]], - emojiTokens: Option[Set[String]], - emoticonTokens: Option[Set[String]], - faceAreas: Option[Seq[Int]], - dominantColorRed: Option[Short], - dominantColorBlue: Option[Short], - dominantColorGreen: Option[Short], - numColors: Option[Short], - stickerIds: Option[Seq[Long]], - mediaOriginProviders: Option[Seq[String]], - isManaged: Option[Boolean], - is360: Option[Boolean], - viewCount: Option[Long], - isMonetizable: Option[Boolean], - isEmbeddable: Option[Boolean], - hasSelectedPreviewImage: Option[Boolean], - hasTitle: Option[Boolean], - hasDescription: Option[Boolean], - hasVisitSiteCallToAction: Option[Boolean], - hasAppInstallCallToAction: Option[Boolean], - hasWatchNowCallToAction: Option[Boolean], - media: Option[Seq[tp.MediaEntity]], - dominantColorPercentage: Option[Double], - posUnigrams: Option[Set[String]], - posBigrams: Option[Set[String]], - semanticCoreAnnotations: Option[Seq[esb.TweetEntityAnnotation]], - selfThreadMetadata: Option[tp.SelfThreadMetadata], - tokens: Option[Seq[String]], - conversationControl: Option[tp.ConversationControl], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.docx new file mode 100644 index 000000000..64763d426 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.scala deleted file mode 100644 index c11631fc8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdBottomTweetFeature -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdResponseTruncatedFeature -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * Determine whether to include a Gap Cursor in the response based on whether a timeline - * is truncated because it has more entries than the max response size. - * There are two ways this can happen: - * 1) There are unused entries in Earlybird. This is determined by a flag returned from Earlybird. - * We respect the Earlybird flag only if there are some entries after deduping and filtering - * to ensure that we do not get stuck repeatedly serving gaps which lead to no tweets. - * 2) Ads injection can take the response size over the max count. Goldfinch truncates tweet - * entries in this case. We can check if the bottom tweet from Earlybird is in the response to - * determine if all Earlybird tweets have been used. - * - * While scrolling down to get older tweets (BottomCursor), responses will generally be - * truncated, but we don't want to render a gap cursor there, so we need to ensure we only - * apply the truncation check to newer (TopCursor) or middle (GapCursor) requests. - * - * We return either a Gap Cursor or a Bottom Cursor, but not both, so the include instruction - * for Bottom should be the inverse of Gap. - */ -object GapIncludeInstruction - extends IncludeInstruction[PipelineQuery with HasPipelineCursor[UrtOrderedCursor]] { - - override def apply( - query: PipelineQuery with HasPipelineCursor[UrtOrderedCursor], - entries: Seq[TimelineEntry] - ): Boolean = { - val wasTruncated = query.features.exists(_.getOrElse(EarlybirdResponseTruncatedFeature, false)) - - // Get oldest tweet or tweets within oldest conversation module - val tweetEntries = entries.view.reverse - .collectFirst { - case item: TweetItem if item.promotedMetadata.isEmpty => Seq(item.id.toString) - case module: TimelineModule if module.items.head.item.isInstanceOf[TweetItem] => - module.items.map(_.item.id.toString) - }.toSeq.flatten - - val bottomCursor = - query.features.flatMap(_.getOrElse(EarlybirdBottomTweetFeature, None)).map(_.toString) - - // Ads truncation happened if we have at least max count entries and bottom tweet is missing - val adsTruncation = query.requestedMaxResults.exists(_ <= entries.size) && - !bottomCursor.exists(tweetEntries.contains) - - query.pipelineCursor.exists(_.cursorType match { - case Some(TopCursor) | Some(GapCursor) => - (wasTruncated && tweetEntries.nonEmpty) || adsTruncation - case _ => false - }) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.docx new file mode 100644 index 000000000..4ca23b759 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.scala deleted file mode 100644 index bfbe722fc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.adserver.thriftscala.RequestTriggerType -import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature -import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature -import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature -import com.twitter.home_mixer.model.HomeFeatures.PollingFeature -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.model.query.ads.AdsQuery -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -/** - * These are for feeds needed for ads only. - */ -trait HomeAdsQuery - extends AdsQuery - with PipelineQuery - with HasDeviceContext - with HasPipelineCursor[UrtOrderedCursor] { - - private val featureToRequestTriggerType = Seq( - (GetInitialFeature, RequestTriggerType.Initial), - (GetNewerFeature, RequestTriggerType.Scroll), - (GetOlderFeature, RequestTriggerType.Scroll), - (PollingFeature, RequestTriggerType.AutoRefresh) - ) - - override val autoplayEnabled: Option[Boolean] = deviceContext.flatMap(_.autoplayEnabled) - - override def requestTriggerType: Option[RequestTriggerType] = { - val features = this.features.getOrElse(FeatureMap.empty) - - featureToRequestTriggerType.collectFirst { - case (feature, requestType) if features.get(feature) => Some(requestType) - }.flatten - } - - override val disableNsfwAvoidance: Option[Boolean] = Some(true) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.docx new file mode 100644 index 000000000..1b854c306 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala deleted file mode 100644 index fe085b1f9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala +++ /dev/null @@ -1,325 +0,0 @@ -package com.twitter.home_mixer.model - -import com.twitter.core_workflows.user_model.{thriftscala => um} -import com.twitter.dal.personal_data.{thriftjava => pd} -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.BoolDataRecordCompatible -import com.twitter.product_mixer.core.feature.datarecord.DataRecordFeature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature -import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible -import com.twitter.product_mixer.core.feature.datarecord.LongDiscreteDataRecordCompatible -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelinemixer.clients.manhattan.DismissInfo -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.timelinemixer.injection.model.candidate.AudioSpaceMetaData -import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures -import com.twitter.timelines.impression.{thriftscala => imp} -import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} -import com.twitter.timelines.model.UserId -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import com.twitter.timelines.prediction.features.engagement_features.EngagementDataRecordFeatures -import com.twitter.timelines.prediction.features.recap.RecapFeatures -import com.twitter.timelines.prediction.features.request_context.RequestContextFeatures -import com.twitter.timelines.service.{thriftscala => tst} -import com.twitter.timelineservice.model.FeedbackEntry -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.tsp.{thriftscala => tsp} -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import com.twitter.util.Time - -object HomeFeatures { - // Candidate Features - object AncestorsFeature extends Feature[TweetCandidate, Seq[ta.TweetAncestor]] - object AudioSpaceMetaDataFeature extends Feature[TweetCandidate, Option[AudioSpaceMetaData]] - object TwitterListIdFeature extends Feature[TweetCandidate, Option[Long]] - - /** - * For Retweets, this should refer to the retweeting user. Use [[SourceUserIdFeature]] if you want to know - * who created the Tweet that was retweeted. - */ - object AuthorIdFeature - extends DataRecordOptionalFeature[TweetCandidate, Long] - with LongDiscreteDataRecordCompatible { - override val featureName: String = SharedFeatures.AUTHOR_ID.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) - } - - object AuthorIsBlueVerifiedFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsGoldVerifiedFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsGrayVerifiedFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsLegacyVerifiedFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsCreatorFeature extends Feature[TweetCandidate, Boolean] - object AuthorIsProtectedFeature extends Feature[TweetCandidate, Boolean] - - object AuthoredByContextualUserFeature extends Feature[TweetCandidate, Boolean] - object CachedCandidatePipelineIdentifierFeature extends Feature[TweetCandidate, Option[String]] - object CandidateSourceIdFeature - extends Feature[TweetCandidate, Option[cts.CandidateTweetSourceId]] - object ConversationFeature extends Feature[TweetCandidate, Option[ConversationFeatures]] - - /** - * This field should be set to the focal Tweet's tweetId for all tweets which are expected to - * be rendered in the same convo module. For non-convo module Tweets, this will be - * set to None. Note this is different from how TweetyPie defines ConversationId which is defined - * on all Tweets and points to the root tweet. This feature is used for grouping convo modules together. - */ - object ConversationModuleFocalTweetIdFeature extends Feature[TweetCandidate, Option[Long]] - - /** - * This field should always be set to the root Tweet in a conversation for all Tweets. For replies, this will - * point back to the root Tweet. For non-replies, this will be the candidate's Tweet id. This is consistent with - * the TweetyPie definition of ConversationModuleId. - */ - object ConversationModuleIdFeature extends Feature[TweetCandidate, Option[Long]] - object DirectedAtUserIdFeature extends Feature[TweetCandidate, Option[Long]] - object EarlybirdFeature extends Feature[TweetCandidate, Option[sc.ThriftTweetFeatures]] - object EarlybirdScoreFeature extends Feature[TweetCandidate, Option[Double]] - object EarlybirdSearchResultFeature extends Feature[TweetCandidate, Option[eb.ThriftSearchResult]] - object EntityTokenFeature extends Feature[TweetCandidate, Option[String]] - object ExclusiveConversationAuthorIdFeature extends Feature[TweetCandidate, Option[Long]] - object FavoritedByCountFeature - extends DataRecordFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - override val featureName: String = - EngagementDataRecordFeatures.InNetworkFavoritesCount.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = - Set(pd.PersonalDataType.CountOfPrivateLikes, pd.PersonalDataType.CountOfPublicLikes) - } - object FavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object FeedbackHistoryFeature extends Feature[TweetCandidate, Seq[FeedbackEntry]] - object RetweetedByCountFeature - extends DataRecordFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - override val featureName: String = - EngagementDataRecordFeatures.InNetworkRetweetsCount.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = - Set(pd.PersonalDataType.CountOfPrivateRetweets, pd.PersonalDataType.CountOfPublicRetweets) - } - object RetweetedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object RepliedByCountFeature - extends DataRecordFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - override val featureName: String = - EngagementDataRecordFeatures.InNetworkRepliesCount.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = - Set(pd.PersonalDataType.CountOfPrivateReplies, pd.PersonalDataType.CountOfPublicReplies) - } - object RepliedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object FollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - - object TopicIdSocialContextFeature extends Feature[TweetCandidate, Option[Long]] - object TopicContextFunctionalityTypeFeature - extends Feature[TweetCandidate, Option[TopicContextFunctionalityType]] - object FromInNetworkSourceFeature extends Feature[TweetCandidate, Boolean] - - object FullScoringSucceededFeature extends Feature[TweetCandidate, Boolean] - object HasDisplayedTextFeature extends Feature[TweetCandidate, Boolean] - object InReplyToTweetIdFeature extends Feature[TweetCandidate, Option[Long]] - object InReplyToUserIdFeature extends Feature[TweetCandidate, Option[Long]] - object IsAncestorCandidateFeature extends Feature[TweetCandidate, Boolean] - object IsExtendedReplyFeature - extends DataRecordFeature[TweetCandidate, Boolean] - with BoolDataRecordCompatible { - override val featureName: String = RecapFeatures.IS_EXTENDED_REPLY.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object IsRandomTweetFeature - extends DataRecordFeature[TweetCandidate, Boolean] - with BoolDataRecordCompatible { - override val featureName: String = TimelinesSharedFeatures.IS_RANDOM_TWEET.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object IsReadFromCacheFeature extends Feature[TweetCandidate, Boolean] - object IsRetweetFeature extends Feature[TweetCandidate, Boolean] - object IsRetweetedReplyFeature extends Feature[TweetCandidate, Boolean] - object IsSupportAccountReplyFeature extends Feature[TweetCandidate, Boolean] - object LastScoredTimestampMsFeature extends Feature[TweetCandidate, Option[Long]] - object NonSelfFavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object NumImagesFeature extends Feature[TweetCandidate, Option[Int]] - object OriginalTweetCreationTimeFromSnowflakeFeature extends Feature[TweetCandidate, Option[Time]] - object PositionFeature extends Feature[TweetCandidate, Option[Int]] - // Internal id generated per prediction service request - object PredictionRequestIdFeature extends Feature[TweetCandidate, Option[Long]] - object QuotedTweetIdFeature extends Feature[TweetCandidate, Option[Long]] - object QuotedUserIdFeature extends Feature[TweetCandidate, Option[Long]] - object ScoreFeature extends Feature[TweetCandidate, Option[Double]] - object SemanticCoreIdFeature extends Feature[TweetCandidate, Option[Long]] - // Key for kafka logging - object ServedIdFeature extends Feature[TweetCandidate, Option[Long]] - object SimclustersTweetTopKClustersWithScoresFeature - extends Feature[TweetCandidate, Map[String, Double]] - object SocialContextFeature extends Feature[TweetCandidate, Option[tst.SocialContext]] - object SourceTweetIdFeature - extends DataRecordOptionalFeature[TweetCandidate, Long] - with LongDiscreteDataRecordCompatible { - override val featureName: String = TimelinesSharedFeatures.SOURCE_TWEET_ID.getFeatureName - override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.TweetId) - } - object SourceUserIdFeature extends Feature[TweetCandidate, Option[Long]] - object StreamToKafkaFeature extends Feature[TweetCandidate, Boolean] - object SuggestTypeFeature extends Feature[TweetCandidate, Option[st.SuggestType]] - object TSPMetricTagFeature extends Feature[TweetCandidate, Set[tsp.MetricTag]] - object TweetLanguageFeature extends Feature[TweetCandidate, Option[String]] - object TweetUrlsFeature extends Feature[TweetCandidate, Seq[String]] - object VideoDurationMsFeature extends Feature[TweetCandidate, Option[Int]] - object ViewerIdFeature - extends DataRecordFeature[TweetCandidate, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = SharedFeatures.USER_ID.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) - } - object WeightedModelScoreFeature extends Feature[TweetCandidate, Option[Double]] - object MentionUserIdFeature extends Feature[TweetCandidate, Seq[Long]] - object MentionScreenNameFeature extends Feature[TweetCandidate, Seq[String]] - object HasImageFeature extends Feature[TweetCandidate, Boolean] - object HasVideoFeature extends Feature[TweetCandidate, Boolean] - - // Tweetypie VF Features - object IsHydratedFeature extends Feature[TweetCandidate, Boolean] - object IsNsfwFeature extends Feature[TweetCandidate, Boolean] - object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean] - // Raw Tweet Text from Tweetypie - object TweetTextFeature extends Feature[TweetCandidate, Option[String]] - - object AuthorEnabledPreviewsFeature extends Feature[TweetCandidate, Boolean] - object IsTweetPreviewFeature extends Feature[TweetCandidate, Boolean] - - // SGS Features - /** - * By convention, this is set to true for retweets of non-followed authors - * E.g. where somebody the viewer follows retweets a Tweet from somebody the viewer doesn't follow - */ - object InNetworkFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Boolean] { - override val defaultValue: Boolean = true - } - - // Query Features - object AccountAgeFeature extends Feature[PipelineQuery, Option[Time]] - object ClientIdFeature - extends DataRecordOptionalFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = SharedFeatures.CLIENT_ID.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.ClientType) - } - object CachedScoredTweetsFeature extends Feature[PipelineQuery, Seq[hmt.ScoredTweet]] - object DeviceLanguageFeature extends Feature[PipelineQuery, Option[String]] - object DismissInfoFeature - extends FeatureWithDefaultOnFailure[PipelineQuery, Map[st.SuggestType, Option[DismissInfo]]] { - override def defaultValue: Map[st.SuggestType, Option[DismissInfo]] = Map.empty - } - object FollowingLastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] - object GetInitialFeature - extends DataRecordFeature[PipelineQuery, Boolean] - with BoolDataRecordCompatible { - override def featureName: String = RequestContextFeatures.IS_GET_INITIAL.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object GetMiddleFeature - extends DataRecordFeature[PipelineQuery, Boolean] - with BoolDataRecordCompatible { - override def featureName: String = RequestContextFeatures.IS_GET_MIDDLE.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object GetNewerFeature - extends DataRecordFeature[PipelineQuery, Boolean] - with BoolDataRecordCompatible { - override def featureName: String = RequestContextFeatures.IS_GET_NEWER.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object GetOlderFeature - extends DataRecordFeature[PipelineQuery, Boolean] - with BoolDataRecordCompatible { - override def featureName: String = RequestContextFeatures.IS_GET_OLDER.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object GuestIdFeature - extends DataRecordOptionalFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = SharedFeatures.GUEST_ID.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.GuestId) - } - object HasDarkRequestFeature extends Feature[PipelineQuery, Option[Boolean]] - object ImpressionBloomFilterFeature - extends FeatureWithDefaultOnFailure[PipelineQuery, blm.ImpressionBloomFilterSeq] { - override def defaultValue: blm.ImpressionBloomFilterSeq = - blm.ImpressionBloomFilterSeq(Seq.empty) - } - object IsForegroundRequestFeature extends Feature[PipelineQuery, Boolean] - object IsLaunchRequestFeature extends Feature[PipelineQuery, Boolean] - object LastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] - object NonPollingTimesFeature extends Feature[PipelineQuery, Seq[Long]] - object PersistenceEntriesFeature extends Feature[PipelineQuery, Seq[TimelineResponseV3]] - object PollingFeature extends Feature[PipelineQuery, Boolean] - object PullToRefreshFeature extends Feature[PipelineQuery, Boolean] - // Scores from Real Graph representing the relationship between the viewer and another user - object RealGraphInNetworkScoresFeature extends Feature[PipelineQuery, Map[UserId, Double]] - object RequestJoinIdFeature extends Feature[TweetCandidate, Option[Long]] - // Internal id generated per request, mainly to deduplicate re-served cached tweets in logging - object ServedRequestIdFeature extends Feature[PipelineQuery, Option[Long]] - object ServedTweetIdsFeature extends Feature[PipelineQuery, Seq[Long]] - object ServedTweetPreviewIdsFeature extends Feature[PipelineQuery, Seq[Long]] - object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]] - object TimestampFeature - extends DataRecordFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = SharedFeatures.TIMESTAMP.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object TimestampGMTDowFeature - extends DataRecordFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = RequestContextFeatures.TIMESTAMP_GMT_DOW.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object TimestampGMTHourFeature - extends DataRecordFeature[PipelineQuery, Long] - with LongDiscreteDataRecordCompatible { - override def featureName: String = RequestContextFeatures.TIMESTAMP_GMT_HOUR.getFeatureName - override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty - } - object TweetImpressionsFeature extends Feature[PipelineQuery, Seq[imp.TweetImpressionsEntry]] - object UserFollowedTopicsCountFeature extends Feature[PipelineQuery, Option[Int]] - object UserFollowingCountFeature extends Feature[PipelineQuery, Option[Int]] - object UserScreenNameFeature extends Feature[PipelineQuery, Option[String]] - object UserStateFeature extends Feature[PipelineQuery, Option[um.UserState]] - object UserTypeFeature extends Feature[PipelineQuery, Option[gt.UserType]] - object WhoToFollowExcludedUserIdsFeature - extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { - override def defaultValue = Seq.empty - } - - // Result Features - object ServedSizeFeature extends Feature[PipelineQuery, Option[Int]] - object HasRandomTweetFeature extends Feature[PipelineQuery, Boolean] - object IsRandomTweetAboveFeature extends Feature[TweetCandidate, Boolean] - object ServedInConversationModuleFeature extends Feature[TweetCandidate, Boolean] - object ConversationModule2DisplayedTweetsFeature extends Feature[TweetCandidate, Boolean] - object ConversationModuleHasGapFeature extends Feature[TweetCandidate, Boolean] - object SGSValidLikedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object SGSValidFollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object PerspectiveFilteredLikedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] - object ScreenNamesFeature extends Feature[TweetCandidate, Map[Long, String]] - object RealNamesFeature extends Feature[TweetCandidate, Map[Long, String]] - - /** - * Features around the focal Tweet for Tweets which should be rendered in convo modules. - * These are needed in order to render social context above the root tweet in a convo modules. - * For example if we have a convo module A-B-C (A Tweets, B replies to A, C replies to B), the descendant features are - * for the Tweet C. These features are None except for the root Tweet for Tweets which should render into - * convo modules. - */ - object FocalTweetAuthorIdFeature extends Feature[TweetCandidate, Option[Long]] - object FocalTweetInNetworkFeature extends Feature[TweetCandidate, Option[Boolean]] - object FocalTweetRealNamesFeature extends Feature[TweetCandidate, Option[Map[Long, String]]] - object FocalTweetScreenNamesFeature extends Feature[TweetCandidate, Option[Map[Long, String]]] - object MediaUnderstandingAnnotationIdsFeature extends Feature[TweetCandidate, Seq[Long]] -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel deleted file mode 100644 index 3883e454e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "timelineservice/common:model", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.docx new file mode 100644 index 000000000..a25e2fe21 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.docx new file mode 100644 index 000000000..c19689564 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.scala deleted file mode 100644 index ef96865ca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.{timelineservice => tls} - -case class DeviceContext( - isPolling: Option[Boolean], - requestContext: Option[String], - latestControlAvailable: Option[Boolean], - autoplayEnabled: Option[Boolean]) { - - lazy val requestContextValue: Option[DeviceContext.RequestContext.Value] = - requestContext.flatMap { value => - val normalizedValue = value.trim.toLowerCase() - DeviceContext.RequestContext.values.find(_.toString == normalizedValue) - } - - def toTimelineServiceDeviceContext(clientContext: ClientContext): tls.DeviceContext = - tls.DeviceContext( - countryCode = clientContext.countryCode, - languageCode = clientContext.languageCode, - clientAppId = clientContext.appId, - ipAddress = clientContext.ipAddress, - guestId = clientContext.guestId, - sessionId = None, - timezone = None, - userAgent = clientContext.userAgent, - deviceId = clientContext.deviceId, - isPolling = isPolling, - requestProvenance = requestContext, - referrer = None, - tfeAuthHeader = None, - mobileDeviceId = clientContext.mobileDeviceId, - isSessionStart = None, - displaySize = None, - isURTRequest = Some(true), - latestControlAvailable = latestControlAvailable, - guestIdMarketing = clientContext.guestIdMarketing, - isInternalOrTwoffice = clientContext.isTwoffice, - browserNotificationPermission = None, - guestIdAds = clientContext.guestIdAds, - ) -} - -object DeviceContext { - val Empty: DeviceContext = DeviceContext( - isPolling = None, - requestContext = None, - latestControlAvailable = None, - autoplayEnabled = None - ) - - /** - * Constants which reflect valid client request provenances (why a request was initiated, encoded - * by the "request_context" HTTP parameter). - */ - object RequestContext extends Enumeration { - val Auto = Value("auto") - val Foreground = Value("foreground") - val Gap = Value("gap") - val Launch = Value("launch") - val ManualRefresh = Value("manual_refresh") - val Navigate = Value("navigate") - val Polling = Value("polling") - val PullToRefresh = Value("ptr") - val Signup = Value("signup") - val TweetSelfThread = Value("tweet_self_thread") - val BackgroundFetch = Value("background_fetch") - } -} - -trait HasDeviceContext { - def deviceContext: Option[DeviceContext] -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.docx new file mode 100644 index 000000000..2d2aa41ae Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.scala deleted file mode 100644 index ece09a0d4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.home_mixer.model.request - -/** - * [[HasListId]] enables shared components to access the list id shared by all list timeline products. - */ -trait HasListId { - def listId: Long -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.docx new file mode 100644 index 000000000..985fdd63a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.scala deleted file mode 100644 index 0ec657384..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.home_mixer.model.request - -/** - * [[HasSeenTweetIds]] enables shared components to access the list of impressed tweet IDs - * sent by clients across different Home Mixer query types (e.g. FollowingQuery, ForYouQuery) - */ -trait HasSeenTweetIds { - def seenTweetIds: Option[Seq[Long]] -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.docx new file mode 100644 index 000000000..556cc57a9 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala deleted file mode 100644 index 818380946..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.product_mixer.core.model.marshalling.request.DebugOptions -import com.twitter.util.Time - -case class HomeMixerDebugOptions( - override val requestTimeOverride: Option[Time]) - extends DebugOptions diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.docx new file mode 100644 index 000000000..869680e29 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala deleted file mode 100644 index 7c27c50d5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product - -/** - * Identifier names on products can be used to create Feature Switch rules by product, - * which useful if bucketing occurs in a component shared by multiple products. - * @see [[Product.identifier]] - */ - -case object FollowingProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("Following") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object ForYouProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ForYou") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object ScoredTweetsProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ScoredTweets") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object ListTweetsProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ListTweets") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object ListRecommendedUsersProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("ListRecommendedUsers") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} - -case object SubscribedProduct extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("Subscribed") - override val stringCenterProject: Option[String] = Some("timelinemixer") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.docx new file mode 100644 index 000000000..124945e3d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala deleted file mode 100644 index dddd733f3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.dspbidder.commons.thriftscala.DspClientContext -import com.twitter.product_mixer.core.model.marshalling.request.ProductContext - -case class FollowingProductContext( - deviceContext: Option[DeviceContext], - seenTweetIds: Option[Seq[Long]], - dspClientContext: Option[DspClientContext]) - extends ProductContext - -case class ForYouProductContext( - deviceContext: Option[DeviceContext], - seenTweetIds: Option[Seq[Long]], - dspClientContext: Option[DspClientContext], - pushToHomeTweetId: Option[Long]) - extends ProductContext - -case class ScoredTweetsProductContext( - deviceContext: Option[DeviceContext], - seenTweetIds: Option[Seq[Long]], - servedTweetIds: Option[Seq[Long]], - backfillTweetIds: Option[Seq[Long]]) - extends ProductContext - -case class ListTweetsProductContext( - listId: Long, - deviceContext: Option[DeviceContext], - dspClientContext: Option[DspClientContext]) - extends ProductContext - -case class ListRecommendedUsersProductContext( - listId: Long, - selectedUserIds: Option[Seq[Long]], - excludedUserIds: Option[Seq[Long]], - listName: Option[String]) - extends ProductContext - -case class SubscribedProductContext( - deviceContext: Option[DeviceContext], - seenTweetIds: Option[Seq[Long]]) - extends ProductContext diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.docx new file mode 100644 index 000000000..909bbafca Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.scala deleted file mode 100644 index 34595c34c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.home_mixer.model.request - -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.DebugParams -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.request.ProductContext -import com.twitter.product_mixer.core.model.marshalling.request.Request - -case class HomeMixerRequest( - override val clientContext: ClientContext, - override val product: Product, - // Product-specific parameters should be placed in the Product Context - override val productContext: Option[ProductContext], - override val serializedRequestCursor: Option[String], - override val maxResults: Option[Int], - override val debugParams: Option[DebugParams], - // Parameters that apply to all products can be promoted to the request-level - homeRequestParam: Boolean) - extends Request diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.docx new file mode 100644 index 000000000..d00124922 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.scala deleted file mode 100644 index d095d2054..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides - -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.ManhattanCluster -import com.twitter.storehaus_internal.manhattan.ManhattanClusters -import com.twitter.timelines.clients.ads.AdvertiserBrandSafetySettingsStore -import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder -import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientConfigWithDataset -import com.twitter.util.Duration - -import javax.inject.Singleton - -object AdvertiserBrandSafetySettingsStoreModule extends TwitterModule { - - @Provides - @Singleton - def providesAdvertiserBrandSafetySettingsStore( - injectedServiceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): ReadableStore[Long, ads.AdvertiserBrandSafetySettings] = { - val advertiserBrandSafetySettingsManhattanClientConfig = new ManhattanClientConfigWithDataset { - override val cluster: ManhattanCluster = ManhattanClusters.apollo - override val appId: String = "brand_safety_apollo" - override val dataset = "advertiser_brand_safety_settings" - override val statsScope: String = "AdvertiserBrandSafetySettingsManhattanClient" - override val defaultGuarantee = Guarantee.Weak - override val defaultMaxTimeout: Duration = 100.milliseconds - override val maxRetryCount: Int = 1 - override val isReadOnly: Boolean = true - override val serviceIdentifier: ServiceIdentifier = injectedServiceIdentifier - } - - val advertiserBrandSafetySettingsManhattanEndpoint = ManhattanClientBuilder - .buildManhattanEndpoint(advertiserBrandSafetySettingsManhattanClientConfig, statsReceiver) - - val advertiserBrandSafetySettingsStore: ReadableStore[Long, ads.AdvertiserBrandSafetySettings] = - AdvertiserBrandSafetySettingsStore - .cached( - advertiserBrandSafetySettingsManhattanEndpoint, - advertiserBrandSafetySettingsManhattanClientConfig.dataset, - ttl = 60.minutes, - maxKeys = 100000, - windowSize = 10L - )(statsReceiver) - - advertiserBrandSafetySettingsStore - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel deleted file mode 100644 index b7fbca0d3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel +++ /dev/null @@ -1,90 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/bijection:thrift", - "3rdparty/jvm/com/twitter/src/java/com/twitter/logpipeline/client:logpipeline-event-publisher-thin", - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/io/netty:netty4-tcnative-boringssl-static", - "eventbus/client/src/main/scala/com/twitter/eventbus/client", - "finagle-internal/finagle-grpc/src/main/scala", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client", - "finagle/finagle-core/src/main", - "finagle/finagle-memcached/src/main/scala", - "finagle/finagle-mux/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/store", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "interests-service/thrift/src/main/thrift:thrift-scala", - "people-discovery/api/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client", - "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client", - "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client", - "servo/client/src/main/scala/com/twitter/servo/client", - "servo/manhattan", - "servo/util", - "socialgraph/server/src/main/scala/com/twitter/socialgraph/util", - "src/scala/com/twitter/ml/featurestore/lib", - "src/scala/com/twitter/scalding_internal/multiformat/format", - "src/scala/com/twitter/storehaus_internal", - "src/scala/com/twitter/summingbird_internal/bijection:bijection-implicits", - "src/scala/com/twitter/timelines/util", - "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", - "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "src/thrift/com/twitter/manhattan:v1-scala", - "src/thrift/com/twitter/manhattan:v2-scala", - "src/thrift/com/twitter/onboarding/relevance/features:features-java", - "src/thrift/com/twitter/search:blender-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/service/metastore/gen:thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", - "src/thrift/com/twitter/timelines/impression_store:thrift-scala", - "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/topic_recos:topic_recos-thrift-java", - "src/thrift/com/twitter/user_session_store:thrift-java", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "stitch/stitch-socialgraph", - "stitch/stitch-tweetypie", - "strato/src/main/scala/com/twitter/strato/client", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", - "timelines:decider", - "timelines/src/main/scala/com/twitter/timelines/clients/ads", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", - "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", - "timelines/src/main/scala/com/twitter/timelines/clients/strato", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", - "timelines/src/main/scala/com/twitter/timelines/impressionstore/store", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "tweetconvosvc/client/src/main/scala/com/twitter/tweetconvosvc/client/builder", - "twitter-config/yaml", - ], - exports = [ - "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.docx new file mode 100644 index 000000000..8b0af304c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.docx new file mode 100644 index 000000000..87275db39 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala deleted file mode 100644 index bc4045181..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder -import com.twitter.product_mixer.shared_library.thrift_client.NonIdempotent -import com.twitter.search.blender.thriftscala.BlenderService -import javax.inject.Singleton - -object BlenderClientModule extends TwitterModule { - - @Singleton - @Provides - def providesBlenderClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): BlenderService.MethodPerEndpoint = { - val clientId = serviceIdentifier.environment.toLowerCase match { - case "prod" => ClientId("") - case _ => ClientId("") - } - - FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ - BlenderService.ServicePerEndpoint, - BlenderService.MethodPerEndpoint - ]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/blender-universal/blender", - label = "blender", - statsReceiver = statsReceiver, - idempotency = NonIdempotent, - timeoutPerRequest = 1000.milliseconds, - timeoutTotal = 1000.milliseconds, - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.docx new file mode 100644 index 000000000..93d623932 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.scala deleted file mode 100644 index f7c5e9bc7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.eventbus.client.EventBusPublisher -import com.twitter.eventbus.client.EventBusPublisherBuilder -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.timelines.config.ConfigUtils -import com.twitter.timelines.config.Env -import com.twitter.timelines.impressionstore.thriftscala.PublishedImpressionList -import javax.inject.Singleton - -object ClientSentImpressionsPublisherModule extends TwitterModule with ConfigUtils { - private val serviceName = "home-mixer" - - @Singleton - @Provides - def providesClientSentImpressionsPublisher( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): EventBusPublisher[PublishedImpressionList] = { - val env = serviceIdentifier.environment.toLowerCase match { - case "prod" => Env.prod - case "staging" => Env.staging - case "local" => Env.local - case _ => Env.devel - } - - val streamName = env match { - case Env.prod => "timelinemixer_client_sent_impressions_prod" - case _ => "timelinemixer_client_sent_impressions_devel" - } - - EventBusPublisherBuilder() - .clientId(clientIdWithScopeOpt(serviceName, env)) - .serviceIdentifier(serviceIdentifier) - .streamName(streamName) - .statsReceiver(statsReceiver.scope("eventbus")) - .thriftStruct(PublishedImpressionList) - .tcpConnectTimeout(20.milliseconds) - .connectTimeout(100.milliseconds) - .requestTimeout(1.second) - .publishTimeout(1.second) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.docx new file mode 100644 index 000000000..a7d327b84 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.scala deleted file mode 100644 index 9a7d8771c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.tweetconvosvc.thriftscala.ConversationService -import com.twitter.util.Duration -import org.apache.thrift.protocol.TCompactProtocol - -object ConversationServiceModule - extends ThriftMethodBuilderClientModule[ - ConversationService.ServicePerEndpoint, - ConversationService.MethodPerEndpoint - ] - with MtlsClient { - - override val label: String = "tweetconvosvc" - override val dest: String = "/s/tweetconvosvc/tweetconvosvc" - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = methodBuilder.withTimeoutPerRequest(100.milliseconds) - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = - super - .configureThriftMuxClient(injector, client) - .withProtocolFactory(new TCompactProtocol.Factory()) - - override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.docx new file mode 100644 index 000000000..6df4bfb43 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala deleted file mode 100644 index c065f7b35..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClient -import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClientConfig -import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder -import com.twitter.util.Duration -import javax.inject.Singleton - -object FeedbackHistoryClientModule extends TwitterModule { - private val ProdDataset = "feedback_history" - private val StagingDataset = "feedback_history_nonprod" - private final val Timeout = "mh_feedback_history.timeout" - - flag[Duration](Timeout, 150.millis, "Timeout per request") - - @Provides - @Singleton - def providesFeedbackHistoryClient( - @Flag(Timeout) timeout: Duration, - serviceId: ServiceIdentifier, - statsReceiver: StatsReceiver - ) = { - val manhattanDataset = serviceId.environment.toLowerCase match { - case "prod" => ProdDataset - case _ => StagingDataset - } - - val config = new FeedbackHistoryManhattanClientConfig { - val dataset = manhattanDataset - val isReadOnly = true - val serviceIdentifier = serviceId - override val defaultMaxTimeout = timeout - } - - new FeedbackHistoryManhattanClient( - ManhattanClientBuilder.buildManhattanEndpoint(config, statsReceiver), - manhattanDataset, - statsReceiver - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.docx new file mode 100644 index 000000000..a9aa6d917 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.scala deleted file mode 100644 index 73e1500a0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.adserver.thriftscala.NewAdServer -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.util.Duration - -object HomeAdsCandidateSourceModule - extends ThriftMethodBuilderClientModule[ - NewAdServer.ServicePerEndpoint, - NewAdServer.MethodPerEndpoint - ] - with MtlsClient { - - override val label = "adserver" - override val dest = "/s/ads/adserver" - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = { - methodBuilder - .withTimeoutPerRequest(1200.milliseconds) - .withTimeoutTotal(1200.milliseconds) - .withMaxRetries(2) - } - - override protected def sessionAcquisitionTimeout: Duration = 150.milliseconds -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.docx new file mode 100644 index 000000000..bb3a4fd73 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala deleted file mode 100644 index e31d2d9bc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.conversions.DurationOps.RichDuration -import com.twitter.home_mixer.param.HomeMixerFlagName -import com.twitter.inject.TwitterModule -import com.twitter.util.Duration - -object HomeMixerFlagsModule extends TwitterModule { - - import HomeMixerFlagName._ - - flag[Boolean]( - name = ScribeClientEventsFlag, - default = false, - help = "Toggles logging client events to Scribe" - ) - - flag[Boolean]( - name = ScribeServedCandidatesFlag, - default = false, - help = "Toggles logging served candidates to Scribe" - ) - - flag[Boolean]( - name = ScribeScoredCandidatesFlag, - default = false, - help = "Toggles logging scored candidates to Scribe" - ) - - flag[Boolean]( - name = ScribeServedCommonFeaturesAndCandidateFeaturesFlag, - default = false, - help = "Toggles logging served common features and candidates features to Scribe" - ) - - flag[String]( - name = DataRecordMetadataStoreConfigsYmlFlag, - default = "", - help = "The YML file that contains the necessary info for creating metadata store MySQL client." - ) - - flag[String]( - name = DarkTrafficFilterDeciderKey, - default = "dark_traffic_filter", - help = "Dark traffic filter decider key" - ) - - flag[Duration]( - TargetFetchLatency, - 300.millis, - "Target fetch latency from candidate sources for Quality Factor" - ) - - flag[Duration]( - TargetScoringLatency, - 700.millis, - "Target scoring latency for Quality Factor" - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.docx new file mode 100644 index 000000000..846a336bc Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala deleted file mode 100644 index b68e5b105..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.inject.TwitterModule - -object HomeMixerResourcesModule extends TwitterModule {} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.docx new file mode 100644 index 000000000..d7bb3cf84 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala deleted file mode 100644 index f37531483..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storehaus_internal.manhattan.ManhattanClusters -import com.twitter.timelines.clients.manhattan.store._ -import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} -import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilterManhattanKeyValueDescriptor -import com.twitter.util.Duration -import javax.inject.Singleton - -object ImpressionBloomFilterModule extends TwitterModule { - - private val ProdAppId = "impression_bloom_filter_store" - private val ProdDataset = "impression_bloom_filter" - private val StagingAppId = "impression_bloom_filter_store_staging" - private val StagingDataset = "impression_bloom_filter_staging" - private val ClientStatsScope = "tweetBloomFilterImpressionManhattanClient" - private val DefaultTTL = 7.days - private final val Timeout = "mh_impression_store_bloom_filter.timeout" - - flag[Duration](Timeout, 150.millis, "Timeout per request") - - @Provides - @Singleton - def providesImpressionBloomFilter( - @Flag(Timeout) timeout: Duration, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): ManhattanStoreClient[blm.ImpressionBloomFilterKey, blm.ImpressionBloomFilterSeq] = { - val (appId, dataset) = serviceIdentifier.environment.toLowerCase match { - case "prod" => (ProdAppId, ProdDataset) - case _ => (StagingAppId, StagingDataset) - } - - implicit val manhattanKeyValueDescriptor: ImpressionBloomFilterManhattanKeyValueDescriptor = - ImpressionBloomFilterManhattanKeyValueDescriptor( - dataset = dataset, - ttl = DefaultTTL - ) - - ManhattanStoreClientBuilder.buildManhattanClient( - serviceIdentifier = serviceIdentifier, - cluster = ManhattanClusters.nash, - appId = appId, - defaultMaxTimeout = timeout, - maxRetryCount = 2, - defaultGuarantee = Some(Guarantee.SoftDcReadMyWrites), - isReadOnly = false, - statsScope = ClientStatsScope, - statsReceiver = statsReceiver - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.docx new file mode 100644 index 000000000..18b113ba0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.scala deleted file mode 100644 index fe274ff1d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.builder.ClientBuilder -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.ssl.OpportunisticTls -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.manhattan.v2.thriftscala.{ManhattanCoordinator => ManhattanV2} -import com.twitter.timelinemixer.clients.manhattan.InjectionHistoryClient -import com.twitter.timelinemixer.clients.manhattan.ManhattanDatasetConfig -import com.twitter.timelines.clients.manhattan.Dataset -import com.twitter.timelines.clients.manhattan.ManhattanClient -import com.twitter.timelines.util.stats.RequestScope -import javax.inject.Singleton -import org.apache.thrift.protocol.TBinaryProtocol -import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration.ConnectTimeout -import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration.TCPConnectTimeout - -object InjectionHistoryClientModule extends TwitterModule { - private val ProdDataset = "suggestion_history" - private val StagingDataset = "suggestion_history_nonprod" - private val AppId = "twitter_suggests" - private val ServiceName = "manhattan.omega" - private val OmegaManhattanDest = "/s/manhattan/omega.native-thrift" - private val InjectionRequestScope = RequestScope("injectionHistoryClient") - private val RequestTimeout = 75.millis - private val Timeout = 150.millis - - val retryPolicy = RetryPolicy.tries( - 2, - RetryPolicy.TimeoutAndWriteExceptionsOnly - .orElse(RetryPolicy.ChannelClosedExceptionsOnly)) - - @Provides - @Singleton - def providesInjectionHistoryClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ) = { - val dataset = serviceIdentifier.environment.toLowerCase match { - case "prod" => ProdDataset - case _ => StagingDataset - } - - val thriftMuxClient = ClientBuilder() - .name(ServiceName) - .daemon(daemonize = true) - .failFast(enabled = true) - .retryPolicy(retryPolicy) - .tcpConnectTimeout(TCPConnectTimeout) - .connectTimeout(ConnectTimeout) - .dest(OmegaManhattanDest) - .requestTimeout(RequestTimeout) - .timeout(Timeout) - .stack(ThriftMux.client - .withMutualTls(serviceIdentifier) - .withOpportunisticTls(OpportunisticTls.Required)) - .build() - - val manhattanOmegaClient = new ManhattanV2.FinagledClient( - service = thriftMuxClient, - protocolFactory = new TBinaryProtocol.Factory(), - serviceName = ServiceName, - ) - - val readOnlyMhClient = new ManhattanClient( - appId = AppId, - manhattan = manhattanOmegaClient, - requestScope = InjectionRequestScope, - serviceName = ServiceName, - statsReceiver = statsReceiver - ).readOnly - - val mhDatasetConfig = new ManhattanDatasetConfig { - override val SuggestionHistoryDataset = Dataset(dataset) - } - - new InjectionHistoryClient( - readOnlyMhClient, - mhDatasetConfig - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.docx new file mode 100644 index 000000000..19227bdaf Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala deleted file mode 100644 index fc0e282af..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphManhattanEndpoint -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.storage.client.manhattan.kv._ -import com.twitter.timelines.config.ConfigUtils -import com.twitter.util.Duration -import javax.inject.Named -import javax.inject.Singleton - -object ManhattanClientsModule extends TwitterModule with ConfigUtils { - - private val ApolloDest = "/s/manhattan/apollo.native-thrift" - private final val Timeout = "mh_real_graph.timeout" - - flag[Duration](Timeout, 150.millis, "Timeout total") - - @Provides - @Singleton - @Named(RealGraphManhattanEndpoint) - def providesRealGraphManhattanEndpoint( - @Flag(Timeout) timeout: Duration, - serviceIdentifier: ServiceIdentifier - ): ManhattanKVEndpoint = { - lazy val client = ManhattanKVClient( - appId = "real_graph", - dest = ApolloDest, - mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier = serviceIdentifier), - label = "real-graph-data" - ) - - ManhattanKVEndpointBuilder(client) - .maxRetryCount(2) - .defaultMaxTimeout(timeout) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.docx new file mode 100644 index 000000000..f24aebf74 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala deleted file mode 100644 index 5668ba0ee..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala +++ /dev/null @@ -1,468 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.scrooge.CompactScalaCodec -import com.twitter.bijection.thrift.ThriftCodec -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.home_mixer.param.HomeMixerInjectionNames._ -import com.twitter.home_mixer.util.InjectionTransformerImplicits._ -import com.twitter.home_mixer.util.LanguageUtil -import com.twitter.home_mixer.util.TensorFlowUtil -import com.twitter.inject.TwitterModule -import com.twitter.manhattan.v1.{thriftscala => mh} -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.{thriftscala => fs} -import com.twitter.onboarding.relevance.features.{thriftjava => rf} -import com.twitter.product_mixer.shared_library.manhattan_client.ManhattanClientBuilder -import com.twitter.scalding_internal.multiformat.format.keyval.KeyValInjection.ScalaBinaryThrift -import com.twitter.search.common.constants.{thriftscala => scc} -import com.twitter.service.metastore.gen.{thriftscala => smg} -import com.twitter.servo.cache._ -import com.twitter.servo.manhattan.ManhattanKeyValueRepository -import com.twitter.servo.repository.CachingKeyValueRepository -import com.twitter.servo.repository.ChunkingStrategy -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.servo.repository.Repository -import com.twitter.servo.repository.keysAsQuery -import com.twitter.servo.util.Transformer -import com.twitter.storage.client.manhattan.bijections.Bijections -import com.twitter.storehaus_internal.manhattan.ManhattanClusters -import com.twitter.timelines.author_features.v1.{thriftjava => af} -import com.twitter.timelines.suggests.common.dense_data_record.{thriftscala => ddr} -import com.twitter.user_session_store.{thriftscala => uss_scala} -import com.twitter.user_session_store.{thriftjava => uss} -import com.twitter.util.Duration -import com.twitter.util.Try -import java.nio.ByteBuffer -import javax.inject.Named -import javax.inject.Singleton -import org.apache.thrift.protocol.TCompactProtocol -import org.apache.thrift.transport.TMemoryInputTransport -import org.apache.thrift.transport.TTransport - -object ManhattanFeatureRepositoryModule extends TwitterModule { - - private val DEFAULT_RPC_CHUNK_SIZE = 50 - - private val ThriftEntityIdInjection = ScalaBinaryThrift(fs.EntityId) - - private val FeatureStoreUserIdKeyTransformer = new Transformer[Long, ByteBuffer] { - override def to(userId: Long): Try[ByteBuffer] = { - Try(ByteBuffer.wrap(ThriftEntityIdInjection.apply(UserId(userId).toThrift))) - } - override def from(b: ByteBuffer): Try[Long] = ??? - } - - private val FloatTensorTransformer = new Transformer[ByteBuffer, ml.FloatTensor] { - override def to(input: ByteBuffer): Try[ml.FloatTensor] = { - val floatTensor = TensorFlowUtil.embeddingByteBufferToFloatTensor(input) - Try(floatTensor) - } - - override def from(b: ml.FloatTensor): Try[ByteBuffer] = ??? - } - - private val LanguageTransformer = new Transformer[ByteBuffer, Seq[scc.ThriftLanguage]] { - override def to(input: ByteBuffer): Try[Seq[scc.ThriftLanguage]] = { - Try.fromScala( - Bijections - .BinaryScalaInjection(smg.UserLanguages) - .andThen(Bijections.byteBuffer2Buf.inverse) - .invert(input).map(LanguageUtil.computeLanguages(_))) - } - - override def from(b: Seq[scc.ThriftLanguage]): Try[ByteBuffer] = ??? - } - - private val LongKeyTransformer = Injection - .connect[Long, Array[Byte]] - .toByteBufferTransformer() - - // manhattan clients - - @Provides - @Singleton - @Named(ManhattanApolloClient) - def providesManhattanApolloClient( - serviceIdentifier: ServiceIdentifier - ): mh.ManhattanCoordinator.MethodPerEndpoint = { - ManhattanClientBuilder - .buildManhattanV1FinagleClient( - ManhattanClusters.apollo, - serviceIdentifier - ) - } - - @Provides - @Singleton - @Named(ManhattanAthenaClient) - def providesManhattanAthenaClient( - serviceIdentifier: ServiceIdentifier - ): mh.ManhattanCoordinator.MethodPerEndpoint = { - ManhattanClientBuilder - .buildManhattanV1FinagleClient( - ManhattanClusters.athena, - serviceIdentifier - ) - } - - @Provides - @Singleton - @Named(ManhattanOmegaClient) - def providesManhattanOmegaClient( - serviceIdentifier: ServiceIdentifier - ): mh.ManhattanCoordinator.MethodPerEndpoint = { - ManhattanClientBuilder - .buildManhattanV1FinagleClient( - ManhattanClusters.omega, - serviceIdentifier - ) - } - - @Provides - @Singleton - @Named(ManhattanStarbuckClient) - def providesManhattanStarbuckClient( - serviceIdentifier: ServiceIdentifier - ): mh.ManhattanCoordinator.MethodPerEndpoint = { - ManhattanClientBuilder - .buildManhattanV1FinagleClient( - ManhattanClusters.starbuck, - serviceIdentifier - ) - } - - // non-cached manhattan repositories - - @Provides - @Singleton - @Named(MetricCenterUserCountingFeatureRepository) - def providesMetricCenterUserCountingFeatureRepository( - @Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, rf.MCUserCountingFeatures] = { - - val valueTransformer = ThriftCodec - .toBinary[rf.MCUserCountingFeatures] - .toByteBufferTransformer() - .flip - - batchedManhattanKeyValueRepository[Long, rf.MCUserCountingFeatures]( - client = client, - keyTransformer = LongKeyTransformer, - valueTransformer = valueTransformer, - appId = "wtf_ml", - dataset = "mc_user_counting_features_v0_starbuck", - timeoutInMillis = 100 - ) - } - - /** - * A repository of the offline aggregate feature metadata necessary to decode - * DenseCompactDataRecords. - * - * This repository is expected to virtually always pick up the metadata form the local cache with - * nearly 0 latency. - */ - @Provides - @Singleton - @Named(TimelineAggregateMetadataRepository) - def providesTimelineAggregateMetadataRepository( - @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): Repository[Int, Option[ddr.DenseFeatureMetadata]] = { - - val keyTransformer = Injection - .connect[Int, Array[Byte]] - .toByteBufferTransformer() - - val valueTransformer = new Transformer[ByteBuffer, ddr.DenseFeatureMetadata] { - private val compactProtocolFactory = new TCompactProtocol.Factory - - def to(buffer: ByteBuffer): Try[ddr.DenseFeatureMetadata] = Try { - val transport = transportFromByteBuffer(buffer) - ddr.DenseFeatureMetadata.decode(compactProtocolFactory.getProtocol(transport)) - } - - // Encoding intentionally not implemented as it is never used - def from(metadata: ddr.DenseFeatureMetadata): Try[ByteBuffer] = ??? - } - - val inProcessCache: Cache[Int, Cached[ddr.DenseFeatureMetadata]] = InProcessLruCacheFactory( - ttl = Duration.fromMinutes(20), - lruSize = 30 - ).apply(serializer = Transformer(_ => ???, _ => ???)) // Serialization is not necessary here. - - val keyValueRepository = new ManhattanKeyValueRepository( - client = client, - keyTransformer = keyTransformer, - valueTransformer = valueTransformer, - appId = "timelines_dense_aggregates_encoding_metadata", // Expected QPS is negligible. - dataset = "user_session_dense_feature_metadata", - timeoutInMillis = 100 - ) - - KeyValueRepository - .singular( - new CachingKeyValueRepository[Seq[Int], Int, ddr.DenseFeatureMetadata]( - keyValueRepository, - new NonLockingCache(inProcessCache), - keysAsQuery[Int] - ) - ) - } - - @Provides - @Singleton - @Named(RealGraphFeatureRepository) - def providesRealGraphFeatureRepository( - @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): Repository[Long, Option[uss_scala.UserSession]] = { - val valueTransformer = CompactScalaCodec(uss_scala.UserSession).toByteBufferTransformer().flip - - KeyValueRepository.singular( - new ManhattanKeyValueRepository( - client = client, - keyTransformer = LongKeyTransformer, - valueTransformer = valueTransformer, - appId = "real_graph", - dataset = "split_real_graph_features", - timeoutInMillis = 100, - ) - ) - } - - // cached manhattan repositories - - @Provides - @Singleton - @Named(AuthorFeatureRepository) - def providesAuthorFeatureRepository( - @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, - @Named(HomeAuthorFeaturesCacheClient) cacheClient: Memcache - ): KeyValueRepository[Seq[Long], Long, af.AuthorFeatures] = { - - val valueInjection = ThriftCodec - .toCompact[af.AuthorFeatures] - - val keyValueRepository = batchedManhattanKeyValueRepository( - client = client, - keyTransformer = LongKeyTransformer, - valueTransformer = valueInjection.toByteBufferTransformer().flip, - appId = "timelines_author_feature_store_athena", - dataset = "timelines_author_features", - timeoutInMillis = 100 - ) - - val remoteCacheRepo = buildMemCachedRepository( - keyValueRepository = keyValueRepository, - cacheClient = cacheClient, - cachePrefix = "AuthorFeatureHydrator", - ttl = 12.hours, - valueInjection = valueInjection) - - buildInProcessCachedRepository( - keyValueRepository = remoteCacheRepo, - ttl = 15.minutes, - size = 8000, - valueInjection = valueInjection - ) - } - - @Provides - @Singleton - @Named(TwhinAuthorFollowFeatureRepository) - def providesTwhinAuthorFollowFeatureRepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, - @Named(TwhinAuthorFollowFeatureCacheClient) cacheClient: Memcache - ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { - val keyValueRepository = - batchedManhattanKeyValueRepository( - client = client, - keyTransformer = FeatureStoreUserIdKeyTransformer, - valueTransformer = FloatTensorTransformer, - appId = "ml_features_apollo", - dataset = "twhin_author_follow_embedding_fsv1__v1_thrift__embedding", - timeoutInMillis = 100 - ) - - val valueInjection: Injection[ml.FloatTensor, Array[Byte]] = - BinaryScalaCodec(ml.FloatTensor) - - buildMemCachedRepository( - keyValueRepository = keyValueRepository, - cacheClient = cacheClient, - cachePrefix = "twhinAuthorFollows", - ttl = 24.hours, - valueInjection = valueInjection - ) - } - - @Provides - @Singleton - @Named(UserLanguagesRepository) - def providesUserLanguagesFeatureRepository( - @Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, Seq[scc.ThriftLanguage]] = { - batchedManhattanKeyValueRepository( - client = client, - keyTransformer = LongKeyTransformer, - valueTransformer = LanguageTransformer, - appId = "user_metadata", - dataset = "languages", - timeoutInMillis = 70 - ) - } - - @Provides - @Singleton - @Named(TwhinUserFollowFeatureRepository) - def providesTwhinUserFollowFeatureRepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { - batchedManhattanKeyValueRepository( - client = client, - keyTransformer = FeatureStoreUserIdKeyTransformer, - valueTransformer = FloatTensorTransformer, - appId = "ml_features_apollo", - dataset = "twhin_user_follow_embedding_fsv1__v1_thrift__embedding", - timeoutInMillis = 100 - ) - } - - @Provides - @Singleton - @Named(TimelineAggregatePartARepository) - def providesTimelineAggregatePartARepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, - ): Repository[Long, Option[uss.UserSession]] = - timelineAggregateRepository( - mhClient = client, - mhDataset = "timelines_aggregates_v2_features_by_user_part_a_apollo", - mhAppId = "timelines_aggregates_v2_features_by_user_part_a_apollo" - ) - - @Provides - @Singleton - @Named(TimelineAggregatePartBRepository) - def providesTimelineAggregatePartBRepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, - ): Repository[Long, Option[uss.UserSession]] = - timelineAggregateRepository( - mhClient = client, - mhDataset = "timelines_aggregates_v2_features_by_user_part_b_apollo", - mhAppId = "timelines_aggregates_v2_features_by_user_part_b_apollo" - ) - - @Provides - @Singleton - @Named(TwhinUserEngagementFeatureRepository) - def providesTwhinUserEngagementFeatureRepository( - @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { - - batchedManhattanKeyValueRepository( - client = client, - keyTransformer = FeatureStoreUserIdKeyTransformer, - valueTransformer = FloatTensorTransformer, - appId = "ml_features_apollo", - dataset = "twhin_user_engagement_embedding_fsv1__v1_thrift__embedding", - timeoutInMillis = 100 - ) - } - - private def buildMemCachedRepository[K, V]( - keyValueRepository: KeyValueRepository[Seq[K], K, V], - cacheClient: Memcache, - cachePrefix: String, - ttl: Duration, - valueInjection: Injection[V, Array[Byte]] - ): CachingKeyValueRepository[Seq[K], K, V] = { - val cachedSerializer = CachedSerializer.binary( - valueInjection.toByteArrayTransformer() - ) - - val cache = MemcacheCacheFactory( - cacheClient, - ttl, - PrefixKeyTransformerFactory(cachePrefix) - )[K, Cached[V]](cachedSerializer) - - new CachingKeyValueRepository( - keyValueRepository, - new NonLockingCache(cache), - keysAsQuery[K] - ) - } - - private def buildInProcessCachedRepository[K, V]( - keyValueRepository: KeyValueRepository[Seq[K], K, V], - ttl: Duration, - size: Int, - valueInjection: Injection[V, Array[Byte]] - ): CachingKeyValueRepository[Seq[K], K, V] = { - val cachedSerializer = CachedSerializer.binary( - valueInjection.toByteArrayTransformer() - ) - - val cache = InProcessLruCacheFactory( - ttl = ttl, - lruSize = size - )[K, Cached[V]](cachedSerializer) - - new CachingKeyValueRepository( - keyValueRepository, - new NonLockingCache(cache), - keysAsQuery[K] - ) - } - - private def batchedManhattanKeyValueRepository[K, V]( - client: mh.ManhattanCoordinator.MethodPerEndpoint, - keyTransformer: Transformer[K, ByteBuffer], - valueTransformer: Transformer[ByteBuffer, V], - appId: String, - dataset: String, - timeoutInMillis: Int, - chunkSize: Int = DEFAULT_RPC_CHUNK_SIZE - ): KeyValueRepository[Seq[K], K, V] = - KeyValueRepository.chunked( - new ManhattanKeyValueRepository( - client = client, - keyTransformer = keyTransformer, - valueTransformer = valueTransformer, - appId = appId, - dataset = dataset, - timeoutInMillis = timeoutInMillis - ), - chunker = ChunkingStrategy.equalSize(chunkSize) - ) - - private def transportFromByteBuffer(buffer: ByteBuffer): TTransport = - new TMemoryInputTransport( - buffer.array(), - buffer.arrayOffset() + buffer.position(), - buffer.remaining()) - - private def timelineAggregateRepository( - mhClient: mh.ManhattanCoordinator.MethodPerEndpoint, - mhDataset: String, - mhAppId: String - ): Repository[Long, Option[uss.UserSession]] = { - val valueInjection = ThriftCodec - .toCompact[uss.UserSession] - - KeyValueRepository.singular( - new ManhattanKeyValueRepository( - client = mhClient, - keyTransformer = LongKeyTransformer, - valueTransformer = valueInjection.toByteBufferTransformer().flip, - appId = mhAppId, - dataset = mhDataset, - timeoutInMillis = 100 - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.docx new file mode 100644 index 000000000..9a8f9ab97 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala deleted file mode 100644 index c6782665a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storehaus_internal.manhattan.ManhattanClusters -import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder -import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClientConfig -import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient -import com.twitter.util.Duration -import javax.inject.Singleton - -object ManhattanTweetImpressionStoreModule extends TwitterModule { - - private val ProdAppId = "timelines_tweet_impression_store_v2" - private val ProdDataset = "timelines_tweet_impressions_v2" - private val StagingAppId = "timelines_tweet_impression_store_staging" - private val StagingDataset = "timelines_tweet_impressions_staging" - private val StatsScope = "manhattanTweetImpressionStoreClient" - private val DefaultTTL = 2.days - private final val Timeout = "mh_impression_store.timeout" - - flag[Duration](Timeout, 150.millis, "Timeout per request") - - @Provides - @Singleton - def providesManhattanTweetImpressionStoreClient( - @Flag(Timeout) timeout: Duration, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): ManhattanTweetImpressionStoreClient = { - - val (appId, dataset) = serviceIdentifier.environment.toLowerCase match { - case "prod" => (ProdAppId, ProdDataset) - case _ => (StagingAppId, StagingDataset) - } - - val config = ManhattanTweetImpressionStoreClientConfig( - cluster = ManhattanClusters.nash, - appId = appId, - dataset = dataset, - statsScope = StatsScope, - defaultGuarantee = Guarantee.SoftDcReadMyWrites, - defaultMaxTimeout = timeout, - maxRetryCount = 2, - isReadOnly = false, - serviceIdentifier = serviceIdentifier, - ttl = DefaultTTL - ) - - val manhattanEndpoint = ManhattanClientBuilder.buildManhattanEndpoint(config, statsReceiver) - ManhattanTweetImpressionStoreClient(config, manhattanEndpoint, statsReceiver) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.docx new file mode 100644 index 000000000..bce70d127 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala deleted file mode 100644 index 8afafbfb7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala +++ /dev/null @@ -1,117 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.Memcached -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.HomeAuthorFeaturesCacheClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelinesRealTimeAggregateClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureCacheClient -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import com.twitter.servo.cache.FinagleMemcacheFactory -import com.twitter.servo.cache.Memcache -import javax.inject.Named -import javax.inject.Singleton - -object MemcachedFeatureRepositoryModule extends TwitterModule { - - // This must match the respective parameter on the write path. Note that servo sets a different - // hasher by default. See [[com.twitter.hashing.KeyHasher]] for the list of other available - // hashers. - private val memcacheKeyHasher = "ketama" - - @Provides - @Singleton - @Named(TimelinesRealTimeAggregateClient) - def providesTimelinesRealTimeAggregateClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Memcache = { - val rawClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 3, - numConnections = 1, - requestTimeout = 100.milliseconds, - globalTimeout = 300.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - buildMemcacheClient(rawClient, "/s/cache/timelines_real_time_aggregates:twemcaches") - } - - @Provides - @Singleton - @Named(HomeAuthorFeaturesCacheClient) - def providesHomeAuthorFeaturesCacheClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Memcache = { - val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 2, - numConnections = 1, - requestTimeout = 150.milliseconds, - globalTimeout = 300.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - buildMemcacheClient(cacheClient, "/s/cache/timelines_author_features:twemcaches") - } - - @Provides - @Singleton - @Named(TwhinAuthorFollowFeatureCacheClient) - def providesTwhinAuthorFollowFeatureCacheClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Memcache = { - val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 2, - numConnections = 1, - requestTimeout = 150.milliseconds, - globalTimeout = 300.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - buildMemcacheClient(cacheClient, "/s/cache/home_twhin_author_features:twemcaches") - } - - @Provides - @Singleton - @Named(RealTimeInteractionGraphUserVertexClient) - def providesRealTimeInteractionGraphUserVertexClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Memcache = { - val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 2, - numConnections = 1, - requestTimeout = 150.milliseconds, - globalTimeout = 300.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - buildMemcacheClient(cacheClient, "/s/cache/realtime_interactive_graph_prod_v2:twemcaches") - } - - private def buildMemcacheClient(cacheClient: Memcached.Client, dest: String): Memcache = - FinagleMemcacheFactory( - client = cacheClient, - dest = dest, - hashName = memcacheKeyHasher - )() - -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.docx new file mode 100644 index 000000000..f7147ffea Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala deleted file mode 100644 index 60d580a73..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.Http -import com.twitter.finagle.grpc.FinagleChannelBuilder -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsStackClientSyntax -import com.twitter.inject.TwitterModule -import com.twitter.timelines.clients.predictionservice.PredictionGRPCService -import com.twitter.util.Duration -import io.grpc.ManagedChannel -import javax.inject.Singleton - -object NaviModelClientModule extends TwitterModule { - - @Singleton - @Provides - def providesPredictionGRPCService( - serviceIdentifier: ServiceIdentifier, - ): PredictionGRPCService = { - // Wily path to the ML Model service (e.g. /s/ml-serving/navi-explore-ranker). - val modelPath = "/s/ml-serving/navi_home_recap_onnx" - - val MaxPredictionTimeoutMs: Duration = 500.millis - val ConnectTimeoutMs: Duration = 200.millis - val AcquisitionTimeoutMs: Duration = 500.millis - val MaxRetryAttempts: Int = 2 - - val client = Http.client - .withLabel(modelPath) - .withMutualTls(serviceIdentifier) - .withRequestTimeout(MaxPredictionTimeoutMs) - .withTransport.connectTimeout(ConnectTimeoutMs) - .withSession.acquisitionTimeout(AcquisitionTimeoutMs) - .withHttpStats - - val channel: ManagedChannel = FinagleChannelBuilder - .forTarget(modelPath) - .overrideAuthority("rustserving") - .maxRetryAttempts(MaxRetryAttempts) - .enableRetryForStatus(io.grpc.Status.RESOURCE_EXHAUSTED) - .enableRetryForStatus(io.grpc.Status.UNKNOWN) - .enableUnsafeFullyBufferingMode() - .httpClient(client) - .build() - - new PredictionGRPCService(channel) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.docx new file mode 100644 index 000000000..945304509 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala deleted file mode 100644 index b9e315acc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.service.Retries -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.ssl.OpportunisticTls -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout -import com.twitter.inject.TwitterModule -import com.twitter.strato.client.Client -import com.twitter.strato.client.Strato -import com.twitter.util.Try -import javax.inject.Named -import javax.inject.Singleton - -object OptimizedStratoClientModule extends TwitterModule { - - private val ModerateStratoServerClientRequestTimeout = 500.millis - - private val DefaultRetryPartialFunction: PartialFunction[Try[Nothing], Boolean] = - RetryPolicy.TimeoutAndWriteExceptionsOnly - .orElse(RetryPolicy.ChannelClosedExceptionsOnly) - - protected def mkRetryPolicy(tries: Int): RetryPolicy[Try[Nothing]] = - RetryPolicy.tries(tries, DefaultRetryPartialFunction) - - @Singleton - @Provides - @Named(BatchedStratoClientWithModerateTimeout) - def providesStratoClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Client = { - Strato.client - .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) - .withSession.acquisitionTimeout(500.milliseconds) - .withRequestTimeout(ModerateStratoServerClientRequestTimeout) - .withPerRequestTimeout(ModerateStratoServerClientRequestTimeout) - .withRpcBatchSize(5) - .configured(Retries.Policy(mkRetryPolicy(1))) - .withStatsReceiver(statsReceiver.scope("strato_client")) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.docx new file mode 100644 index 000000000..fae0742d0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.scala deleted file mode 100644 index 47353afa7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.peoplediscovery.api.thriftscala.ThriftPeopleDiscoveryService -import com.twitter.util.Duration - -/** - * Copy of com.twitter.product_mixer.component_library.module.PeopleDiscoveryServiceModule - */ -object PeopleDiscoveryServiceModule - extends ThriftMethodBuilderClientModule[ - ThriftPeopleDiscoveryService.ServicePerEndpoint, - ThriftPeopleDiscoveryService.MethodPerEndpoint - ] - with MtlsClient { - - override val label: String = "people-discovery-api" - - override val dest: String = "/s/people-discovery-api/people-discovery-api:thrift" - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = { - methodBuilder - .withTimeoutPerRequest(350.millis) - .withTimeoutTotal(350.millis) - } - - override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.docx new file mode 100644 index 000000000..db5356579 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala deleted file mode 100644 index ad15988cb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.home_mixer.module - -import com.twitter.finatra.thrift.exceptions.ExceptionMapper -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.util.logging.Logging -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled -import com.twitter.scrooge.ThriftException -import com.twitter.util.Future -import javax.inject.Singleton - -@Singleton -class PipelineFailureExceptionMapper - extends ExceptionMapper[PipelineFailure, ThriftException] - with Logging { - - def handleException(throwable: PipelineFailure): Future[ThriftException] = { - throwable match { - // SliceService (unlike UrtService) throws an exception when the requested product is disabled - case PipelineFailure(ProductDisabled, reason, _, _) => - Future.exception( - t.ValidationExceptionList(errors = - Seq(t.ValidationException(t.ValidationErrorCode.ProductDisabled, reason)))) - case _ => - error("Unhandled PipelineFailure", throwable) - Future.exception(throwable) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.docx new file mode 100644 index 000000000..c9006e138 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala deleted file mode 100644 index 7dc6a072d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScores -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphManhattanEndpoint -import com.twitter.home_mixer.store.RealGraphInNetworkScoresStore -import com.twitter.inject.TwitterModule -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.util.CommonTypes.ViewerId -import com.twitter.wtf.candidate.thriftscala.Candidate - -import javax.inject.Singleton - -object RealGraphInNetworkScoresModule extends TwitterModule { - - @Provides - @Singleton - @Named(RealGraphInNetworkScores) - def providesRealGraphInNetworkScoresFeaturesStore( - @Named(RealGraphManhattanEndpoint) realGraphInNetworkScoresManhattanKVEndpoint: ManhattanKVEndpoint - ): ReadableStore[ViewerId, Seq[Candidate]] = { - new RealGraphInNetworkScoresStore(realGraphInNetworkScoresManhattanKVEndpoint) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.docx new file mode 100644 index 000000000..642a4ec70 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala deleted file mode 100644 index c3c545819..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala +++ /dev/null @@ -1,231 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.thrift.ThriftCodec -import com.twitter.home_mixer.param.HomeMixerInjectionNames.EngagementsReceivedByAuthorCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelinesRealTimeAggregateClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicCountryEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwitterListEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserAuthorEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserEngagementCache -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserTopicEngagementForNewUserCache -import com.twitter.home_mixer.util.InjectionTransformerImplicits._ -import com.twitter.inject.TwitterModule -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.{api => ml} -import com.twitter.servo.cache.KeyValueTransformingReadCache -import com.twitter.servo.cache.Memcache -import com.twitter.servo.cache.ReadCache -import com.twitter.servo.util.Transformer -import com.twitter.storehaus_internal.memcache.MemcacheHelper -import com.twitter.summingbird.batch.Batcher -import com.twitter.summingbird_internal.bijection.BatchPairImplicits -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKeyInjection -import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} - -import javax.inject.Singleton - -object RealtimeAggregateFeatureRepositoryModule - extends TwitterModule - with RealtimeAggregateHelpers { - - private val authorIdFeature = new Feature.Discrete("entities.source_author_id").getFeatureId - private val countryCodeFeature = new Feature.Text("geo.user_location.country_code").getFeatureId - private val listIdFeature = new Feature.Discrete("list.id").getFeatureId - private val userIdFeature = new Feature.Discrete("meta.user_id").getFeatureId - private val topicIdFeature = new Feature.Discrete("entities.topic_id").getFeatureId - private val tweetIdFeature = new Feature.Discrete("entities.source_tweet_id").getFeatureId - - @Provides - @Singleton - @Named(UserTopicEngagementForNewUserCache) - def providesUserTopicEngagementForNewUserCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[(Long, Long), ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD2(userIdFeature, topicIdFeature) - ) - } - - @Provides - @Singleton - @Named(TwitterListEngagementCache) - def providesTwitterListEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(listIdFeature) - ) - } - - @Provides - @Singleton - @Named(TopicEngagementCache) - def providesTopicEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(topicIdFeature) - ) - } - - @Provides - @Singleton - @Named(UserAuthorEngagementCache) - def providesUserAuthorEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[(Long, Long), ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD2(userIdFeature, authorIdFeature) - ) - } - - @Provides - @Singleton - @Named(UserEngagementCache) - def providesUserEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(userIdFeature) - ) - } - - @Provides - @Singleton - @Named(TweetCountryEngagementCache) - def providesTweetCountryEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[(Long, String), ml.DataRecord] = { - - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1T1(tweetIdFeature, countryCodeFeature) - ) - } - - @Provides - @Singleton - @Named(TweetEngagementCache) - def providesTweetEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(tweetIdFeature) - ) - } - - @Provides - @Singleton - @Named(EngagementsReceivedByAuthorCache) - def providesEngagementsReceivedByAuthorCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[Long, ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1(authorIdFeature) - ) - } - - @Provides - @Singleton - @Named(TopicCountryEngagementCache) - def providesTopicCountryEngagementCache( - @Named(TimelinesRealTimeAggregateClient) client: Memcache - ): ReadCache[(Long, String), ml.DataRecord] = { - new KeyValueTransformingReadCache( - client, - dataRecordValueTransformer, - keyTransformD1T1(topicIdFeature, countryCodeFeature) - ) - } - - @Provides - @Singleton - @Named(RealTimeInteractionGraphUserVertexCache) - def providesRealTimeInteractionGraphUserVertexCache( - @Named(RealTimeInteractionGraphUserVertexClient) client: Memcache - ): ReadCache[Long, ig.UserVertex] = { - - val valueTransformer = BinaryScalaCodec(ig.UserVertex).toByteArrayTransformer() - - val underlyingKey: Long => String = { - val cacheKeyPrefix = "user_vertex" - val defaultBatchID = Batcher.unit.currentBatch - val batchPairInjection = BatchPairImplicits.keyInjection(Injection.connect[Long, Array[Byte]]) - MemcacheHelper - .keyEncoder(cacheKeyPrefix)(batchPairInjection) - .compose((k: Long) => (k, defaultBatchID)) - } - - new KeyValueTransformingReadCache( - client, - valueTransformer, - underlyingKey - ) - } -} - -trait RealtimeAggregateHelpers { - - private def customKeyBuilder[K](prefix: String, f: K => Array[Byte]): K => String = { - // intentionally not implementing injection inverse because it is never used - def g(arr: Array[Byte]) = ??? - - MemcacheHelper.keyEncoder(prefix)(Injection.build(f)(g)) - } - - private val keyEncoder: AggregationKey => String = { - val cacheKeyPrefix = "" - val defaultBatchID = Batcher.unit.currentBatch - - val batchPairInjection = BatchPairImplicits.keyInjection(AggregationKeyInjection) - customKeyBuilder(cacheKeyPrefix, batchPairInjection) - .compose((k: AggregationKey) => (k, defaultBatchID)) - } - - protected def keyTransformD1(f1: Long)(key: Long): String = { - val aggregationKey = AggregationKey(Map(f1 -> key), Map.empty) - keyEncoder(aggregationKey) - } - - protected def keyTransformD2(f1: Long, f2: Long)(keys: (Long, Long)): String = { - val (k1, k2) = keys - val aggregationKey = AggregationKey(Map(f1 -> k1, f2 -> k2), Map.empty) - keyEncoder(aggregationKey) - } - - protected def keyTransformD1T1(f1: Long, f2: Long)(keys: (Long, String)): String = { - val (k1, k2) = keys - val aggregationKey = AggregationKey(Map(f1 -> k1), Map(f2 -> k2)) - keyEncoder(aggregationKey) - } - - protected val dataRecordValueTransformer: Transformer[DataRecord, Array[Byte]] = ThriftCodec - .toCompact[ml.DataRecord] - .toByteArrayTransformer() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.docx new file mode 100644 index 000000000..a15f972c4 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala deleted file mode 100644 index 4bed31c5c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import com.twitter.servo.cache.FinagleMemcache -import com.twitter.servo.cache.KeyTransformer -import com.twitter.servo.cache.KeyValueTransformingTtlCache -import com.twitter.servo.cache.Serializer -import com.twitter.servo.cache.ThriftSerializer -import com.twitter.servo.cache.TtlCache -import com.twitter.timelines.model.UserId -import org.apache.thrift.protocol.TCompactProtocol - -import javax.inject.Singleton - -object ScoredTweetsMemcacheModule extends TwitterModule { - - private val ScopeName = "ScoredTweetsCache" - private val ProdDestName = "/srv#/prod/local/cache/home_scored_tweets:twemcaches" - private val StagingDestName = "/srv#/test/local/cache/twemcache_home_scored_tweets:twemcaches" - private val scoredTweetsSerializer: Serializer[t.ScoredTweetsResponse] = - new ThriftSerializer[t.ScoredTweetsResponse]( - t.ScoredTweetsResponse, - new TCompactProtocol.Factory()) - private val userIdKeyTransformer: KeyTransformer[UserId] = (userId: UserId) => userId.toString - - @Singleton - @Provides - def providesScoredTweetsCache( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): TtlCache[UserId, t.ScoredTweetsResponse] = { - val destName = serviceIdentifier.environment.toLowerCase match { - case "prod" => ProdDestName - case _ => StagingDestName - } - val client = MemcachedClientBuilder.buildMemcachedClient( - destName = destName, - numTries = 2, - numConnections = 1, - requestTimeout = 200.milliseconds, - globalTimeout = 400.milliseconds, - connectTimeout = 100.milliseconds, - acquisitionTimeout = 100.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver.scope(ScopeName) - ) - val underlyingCache = new FinagleMemcache(client) - - new KeyValueTransformingTtlCache( - underlyingCache = underlyingCache, - transformer = scoredTweetsSerializer, - underlyingKey = userIdKeyTransformer - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.docx new file mode 100644 index 000000000..7656e033c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala deleted file mode 100644 index 99bf61630..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala +++ /dev/null @@ -1,77 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.home_mixer.param.HomeMixerInjectionNames.CandidateFeaturesScribeEventPublisher -import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher -import com.twitter.home_mixer.param.HomeMixerInjectionNames.MinimumFeaturesScribeEventPublisher -import com.twitter.inject.TwitterModule -import com.twitter.logpipeline.client.EventPublisherManager -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.logpipeline.client.serializers.EventLogMsgTBinarySerializer -import com.twitter.logpipeline.client.serializers.EventLogMsgThriftStructSerializer -import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} -import com.twitter.timelines.timeline_logging.{thriftscala => tl} -import javax.inject.Named -import javax.inject.Singleton - -object ScribeEventPublisherModule extends TwitterModule { - - val ClientEventLogCategory = "client_event" - val ServedCandidatesLogCategory = "home_timeline_served_candidates_flattened" - val ScoredCandidatesLogCategory = "home_timeline_scored_candidates" - val ServedCommonFeaturesLogCategory = "tq_served_common_features_offline" - val ServedCandidateFeaturesLogCategory = "tq_served_candidate_features_offline" - val ServedMinimumFeaturesLogCategory = "tq_served_minimum_features_offline" - - @Provides - @Singleton - def providesClientEventsScribeEventPublisher: EventPublisher[ca.LogEvent] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[ca.LogEvent]() - EventPublisherManager.buildScribeLogPipelinePublisher(ClientEventLogCategory, serializer) - } - - @Provides - @Singleton - @Named(CommonFeaturesScribeEventPublisher) - def providesCommonFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { - val serializer = EventLogMsgTBinarySerializer.getNewSerializer - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedCommonFeaturesLogCategory, - serializer) - } - - @Provides - @Singleton - @Named(CandidateFeaturesScribeEventPublisher) - def providesCandidateFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { - val serializer = EventLogMsgTBinarySerializer.getNewSerializer - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedCandidateFeaturesLogCategory, - serializer) - } - - @Provides - @Singleton - @Named(MinimumFeaturesScribeEventPublisher) - def providesMinimumFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { - val serializer = EventLogMsgTBinarySerializer.getNewSerializer - EventPublisherManager.buildScribeLogPipelinePublisher( - ServedMinimumFeaturesLogCategory, - serializer) - } - - @Provides - @Singleton - def providesServedCandidatesScribeEventPublisher: EventPublisher[tl.ServedEntry] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[tl.ServedEntry]() - EventPublisherManager.buildScribeLogPipelinePublisher(ServedCandidatesLogCategory, serializer) - } - - @Provides - @Singleton - def provideScoredCandidatesScribeEventPublisher: EventPublisher[tl.ScoredCandidate] = { - val serializer = EventLogMsgThriftStructSerializer.getNewSerializer[tl.ScoredCandidate]() - EventPublisherManager.buildScribeLogPipelinePublisher(ScoredCandidatesLogCategory, serializer) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.docx new file mode 100644 index 000000000..ca343d380 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.scala deleted file mode 100644 index 7e819d51e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout -import com.twitter.inject.TwitterModule -import com.twitter.strato.client.Client -import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient -import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClientImpl -import javax.inject.Named -import javax.inject.Singleton - -object SimClustersRecentEngagementsClientModule extends TwitterModule { - @Singleton - @Provides - def providesSimilarityClient( - @Named(BatchedStratoClientWithModerateTimeout) - stratoClient: Client, - statsReceiver: StatsReceiver - ): SimClustersRecentEngagementSimilarityClient = { - new SimClustersRecentEngagementSimilarityClientImpl(stratoClient, statsReceiver) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.docx new file mode 100644 index 000000000..047d4e5c0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala deleted file mode 100644 index 301dc51c2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hashing.KeyHasher -import com.twitter.home_mixer.param.HomeMixerInjectionNames.StaleTweetsCache -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import javax.inject.Singleton - -object StaleTweetsCacheModule extends TwitterModule { - - @Singleton - @Provides - @Named(StaleTweetsCache) - def providesCache( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): MemcachedClient = { - MemcachedClientBuilder.buildMemcachedClient( - destName = "/srv#/prod/local/cache/staletweetscache:twemcaches", - numTries = 3, - numConnections = 1, - requestTimeout = 200.milliseconds, - globalTimeout = 500.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver, - failureAccrualPolicy = None, - keyHasher = Some(KeyHasher.FNV1_32) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.docx new file mode 100644 index 000000000..82cb61a30 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala deleted file mode 100644 index dcbf62451..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala +++ /dev/null @@ -1,375 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.conversions.PercentOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.graph_feature_service.{thriftscala => gfs} -import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.InterestsThriftServiceClient -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserFollowedTopicIdsRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository -import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil -import com.twitter.home_mixer.util.tweetypie.RequestFields -import com.twitter.inject.TwitterModule -import com.twitter.interests.{thriftscala => int} -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder -import com.twitter.product_mixer.shared_library.thrift_client.Idempotent -import com.twitter.recos.recos_common.{thriftscala => rc} -import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.servo.cache.Cached -import com.twitter.servo.cache.CachedSerializer -import com.twitter.servo.cache.FinagleMemcacheFactory -import com.twitter.servo.cache.MemcacheCacheFactory -import com.twitter.servo.cache.NonLockingCache -import com.twitter.servo.cache.ThriftSerializer -import com.twitter.servo.keyvalue.KeyValueResultBuilder -import com.twitter.servo.repository.CachingKeyValueRepository -import com.twitter.servo.repository.ChunkingStrategy -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.servo.repository.KeyValueResult -import com.twitter.servo.repository.keysAsQuery -import com.twitter.spam.rtf.{thriftscala => sp} -import com.twitter.tweetypie.{thriftscala => tp} -import com.twitter.util.Future -import com.twitter.util.Return -import javax.inject.Named -import javax.inject.Singleton -import org.apache.thrift.protocol.TCompactProtocol - -object ThriftFeatureRepositoryModule extends TwitterModule { - - private val DefaultRPCChunkSize = 50 - private val GFSInteractionIdsLimit = 10 - - type EarlybirdQuery = (Seq[Long], Long) - type UtegQuery = (Seq[Long], (Long, Map[Long, Double])) - - @Provides - @Singleton - @Named(InterestsThriftServiceClient) - def providesInterestsThriftServiceClient( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): int.InterestsThriftService.MethodPerEndpoint = { - FinagleThriftClientBuilder - .buildFinagleMethodPerEndpoint[ - int.InterestsThriftService.ServicePerEndpoint, - int.InterestsThriftService.MethodPerEndpoint]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/interests-thrift-service/interests-thrift-service", - label = "interests", - statsReceiver = statsReceiver, - idempotency = Idempotent(1.percent), - timeoutPerRequest = 350.milliseconds, - timeoutTotal = 350.milliseconds - ) - } - - @Provides - @Singleton - @Named(UserFollowedTopicIdsRepository) - def providesUserFollowedTopicIdsRepository( - @Named(InterestsThriftServiceClient) client: int.InterestsThriftService.MethodPerEndpoint - ): KeyValueRepository[Seq[Long], Long, Seq[Long]] = { - - val lookupContext = Some( - int.ExplicitInterestLookupContext(Some(Seq(int.InterestRelationType.Followed))) - ) - - def lookup(userId: Long): Future[Seq[Long]] = { - client.getUserExplicitInterests(userId, lookupContext).map { interests => - interests.flatMap { - _.interestId match { - case int.InterestId.SemanticCore(semanticCoreInterest) => Some(semanticCoreInterest.id) - case _ => None - } - } - } - } - - val keyValueRepository = toRepository(lookup) - - keyValueRepository - } - - @Provides - @Singleton - @Named(UtegSocialProofRepository) - def providesUtegSocialProofRepository( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): KeyValueRepository[UtegQuery, Long, uteg.TweetRecommendation] = { - val client = FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ - uteg.UserTweetEntityGraph.ServicePerEndpoint, - uteg.UserTweetEntityGraph.MethodPerEndpoint]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/cassowary/user_tweet_entity_graph", - label = "uteg-social-proof-repo", - statsReceiver = statsReceiver, - idempotency = Idempotent(1.percent), - timeoutPerRequest = 150.milliseconds, - timeoutTotal = 250.milliseconds - ) - - val utegSocialProofTypes = Seq( - rc.SocialProofType.Favorite, - rc.SocialProofType.Retweet, - rc.SocialProofType.Reply - ) - - def lookup( - tweetIds: Seq[Long], - view: (Long, Map[Long, Double]) - ): Future[Seq[Option[uteg.TweetRecommendation]]] = { - val (userId, seedsWithWeights) = view - val socialProofRequest = uteg.SocialProofRequest( - requesterId = Some(userId), - seedsWithWeights = seedsWithWeights, - inputTweets = tweetIds, - socialProofTypes = Some(utegSocialProofTypes) - ) - client.findTweetSocialProofs(socialProofRequest).map { result => - val resultMap = result.socialProofResults.map(t => t.tweetId -> t).toMap - tweetIds.map(resultMap.get) - } - } - - toRepositoryBatchWithView(lookup, chunkSize = 200) - } - - @Provides - @Singleton - @Named(TweetypieContentRepository) - def providesTweetypieContentRepository( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): KeyValueRepository[Seq[Long], Long, tp.Tweet] = { - val client = FinagleThriftClientBuilder - .buildFinagleMethodPerEndpoint[ - tp.TweetService.ServicePerEndpoint, - tp.TweetService.MethodPerEndpoint]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/tweetypie/tweetypie", - label = "tweetypie-content-repo", - statsReceiver = statsReceiver, - idempotency = Idempotent(1.percent), - timeoutPerRequest = 300.milliseconds, - timeoutTotal = 500.milliseconds - ) - - def lookup(tweetIds: Seq[Long]): Future[Seq[Option[tp.Tweet]]] = { - val getTweetFieldsOptions = tp.GetTweetFieldsOptions( - tweetIncludes = RequestFields.ContentFields, - includeRetweetedTweet = false, - includeQuotedTweet = false, - forUserId = None, - safetyLevel = Some(sp.SafetyLevel.FilterNone), - visibilityPolicy = tp.TweetVisibilityPolicy.NoFiltering - ) - - val request = tp.GetTweetFieldsRequest(tweetIds = tweetIds, options = getTweetFieldsOptions) - - client.getTweetFields(request).map { results => - results.map { - case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), _, _) => - Some(found.tweet) - case _ => None - } - } - } - - val keyValueRepository = toRepositoryBatch(lookup, chunkSize = 20) - - val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( - numTries = 1, - numConnections = 1, - requestTimeout = 200.milliseconds, - globalTimeout = 200.milliseconds, - connectTimeout = 200.milliseconds, - acquisitionTimeout = 200.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - val finagleMemcacheFactory = - FinagleMemcacheFactory(cacheClient, "/s/cache/home_content_features:twemcaches") - val cacheValueTransformer = - new ThriftSerializer[tp.Tweet](tp.Tweet, new TCompactProtocol.Factory()) - val cachedSerializer = CachedSerializer.binary(cacheValueTransformer) - - val cache = MemcacheCacheFactory( - memcache = finagleMemcacheFactory(), - ttl = 48.hours - )[Long, Cached[tp.Tweet]](cachedSerializer) - - val lockingCache = new NonLockingCache(cache) - val cachedKeyValueRepository = new CachingKeyValueRepository( - keyValueRepository, - lockingCache, - keysAsQuery[Long] - ) - cachedKeyValueRepository - } - - @Provides - @Singleton - @Named(GraphTwoHopRepository) - def providesGraphTwoHopRepository( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): KeyValueRepository[(Seq[Long], Long), Long, Seq[gfs.IntersectionValue]] = { - val client = FinagleThriftClientBuilder - .buildFinagleMethodPerEndpoint[gfs.Server.ServicePerEndpoint, gfs.Server.MethodPerEndpoint]( - serviceIdentifier = serviceIdentifier, - clientId = clientId, - dest = "/s/cassowary/graph_feature_service-server", - label = "gfs-repo", - statsReceiver = statsReceiver, - idempotency = Idempotent(1.percent), - timeoutPerRequest = 350.milliseconds, - timeoutTotal = 500.milliseconds - ) - - def lookup( - userIds: Seq[Long], - viewerId: Long - ): Future[Seq[Option[Seq[gfs.IntersectionValue]]]] = { - val gfsIntersectionRequest = gfs.GfsPresetIntersectionRequest( - userId = viewerId, - candidateUserIds = userIds, - presetFeatureTypes = gfs.PresetFeatureTypes.HtlTwoHop, - intersectionIdLimit = Some(GFSInteractionIdsLimit) - ) - - client - .getPresetIntersection(gfsIntersectionRequest) - .map { graphFeatureServiceResponse => - val resultMap = graphFeatureServiceResponse.results - .map(result => result.candidateUserId -> result.intersectionValues).toMap - userIds.map(resultMap.get(_)) - } - } - - toRepositoryBatchWithView(lookup, chunkSize = 200) - } - - @Provides - @Singleton - @Named(EarlybirdRepository) - def providesEarlybirdSearchRepository( - client: eb.EarlybirdService.MethodPerEndpoint, - clientId: ClientId - ): KeyValueRepository[EarlybirdQuery, Long, eb.ThriftSearchResult] = { - - def lookup( - tweetIds: Seq[Long], - viewerId: Long - ): Future[Seq[Option[eb.ThriftSearchResult]]] = { - val request = EarlybirdRequestUtil.getTweetsFeaturesRequest( - userId = Some(viewerId), - tweetIds = Some(tweetIds), - clientId = Some(clientId.name), - authorScoreMap = None, - tensorflowModel = Some("timelines_rectweet_replica") - ) - - client - .search(request).map { response => - val resultMap = response.searchResults - .map(_.results.map { result => result.id -> result }.toMap).getOrElse(Map.empty) - tweetIds.map(resultMap.get) - } - } - toRepositoryBatchWithView(lookup) - } - - protected def toRepository[K, V]( - hydrate: K => Future[V] - ): KeyValueRepository[Seq[K], K, V] = { - def asRepository(keys: Seq[K]): Future[KeyValueResult[K, V]] = { - Future.collect(keys.map(hydrate(_).liftToTry)).map { results => - keys - .zip(results) - .foldLeft(new KeyValueResultBuilder[K, V]()) { - case (bldr, (k, result)) => - result match { - case Return(v) => bldr.addFound(k, v) - case _ => bldr.addNotFound(k) - } - }.result - } - } - - asRepository - } - - protected def toRepositoryBatch[K, V]( - hydrate: Seq[K] => Future[Seq[Option[V]]], - chunkSize: Int = DefaultRPCChunkSize - ): KeyValueRepository[Seq[K], K, V] = { - def repository(keys: Seq[K]): Future[KeyValueResult[K, V]] = - batchRepositoryProcess(keys, hydrate(keys)) - - KeyValueRepository.chunked(repository, ChunkingStrategy.equalSize(chunkSize)) - } - - protected def toRepositoryBatchWithView[K, T, V]( - hydrate: (Seq[K], T) => Future[Seq[Option[V]]], - chunkSize: Int = DefaultRPCChunkSize - ): KeyValueRepository[(Seq[K], T), K, V] = { - def repository(input: (Seq[K], T)): Future[KeyValueResult[K, V]] = { - val (keys, view) = input - batchRepositoryProcess(keys, hydrate(keys, view)) - } - - KeyValueRepository.chunked(repository, CustomChunkingStrategy.equalSizeWithView(chunkSize)) - } - - private def batchRepositoryProcess[K, V]( - keys: Seq[K], - f: Future[Seq[Option[V]]] - ): Future[KeyValueResult[K, V]] = { - f.liftToTry - .map { - case Return(values) => - keys - .zip(values) - .foldLeft(new KeyValueResultBuilder[K, V]()) { - case (bldr, (k, value)) => - value match { - case Some(v) => bldr.addFound(k, v) - case _ => bldr.addNotFound(k) - } - }.result - case _ => - keys - .foldLeft(new KeyValueResultBuilder[K, V]()) { - case (bldr, k) => bldr.addNotFound(k) - }.result - } - } - - // Use only for cases not already covered by Servo's [[ChunkingStrategy]] - object CustomChunkingStrategy { - def equalSizeWithView[K, T](maxSize: Int): ((Seq[K], T)) => Seq[(Seq[K], T)] = { - case (keys, view) => - ChunkingStrategy - .equalSize[K](maxSize)(keys) - .map { chunk: Seq[K] => (chunk, view) } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.docx new file mode 100644 index 000000000..0810dc4c3 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala deleted file mode 100644 index 4af39fd32..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.timelinemixer.clients.persistence.TimelinePersistenceManhattanClientBuilder -import com.twitter.timelinemixer.clients.persistence.TimelinePersistenceManhattanClientConfig -import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient -import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 -import com.twitter.util.Duration -import javax.inject.Singleton - -object TimelinesPersistenceStoreClientModule extends TwitterModule { - private val StagingDataset = "timeline_response_batches_v5_nonprod" - private val ProdDataset = "timeline_response_batches_v5" - private final val Timeout = "mh_persistence_store.timeout" - - flag[Duration](Timeout, 300.millis, "Timeout per request") - - @Provides - @Singleton - def providesTimelinesPersistenceStoreClient( - @Flag(Timeout) timeout: Duration, - injectedServiceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): TimelineResponseBatchesClient[TimelineResponseV3] = { - val timelineResponseBatchesDataset = - injectedServiceIdentifier.environment.toLowerCase match { - case "prod" => ProdDataset - case _ => StagingDataset - } - - val timelineResponseBatchesConfig = new TimelinePersistenceManhattanClientConfig { - val dataset = timelineResponseBatchesDataset - val isReadOnly = false - val serviceIdentifier = injectedServiceIdentifier - override val defaultMaxTimeout = timeout - override val maxRetryCount = 2 - } - - TimelinePersistenceManhattanClientBuilder.buildTimelineResponseV3BatchesClient( - timelineResponseBatchesConfig, - statsReceiver - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.docx new file mode 100644 index 000000000..452e57947 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala deleted file mode 100644 index 9333e0f84..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout -import com.twitter.inject.TwitterModule -import com.twitter.strato.client.Client -import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient -import com.twitter.timelines.clients.strato.topics.TopicSocialProofClientImpl -import javax.inject.Named -import javax.inject.Singleton - -object TopicSocialProofClientModule extends TwitterModule { - - @Singleton - @Provides - def providesSimilarityClient( - @Named(BatchedStratoClientWithModerateTimeout) - stratoClient: Client, - statsReceiver: StatsReceiver - ): TopicSocialProofClient = new TopicSocialProofClientImpl(stratoClient, statsReceiver) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.docx new file mode 100644 index 000000000..7f2e5f7d7 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala deleted file mode 100644 index 1eb49206c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.annotations.Flags -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.stitch.tweetypie.TweetyPie -import com.twitter.tweetypie.thriftscala.TweetService -import com.twitter.util.Duration -import javax.inject.Singleton - -/** - * Idempotent Tweetypie Thrift and Stitch client. - */ -object TweetypieClientModule - extends ThriftMethodBuilderClientModule[ - TweetService.ServicePerEndpoint, - TweetService.MethodPerEndpoint - ] - with MtlsClient { - - private val TimeoutRequest = "tweetypie.timeout_request" - private val TimeoutTotal = "tweetypie.timeout_total" - - flag[Duration](TimeoutRequest, 1000.millis, "Timeout per request") - flag[Duration](TimeoutTotal, 1000.millis, "Total timeout") - - override val label: String = "tweetypie" - override val dest: String = "/s/tweetypie/tweetypie" - - @Singleton - @Provides - def providesTweetypieStitchClient(tweetService: TweetService.MethodPerEndpoint): TweetyPie = - new TweetyPie(tweetService) - - /** - * TweetyPie client id must be in the form of {service.env} or it will not be treated as an - * unauthorized client - */ - override protected def clientId(injector: Injector): ClientId = { - val serviceIdentifier = injector.instance[ServiceIdentifier] - ClientId(s"${serviceIdentifier.service}.${serviceIdentifier.environment}") - } - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = { - val timeoutRequest = injector.instance[Duration](Flags.named(TimeoutRequest)) - val timeoutTotal = injector.instance[Duration](Flags.named(TimeoutTotal)) - - methodBuilder - .withTimeoutPerRequest(timeoutRequest) - .withTimeoutTotal(timeoutTotal) - } - - override protected def sessionAcquisitionTimeout: Duration = 500.millis -}