mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-12-25 03:31:51 +01:00
[docx] split commit for file 1000
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer
parent
eb3e2f9bcf
commit
7ed90fef05
@ -1,20 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"cortex-deepbird/prediction/src/main/scala/com/twitter/cortex/deepbird/prediction",
|
||||
"cortex-deepbird/thrift/src/main/thrift:thrift-java",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common",
|
||||
"src/scala/com/twitter/ml/api/util",
|
||||
"util/util-core:scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,67 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.deepbirdv2
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.bijection.scrooge.TBinaryProtocol
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService
|
||||
import com.twitter.finagle.ThriftMux
|
||||
import com.twitter.finagle.builder.ClientBuilder
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.mtls.client.MtlsStackClient._
|
||||
import com.twitter.finagle.stats.NullStatsReceiver
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.finagle.thrift.RichClientParam
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants
|
||||
import com.twitter.inject.TwitterModule
|
||||
|
||||
/**
|
||||
* Module that provides multiple deepbirdv2 prediction service clients
|
||||
* We use the java api since data records are native java objects and we want to reduce overhead
|
||||
* while serializing/deserializing data.
|
||||
*/
|
||||
object DeepBirdV2PredictionServiceClientModule extends TwitterModule {
|
||||
|
||||
val RequestTimeout = 300.millis
|
||||
|
||||
private def getDeepbirdPredictionServiceClient(
|
||||
clientId: ClientId,
|
||||
label: String,
|
||||
dest: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
serviceIdentifier: ServiceIdentifier
|
||||
): DeepbirdPredictionService.ServiceToClient = {
|
||||
val clientStatsReceiver = statsReceiver.scope("clnt")
|
||||
val mTlsClient = ThriftMux.client.withClientId(clientId).withMutualTls(serviceIdentifier)
|
||||
new DeepbirdPredictionService.ServiceToClient(
|
||||
ClientBuilder()
|
||||
.name(label)
|
||||
.stack(mTlsClient)
|
||||
.dest(dest)
|
||||
.requestTimeout(RequestTimeout)
|
||||
.reportHostStats(NullStatsReceiver)
|
||||
.build(),
|
||||
RichClientParam(
|
||||
new TBinaryProtocol.Factory(),
|
||||
clientStats = clientStatsReceiver
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT)
|
||||
def providesWtfProdDeepbirdV2PredictionService(
|
||||
clientId: ClientId,
|
||||
statsReceiver: StatsReceiver,
|
||||
serviceIdentifier: ServiceIdentifier
|
||||
): DeepbirdPredictionService.ServiceToClient = {
|
||||
getDeepbirdPredictionServiceClient(
|
||||
clientId = clientId,
|
||||
label = "WtfProdDeepbirdV2PredictionService",
|
||||
dest = "/s/cassowary/deepbirdv2-hermit-wtf",
|
||||
statsReceiver = statsReceiver,
|
||||
serviceIdentifier = serviceIdentifier
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/github/nscala_time",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"src/thrift/com/twitter/onboarding/relevance/store:store-scala",
|
||||
"util/util-core:scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,60 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.dismiss_store
|
||||
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.onboarding.relevance.store.thriftscala.WhoToFollowDismissEventDetails
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.strato.catalog.Scan.Slice
|
||||
import com.twitter.strato.client.Scanner
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Named
|
||||
import javax.inject.Singleton
|
||||
|
||||
/**
|
||||
* this store gets the list of dismissed candidates since a certain time
|
||||
* primarily used for filtering out accounts that a user has explicitly dismissed
|
||||
*
|
||||
* we fail open on timeouts, but loudly on other errors
|
||||
*/
|
||||
@Singleton
|
||||
class DismissStore @Inject() (
|
||||
@Named(GuiceNamedConstants.DISMISS_STORE_SCANNER)
|
||||
scanner: Scanner[(Long, Slice[
|
||||
(Long, Long)
|
||||
]), Unit, (Long, (Long, Long)), WhoToFollowDismissEventDetails],
|
||||
stats: StatsReceiver)
|
||||
extends Logging {
|
||||
|
||||
private val MaxCandidatesToReturn = 100
|
||||
|
||||
// gets a list of dismissed candidates. if numCandidatesToFetchOption is none, we will fetch the default number of candidates
|
||||
def get(
|
||||
userId: Long,
|
||||
negStartTimeMs: Long,
|
||||
maxCandidatesToFetchOption: Option[Int]
|
||||
): Stitch[Seq[Long]] = {
|
||||
|
||||
val maxCandidatesToFetch = maxCandidatesToFetchOption.getOrElse(MaxCandidatesToReturn)
|
||||
|
||||
scanner
|
||||
.scan(
|
||||
(
|
||||
userId,
|
||||
Slice(
|
||||
from = None,
|
||||
to = Some((negStartTimeMs, Long.MaxValue)),
|
||||
limit = Some(maxCandidatesToFetch)
|
||||
)
|
||||
)
|
||||
)
|
||||
.map {
|
||||
case s: Seq[((Long, (Long, Long)), WhoToFollowDismissEventDetails)] if s.nonEmpty =>
|
||||
s.map {
|
||||
case ((_: Long, (_: Long, candidateId: Long)), _: WhoToFollowDismissEventDetails) =>
|
||||
candidateId
|
||||
}
|
||||
case _ => Nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"emailstorage/server/src/main/thrift/com/twitter/emailstorage/api:email-storage-service-scala",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,28 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.email_storage_service
|
||||
|
||||
import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing
|
||||
import com.twitter.emailstorage.api.thriftscala.EmailStorageService
|
||||
import com.twitter.emailstorage.api.thriftscala.GetUsersEmailsRequest
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class EmailStorageServiceClient @Inject() (
|
||||
val emailStorageService: EmailStorageService.MethodPerEndpoint) {
|
||||
|
||||
def getVerifiedEmail(
|
||||
userId: Long,
|
||||
purposeOfProcessing: PurposeOfProcessing
|
||||
): Stitch[Option[String]] = {
|
||||
val req = GetUsersEmailsRequest(
|
||||
userIds = Seq(userId),
|
||||
clientIdentifier = Some("follow-recommendations-service"),
|
||||
purposesOfProcessing = Some(Seq(purposeOfProcessing))
|
||||
)
|
||||
|
||||
Stitch.callFuture(emailStorageService.getUsersEmails(req)) map {
|
||||
_.usersEmails.map(_.confirmedEmail.map(_.email)).head
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,12 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.email_storage_service
|
||||
|
||||
import com.twitter.emailstorage.api.thriftscala.EmailStorageService
|
||||
import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient
|
||||
import com.twitter.follow_recommendations.common.clients.common.BaseClientModule
|
||||
|
||||
object EmailStorageServiceModule
|
||||
extends BaseClientModule[EmailStorageService.MethodPerEndpoint]
|
||||
with MtlsClient {
|
||||
override val label = "email-storage-service"
|
||||
override val dest = "/s/email-server/email-server"
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/github/nscala_time",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"src/thrift/com/twitter/geoduck:geoduck-scala",
|
||||
"src/thrift/com/twitter/geoduck:geoduckpartnerplaces-thrift-scala",
|
||||
"stitch/stitch-core",
|
||||
"util/util-core:scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,62 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.geoduck
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode
|
||||
import com.twitter.geoduck.common.thriftscala.LocationSource
|
||||
import com.twitter.geoduck.common.thriftscala.PlaceQuery
|
||||
import com.twitter.geoduck.common.thriftscala.TransactionLocation
|
||||
import com.twitter.geoduck.common.thriftscala.UserLocationRequest
|
||||
import com.twitter.geoduck.thriftscala.LocationService
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class LocationServiceClient @Inject() (locationService: LocationService.MethodPerEndpoint) {
|
||||
def getGeohashAndCountryCode(userId: Long): Stitch[GeohashAndCountryCode] = {
|
||||
Stitch
|
||||
.callFuture {
|
||||
locationService
|
||||
.userLocation(
|
||||
UserLocationRequest(
|
||||
Seq(userId),
|
||||
Some(PlaceQuery(allPlaceTypes = Some(true))),
|
||||
simpleReverseGeocode = true))
|
||||
.map(_.found.get(userId)).map { transactionLocationOpt =>
|
||||
val geohashOpt = transactionLocationOpt.flatMap(getGeohashFromTransactionLocation)
|
||||
val countryCodeOpt =
|
||||
transactionLocationOpt.flatMap(_.simpleRgcResult.flatMap(_.countryCodeAlpha2))
|
||||
GeohashAndCountryCode(geohashOpt, countryCodeOpt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private[this] def getGeohashFromTransactionLocation(
|
||||
transactionLocation: TransactionLocation
|
||||
): Option[String] = {
|
||||
transactionLocation.geohash.flatMap { geohash =>
|
||||
val geohashPrefixLength = transactionLocation.locationSource match {
|
||||
// if location source is logical, keep the first 4 chars in geohash
|
||||
case Some(LocationSource.Logical) => Some(4)
|
||||
// if location source is physical, keep the prefix according to accuracy
|
||||
// accuracy is the accuracy of GPS readings in the unit of meter
|
||||
case Some(LocationSource.Physical) =>
|
||||
transactionLocation.coordinate.flatMap { coordinate =>
|
||||
coordinate.accuracy match {
|
||||
case Some(accuracy) if (accuracy < 50) => Some(7)
|
||||
case Some(accuracy) if (accuracy < 200) => Some(6)
|
||||
case Some(accuracy) if (accuracy < 1000) => Some(5)
|
||||
case Some(accuracy) if (accuracy < 50000) => Some(4)
|
||||
case Some(accuracy) if (accuracy < 100000) => Some(3)
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
case Some(LocationSource.Model) => Some(4)
|
||||
case _ => None
|
||||
}
|
||||
geohashPrefixLength match {
|
||||
case Some(l: Int) => geohash.stringGeohash.map(_.take(l))
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,12 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.geoduck
|
||||
|
||||
import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient
|
||||
import com.twitter.follow_recommendations.common.clients.common.BaseClientModule
|
||||
import com.twitter.geoduck.thriftscala.LocationService
|
||||
|
||||
object LocationServiceModule
|
||||
extends BaseClientModule[LocationService.MethodPerEndpoint]
|
||||
with MtlsClient {
|
||||
override val label = "geoduck_locationservice"
|
||||
override val dest = "/s/geo/geoduck_locationservice"
|
||||
}
|
||||
Binary file not shown.
@ -1,57 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.geoduck
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode
|
||||
import com.twitter.geoduck.common.thriftscala.Location
|
||||
import com.twitter.geoduck.common.thriftscala.PlaceQuery
|
||||
import com.twitter.geoduck.common.thriftscala.ReverseGeocodeIPRequest
|
||||
import com.twitter.geoduck.service.thriftscala.GeoContext
|
||||
import com.twitter.geoduck.thriftscala.ReverseGeocoder
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ReverseGeocodeClient @Inject() (rgcService: ReverseGeocoder.MethodPerEndpoint) {
|
||||
def getGeohashAndCountryCode(ipAddress: String): Stitch[GeohashAndCountryCode] = {
|
||||
Stitch
|
||||
.callFuture {
|
||||
rgcService
|
||||
.reverseGeocodeIp(
|
||||
ReverseGeocodeIPRequest(
|
||||
Seq(ipAddress),
|
||||
PlaceQuery(None),
|
||||
simpleReverseGeocode = true
|
||||
) // note: simpleReverseGeocode means that country code will be included in response
|
||||
).map { response =>
|
||||
response.found.get(ipAddress) match {
|
||||
case Some(location) => getGeohashAndCountryCodeFromLocation(location)
|
||||
case _ => GeohashAndCountryCode(None, None)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private def getGeohashAndCountryCodeFromLocation(location: Location): GeohashAndCountryCode = {
|
||||
val countryCode: Option[String] = location.simpleRgcResult.flatMap { _.countryCodeAlpha2 }
|
||||
|
||||
val geohashString: Option[String] = location.geohash.flatMap { hash =>
|
||||
hash.stringGeohash.flatMap { hashString =>
|
||||
Some(ReverseGeocodeClient.truncate(hashString))
|
||||
}
|
||||
}
|
||||
|
||||
GeohashAndCountryCode(geohashString, countryCode)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
object ReverseGeocodeClient {
|
||||
|
||||
val DefaultGeoduckIPRequestContext: GeoContext =
|
||||
GeoContext(allPlaceTypes = true, includeGeohash = true, includeCountryCode = true)
|
||||
|
||||
// All these geohashes are guessed by IP (Logical Location Source).
|
||||
// So take the four letters to make sure it is consistent with LocationServiceClient
|
||||
val GeohashLengthAfterTruncation = 4
|
||||
def truncate(geohash: String): String = geohash.take(GeohashLengthAfterTruncation)
|
||||
}
|
||||
Binary file not shown.
@ -1,59 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.geoduck
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class UserLocationFetcher @Inject() (
|
||||
locationServiceClient: LocationServiceClient,
|
||||
reverseGeocodeClient: ReverseGeocodeClient,
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
private val stats: StatsReceiver = statsReceiver.scope("user_location_fetcher")
|
||||
private val totalRequestsCounter = stats.counter("requests")
|
||||
private val emptyResponsesCounter = stats.counter("empty")
|
||||
private val locationServiceExceptionCounter = stats.counter("location_service_exception")
|
||||
private val reverseGeocodeExceptionCounter = stats.counter("reverse_geocode_exception")
|
||||
|
||||
def getGeohashAndCountryCode(
|
||||
userId: Option[Long],
|
||||
ipAddress: Option[String]
|
||||
): Stitch[Option[GeohashAndCountryCode]] = {
|
||||
totalRequestsCounter.incr()
|
||||
val lscLocationStitch = Stitch
|
||||
.collect {
|
||||
userId.map(locationServiceClient.getGeohashAndCountryCode)
|
||||
}.rescue {
|
||||
case _: Exception =>
|
||||
locationServiceExceptionCounter.incr()
|
||||
Stitch.None
|
||||
}
|
||||
|
||||
val ipLocationStitch = Stitch
|
||||
.collect {
|
||||
ipAddress.map(reverseGeocodeClient.getGeohashAndCountryCode)
|
||||
}.rescue {
|
||||
case _: Exception =>
|
||||
reverseGeocodeExceptionCounter.incr()
|
||||
Stitch.None
|
||||
}
|
||||
|
||||
Stitch.join(lscLocationStitch, ipLocationStitch).map {
|
||||
case (lscLocation, ipLocation) => {
|
||||
val geohash = lscLocation.flatMap(_.geohash).orElse(ipLocation.flatMap(_.geohash))
|
||||
val countryCode =
|
||||
lscLocation.flatMap(_.countryCode).orElse(ipLocation.flatMap(_.countryCode))
|
||||
(geohash, countryCode) match {
|
||||
case (None, None) =>
|
||||
emptyResponsesCounter.incr()
|
||||
None
|
||||
case _ => Some(GeohashAndCountryCode(geohash, countryCode))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,21 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/github/nscala_time",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common",
|
||||
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||
"stitch/stitch-gizmoduck",
|
||||
"util/util-core:scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,81 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.gizmoduck
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.StatsUtil
|
||||
import com.twitter.gizmoduck.thriftscala.LookupContext
|
||||
import com.twitter.gizmoduck.thriftscala.PerspectiveEdge
|
||||
import com.twitter.gizmoduck.thriftscala.QueryFields
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.gizmoduck.Gizmoduck
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class GizmoduckClient @Inject() (gizmoduckStitchClient: Gizmoduck, statsReceiver: StatsReceiver) {
|
||||
val stats = statsReceiver.scope("gizmoduck_client")
|
||||
val getByIdStats = stats.scope("get_by_id")
|
||||
val getUserById = stats.scope("get_user_by_id")
|
||||
|
||||
def isProtected(userId: Long): Stitch[Boolean] = {
|
||||
// get latency metrics with StatsUtil.profileStitch when calling .getById
|
||||
val response = StatsUtil.profileStitch(
|
||||
gizmoduckStitchClient.getById(userId, Set(QueryFields.Safety)),
|
||||
getByIdStats
|
||||
)
|
||||
response.map { result =>
|
||||
result.user.flatMap(_.safety).map(_.isProtected).getOrElse(true)
|
||||
}
|
||||
}
|
||||
|
||||
def getUserName(userId: Long, forUserId: Long): Stitch[Option[String]] = {
|
||||
val queryFields = GizmoduckClient.GetUserByIdUserNameQueryFields
|
||||
val lookupContext = LookupContext(
|
||||
forUserId = Some(forUserId),
|
||||
perspectiveEdges = Some(GizmoduckClient.DefaultPerspectiveEdges)
|
||||
)
|
||||
// get latency metrics with StatsUtil.profileStitch when calling .getUserById
|
||||
val response = StatsUtil.profileStitch(
|
||||
gizmoduckStitchClient.getUserById(userId, queryFields, lookupContext),
|
||||
getUserById
|
||||
)
|
||||
response.map(_.profile.map(_.name))
|
||||
}
|
||||
}
|
||||
|
||||
object GizmoduckClient {
|
||||
// Similar to GizmoduckUserRepository.DefaultPerspectiveEdges
|
||||
val DefaultPerspectiveEdges: Set[PerspectiveEdge] =
|
||||
Set(
|
||||
PerspectiveEdge.Blocking,
|
||||
PerspectiveEdge.BlockedBy,
|
||||
PerspectiveEdge.DeviceFollowing,
|
||||
PerspectiveEdge.FollowRequestSent,
|
||||
PerspectiveEdge.Following,
|
||||
PerspectiveEdge.FollowedBy,
|
||||
PerspectiveEdge.LifelineFollowing,
|
||||
PerspectiveEdge.LifelineFollowedBy,
|
||||
PerspectiveEdge.Muting,
|
||||
PerspectiveEdge.NoRetweetsFrom
|
||||
)
|
||||
|
||||
// From GizmoduckUserRepository.DefaultQueryFields
|
||||
val GetUserByIdQueryFields: Set[QueryFields] = Set(
|
||||
QueryFields.Account,
|
||||
QueryFields.Counts,
|
||||
QueryFields.ExtendedProfile,
|
||||
QueryFields.Perspective,
|
||||
QueryFields.Profile,
|
||||
QueryFields.ProfileDesign,
|
||||
QueryFields.ProfileLocation,
|
||||
QueryFields.Safety,
|
||||
QueryFields.Roles,
|
||||
QueryFields.Takedowns,
|
||||
QueryFields.UrlEntities,
|
||||
QueryFields.DirectMessageView,
|
||||
QueryFields.MediaView
|
||||
)
|
||||
|
||||
val GetUserByIdUserNameQueryFields: Set[QueryFields] = Set(
|
||||
QueryFields.Profile
|
||||
)
|
||||
}
|
||||
Binary file not shown.
@ -1,24 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.gizmoduck
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient
|
||||
import com.twitter.follow_recommendations.common.clients.common.BaseClientModule
|
||||
import com.twitter.gizmoduck.thriftscala.QueryFields
|
||||
import com.twitter.gizmoduck.thriftscala.UserService
|
||||
import com.twitter.stitch.gizmoduck.Gizmoduck
|
||||
import javax.inject.Singleton
|
||||
|
||||
object GizmoduckModule extends BaseClientModule[UserService.MethodPerEndpoint] with MtlsClient {
|
||||
override val label = "gizmoduck"
|
||||
override val dest = "/s/gizmoduck/gizmoduck"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def provideExtraGizmoduckQueryFields: Set[QueryFields] = Set.empty
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesStitchClient(futureIface: UserService.MethodPerEndpoint): Gizmoduck = {
|
||||
Gizmoduck(futureIface)
|
||||
}
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,50 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.graph_feature_service
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.FollowProof
|
||||
import com.twitter.graph_feature_service.thriftscala.PresetFeatureTypes.WtfTwoHop
|
||||
import com.twitter.graph_feature_service.thriftscala.EdgeType
|
||||
import com.twitter.graph_feature_service.thriftscala.GfsIntersectionResponse
|
||||
import com.twitter.graph_feature_service.thriftscala.GfsPresetIntersectionRequest
|
||||
import com.twitter.graph_feature_service.thriftscala.{Server => GraphFeatureService}
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.{Inject, Singleton}
|
||||
|
||||
@Singleton
|
||||
class GraphFeatureServiceClient @Inject() (
|
||||
graphFeatureService: GraphFeatureService.MethodPerEndpoint) {
|
||||
|
||||
import GraphFeatureServiceClient._
|
||||
def getIntersections(
|
||||
userId: Long,
|
||||
candidateIds: Seq[Long],
|
||||
numIntersectionIds: Int
|
||||
): Stitch[Map[Long, FollowProof]] = {
|
||||
Stitch
|
||||
.callFuture(
|
||||
graphFeatureService.getPresetIntersection(
|
||||
GfsPresetIntersectionRequest(userId, candidateIds, WtfTwoHop, Some(numIntersectionIds))
|
||||
)
|
||||
).map {
|
||||
case GfsIntersectionResponse(gfsIntersectionResults) =>
|
||||
(for {
|
||||
candidateId <- candidateIds
|
||||
gfsIntersectionResultForCandidate =
|
||||
gfsIntersectionResults.filter(_.candidateUserId == candidateId)
|
||||
followProof <- for {
|
||||
result <- gfsIntersectionResultForCandidate
|
||||
intersection <- result.intersectionValues
|
||||
if leftEdgeTypes.contains(intersection.featureType.leftEdgeType)
|
||||
if rightEdgeTypes.contains(intersection.featureType.rightEdgeType)
|
||||
intersectionIds <- intersection.intersectionIds.toSeq
|
||||
} yield FollowProof(intersectionIds, intersection.count.getOrElse(0))
|
||||
} yield {
|
||||
candidateId -> followProof
|
||||
}).toMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object GraphFeatureServiceClient {
|
||||
val leftEdgeTypes: Set[EdgeType] = Set(EdgeType.Following)
|
||||
val rightEdgeTypes: Set[EdgeType] = Set(EdgeType.FollowedBy)
|
||||
}
|
||||
Binary file not shown.
@ -1,12 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.graph_feature_service
|
||||
|
||||
import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient
|
||||
import com.twitter.follow_recommendations.common.clients.common.BaseClientModule
|
||||
import com.twitter.graph_feature_service.thriftscala.{Server => GraphFeatureService}
|
||||
|
||||
object GraphFeatureStoreModule
|
||||
extends BaseClientModule[GraphFeatureService.MethodPerEndpoint]
|
||||
with MtlsClient {
|
||||
override val label = "graph_feature_service"
|
||||
override val dest = "/s/cassowary/graph_feature_service-server"
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/github/nscala_time",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"stitch/stitch-socialgraph",
|
||||
"util/util-core:scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,31 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.impression_store
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.follow_recommendations.thriftscala.DisplayLocation
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.strato.catalog.Scan.Slice
|
||||
import com.twitter.strato.client.Client
|
||||
import com.twitter.strato.thrift.ScroogeConvImplicits._
|
||||
|
||||
object ImpressionStoreModule extends TwitterModule {
|
||||
|
||||
val columnPath: String = "onboarding/userrecs/wtfImpressionCountsStore"
|
||||
|
||||
type PKey = (Long, DisplayLocation)
|
||||
type LKey = Long
|
||||
type Value = (Long, Int)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesImpressionStore(stratoClient: Client): WtfImpressionStore = {
|
||||
new WtfImpressionStore(
|
||||
stratoClient.scanner[
|
||||
(PKey, Slice[LKey]),
|
||||
Unit,
|
||||
(PKey, LKey),
|
||||
Value
|
||||
](columnPath)
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,42 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.impression_store
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.follow_recommendations.common.models.WtfImpression
|
||||
import com.twitter.follow_recommendations.thriftscala.{DisplayLocation => TDisplayLocation}
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.strato.catalog.Scan.Slice
|
||||
import com.twitter.strato.client.Scanner
|
||||
import com.twitter.util.Time
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class WtfImpressionStore @Inject() (
|
||||
scanner: Scanner[
|
||||
((Long, TDisplayLocation), Slice[Long]),
|
||||
Unit,
|
||||
((Long, TDisplayLocation), Long),
|
||||
(Long, Int)
|
||||
]) extends Logging {
|
||||
def get(userId: Long, dl: DisplayLocation): Stitch[Seq[WtfImpression]] = {
|
||||
val thriftDl = dl.toThrift
|
||||
scanner.scan(((userId, thriftDl), Slice.all[Long])).map { impressionsPerDl =>
|
||||
val wtfImpressions =
|
||||
for {
|
||||
(((_, _), candidateId), (latestTs, counts)) <- impressionsPerDl
|
||||
} yield WtfImpression(
|
||||
candidateId = candidateId,
|
||||
displayLocation = dl,
|
||||
latestTime = Time.fromMilliseconds(latestTs),
|
||||
counts = counts
|
||||
)
|
||||
wtfImpressions
|
||||
} rescue {
|
||||
// fail open so that the request can still go through
|
||||
case ex: Throwable =>
|
||||
logger.warn(s"$dl WtfImpressionsStore warn: " + ex.getMessage)
|
||||
Stitch.Nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
scala_library(
|
||||
name = "interests_service",
|
||||
sources = ["InterestServiceClient.scala"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests",
|
||||
"interests-service/thrift/src/main/thrift:thrift-scala",
|
||||
"strato/src/main/scala/com/twitter/strato/catalog",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
"strato/src/main/scala/com/twitter/strato/data",
|
||||
"strato/src/main/scala/com/twitter/strato/thrift",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,115 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.interests_service
|
||||
|
||||
import com.google.inject.Inject
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.finagle.stats.NullStatsReceiver
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.store.InterestedInInterestsFetchKey
|
||||
import com.twitter.inject.Logging
|
||||
import com.twitter.interests.thriftscala.InterestId
|
||||
import com.twitter.interests.thriftscala.InterestRelationship
|
||||
import com.twitter.interests.thriftscala.InterestedInInterestModel
|
||||
import com.twitter.interests.thriftscala.UserInterest
|
||||
import com.twitter.interests.thriftscala.UserInterestData
|
||||
import com.twitter.interests.thriftscala.UserInterestsResponse
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.strato.client.Client
|
||||
import com.twitter.strato.thrift.ScroogeConvImplicits._
|
||||
|
||||
@Singleton
|
||||
class InterestServiceClient @Inject() (
|
||||
stratoClient: Client,
|
||||
statsReceiver: StatsReceiver = NullStatsReceiver)
|
||||
extends Logging {
|
||||
|
||||
val interestsServiceStratoColumnPath = "interests/interestedInInterests"
|
||||
val stats = statsReceiver.scope("interest_service_client")
|
||||
val errorCounter = stats.counter("error")
|
||||
|
||||
private val interestsFetcher =
|
||||
stratoClient.fetcher[InterestedInInterestsFetchKey, UserInterestsResponse](
|
||||
interestsServiceStratoColumnPath,
|
||||
checkTypes = true
|
||||
)
|
||||
|
||||
def fetchUttInterestIds(
|
||||
userId: Long
|
||||
): Stitch[Seq[Long]] = {
|
||||
fetchInterestRelationships(userId)
|
||||
.map(_.toSeq.flatten.flatMap(extractUttInterest))
|
||||
}
|
||||
|
||||
def extractUttInterest(
|
||||
interestRelationShip: InterestRelationship
|
||||
): Option[Long] = {
|
||||
interestRelationShip match {
|
||||
case InterestRelationship.V1(relationshipV1) =>
|
||||
relationshipV1.interestId match {
|
||||
case InterestId.SemanticCore(semanticCoreInterest) => Some(semanticCoreInterest.id)
|
||||
case _ => None
|
||||
}
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
|
||||
def fetchCustomInterests(
|
||||
userId: Long
|
||||
): Stitch[Seq[String]] = {
|
||||
fetchInterestRelationships(userId)
|
||||
.map(_.toSeq.flatten.flatMap(extractCustomInterest))
|
||||
}
|
||||
|
||||
def extractCustomInterest(
|
||||
interestRelationShip: InterestRelationship
|
||||
): Option[String] = {
|
||||
interestRelationShip match {
|
||||
case InterestRelationship.V1(relationshipV1) =>
|
||||
relationshipV1.interestId match {
|
||||
case InterestId.FreeForm(freeFormInterest) => Some(freeFormInterest.interest)
|
||||
case _ => None
|
||||
}
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
|
||||
def fetchInterestRelationships(
|
||||
userId: Long
|
||||
): Stitch[Option[Seq[InterestRelationship]]] = {
|
||||
interestsFetcher
|
||||
.fetch(
|
||||
InterestedInInterestsFetchKey(
|
||||
userId = userId,
|
||||
labels = None,
|
||||
None
|
||||
))
|
||||
.map(_.v)
|
||||
.map {
|
||||
case Some(response) =>
|
||||
response.interests.interests.map { interests =>
|
||||
interests.collect {
|
||||
case UserInterest(_, Some(interestData)) =>
|
||||
getInterestRelationship(interestData)
|
||||
}.flatten
|
||||
}
|
||||
case _ => None
|
||||
}
|
||||
.rescue {
|
||||
case e: Throwable => // we are swallowing all errors
|
||||
logger.warn(s"interests could not be retrieved for user $userId due to ${e.getCause}")
|
||||
errorCounter.incr
|
||||
Stitch.None
|
||||
}
|
||||
}
|
||||
|
||||
private def getInterestRelationship(
|
||||
interestData: UserInterestData
|
||||
): Seq[InterestRelationship] = {
|
||||
interestData match {
|
||||
case UserInterestData.InterestedIn(interestModels) =>
|
||||
interestModels.collect {
|
||||
case InterestedInInterestModel.ExplicitModel(model) => model
|
||||
}
|
||||
case _ => Nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"phonestorage/server/src/main/thrift/com/twitter/phonestorage/api:phone-storage-service-scala",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,34 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.phone_storage_service
|
||||
|
||||
import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing
|
||||
import com.twitter.phonestorage.api.thriftscala.GetUserPhonesByUsersRequest
|
||||
import com.twitter.phonestorage.api.thriftscala.PhoneStorageService
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class PhoneStorageServiceClient @Inject() (
|
||||
val phoneStorageService: PhoneStorageService.MethodPerEndpoint) {
|
||||
|
||||
/**
|
||||
* PSS can potentially return multiple phone records.
|
||||
* The current implementation of getUserPhonesByUsers returns only a single phone for a single user_id but
|
||||
* we can trivially support handling multiple in case that changes in the future.
|
||||
*/
|
||||
def getPhoneNumbers(
|
||||
userId: Long,
|
||||
purposeOfProcessing: PurposeOfProcessing,
|
||||
forceCarrierLookup: Option[Boolean] = None
|
||||
): Stitch[Seq[String]] = {
|
||||
val req = GetUserPhonesByUsersRequest(
|
||||
userIds = Seq(userId),
|
||||
forceCarrierLookup = forceCarrierLookup,
|
||||
purposesOfProcessing = Some(Seq(purposeOfProcessing))
|
||||
)
|
||||
|
||||
Stitch.callFuture(phoneStorageService.getUserPhonesByUsers(req)) map {
|
||||
_.userPhones.map(_.phoneNumber)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,12 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.phone_storage_service
|
||||
|
||||
import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient
|
||||
import com.twitter.follow_recommendations.common.clients.common.BaseClientModule
|
||||
import com.twitter.phonestorage.api.thriftscala.PhoneStorageService
|
||||
|
||||
object PhoneStorageServiceModule
|
||||
extends BaseClientModule[PhoneStorageService.MethodPerEndpoint]
|
||||
with MtlsClient {
|
||||
override val label = "phone-storage-service"
|
||||
override val dest = "/s/ibis-ds-api/ibis-ds-api:thrift2"
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"strato/config/columns/ml/featureStore:featureStore-strato-client",
|
||||
"strato/config/columns/onboarding/userrecs:userrecs-strato-client",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,14 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.real_time_real_graph
|
||||
|
||||
sealed trait EngagementType
|
||||
|
||||
// We do not include SoftFollow since it's deprecated
|
||||
object EngagementType {
|
||||
object Click extends EngagementType
|
||||
object Like extends EngagementType
|
||||
object Mention extends EngagementType
|
||||
object Retweet extends EngagementType
|
||||
object ProfileView extends EngagementType
|
||||
}
|
||||
|
||||
case class Engagement(engagementType: EngagementType, timestamp: Long)
|
||||
Binary file not shown.
@ -1,58 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.real_time_real_graph
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.util.Time
|
||||
|
||||
object EngagementScorer {
|
||||
private[real_time_real_graph] val MemoryDecayHalfLife = 24.hour
|
||||
private val ScoringFunctionBase = 0.5
|
||||
|
||||
def apply(
|
||||
engagements: Map[Long, Seq[Engagement]],
|
||||
engagementScoreMap: Map[EngagementType, Double],
|
||||
minScore: Double = 0.0
|
||||
): Seq[(Long, Double, Seq[EngagementType])] = {
|
||||
val now = Time.now
|
||||
engagements
|
||||
.mapValues { engags =>
|
||||
val totalScore = engags.map { engagement => score(engagement, now, engagementScoreMap) }.sum
|
||||
val engagementProof = getEngagementProof(engags, engagementScoreMap)
|
||||
(totalScore, engagementProof)
|
||||
}
|
||||
.collect { case (uid, (score, proof)) if score > minScore => (uid, score, proof) }
|
||||
.toSeq
|
||||
.sortBy(-_._2)
|
||||
}
|
||||
|
||||
/**
|
||||
* The engagement score is the base score decayed via timestamp, loosely model the human memory forgetting
|
||||
* curve, see https://en.wikipedia.org/wiki/Forgetting_curve
|
||||
*/
|
||||
private[real_time_real_graph] def score(
|
||||
engagement: Engagement,
|
||||
now: Time,
|
||||
engagementScoreMap: Map[EngagementType, Double]
|
||||
): Double = {
|
||||
val timeLapse = math.max(now.inMillis - engagement.timestamp, 0)
|
||||
val engagementScore = engagementScoreMap.getOrElse(engagement.engagementType, 0.0)
|
||||
engagementScore * math.pow(
|
||||
ScoringFunctionBase,
|
||||
timeLapse.toDouble / MemoryDecayHalfLife.inMillis)
|
||||
}
|
||||
|
||||
private def getEngagementProof(
|
||||
engagements: Seq[Engagement],
|
||||
engagementScoreMap: Map[EngagementType, Double]
|
||||
): Seq[EngagementType] = {
|
||||
|
||||
val filteredEngagement = engagements
|
||||
.collectFirst {
|
||||
case engagement
|
||||
if engagement.engagementType != EngagementType.Click
|
||||
&& engagementScoreMap.get(engagement.engagementType).exists(_ > 0.0) =>
|
||||
engagement.engagementType
|
||||
}
|
||||
|
||||
Seq(filteredEngagement.getOrElse(EngagementType.Click))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,128 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.real_time_real_graph
|
||||
|
||||
import com.google.inject.Inject
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.snowflake.id.SnowflakeId
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.strato.generated.client.ml.featureStore.TimelinesUserVertexOnUserClientColumn
|
||||
import com.twitter.strato.generated.client.onboarding.userrecs.RealGraphScoresMhOnUserClientColumn
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Time
|
||||
import com.twitter.wtf.real_time_interaction_graph.thriftscala._
|
||||
|
||||
@Singleton
|
||||
class RealTimeRealGraphClient @Inject() (
|
||||
timelinesUserVertexOnUserClientColumn: TimelinesUserVertexOnUserClientColumn,
|
||||
realGraphScoresMhOnUserClientColumn: RealGraphScoresMhOnUserClientColumn) {
|
||||
|
||||
def mapUserVertexToEngagementAndFilter(userVertex: UserVertex): Map[Long, Seq[Engagement]] = {
|
||||
val minTimestamp = (Time.now - RealTimeRealGraphClient.MaxEngagementAge).inMillis
|
||||
userVertex.outgoingInteractionMap.mapValues { interactions =>
|
||||
interactions
|
||||
.flatMap { interaction => RealTimeRealGraphClient.toEngagement(interaction) }.filter(
|
||||
_.timestamp >= minTimestamp)
|
||||
}.toMap
|
||||
}
|
||||
|
||||
def getRecentProfileViewEngagements(userId: Long): Stitch[Map[Long, Seq[Engagement]]] = {
|
||||
timelinesUserVertexOnUserClientColumn.fetcher
|
||||
.fetch(userId).map(_.v).map { input =>
|
||||
input
|
||||
.map { userVertex =>
|
||||
val targetToEngagements = mapUserVertexToEngagementAndFilter(userVertex)
|
||||
targetToEngagements.mapValues { engagements =>
|
||||
engagements.filter(engagement =>
|
||||
engagement.engagementType == EngagementType.ProfileView)
|
||||
}
|
||||
}.getOrElse(Map.empty)
|
||||
}
|
||||
}
|
||||
|
||||
def getUsersRecentlyEngagedWith(
|
||||
userId: Long,
|
||||
engagementScoreMap: Map[EngagementType, Double],
|
||||
includeDirectFollowCandidates: Boolean,
|
||||
includeNonDirectFollowCandidates: Boolean
|
||||
): Stitch[Seq[CandidateUser]] = {
|
||||
val isNewUser =
|
||||
SnowflakeId.timeFromIdOpt(userId).exists { signupTime =>
|
||||
(Time.now - signupTime) < RealTimeRealGraphClient.MaxNewUserAge
|
||||
}
|
||||
val updatedEngagementScoreMap =
|
||||
if (isNewUser)
|
||||
engagementScoreMap + (EngagementType.ProfileView -> RealTimeRealGraphClient.ProfileViewScore)
|
||||
else engagementScoreMap
|
||||
|
||||
Stitch
|
||||
.join(
|
||||
timelinesUserVertexOnUserClientColumn.fetcher.fetch(userId).map(_.v),
|
||||
realGraphScoresMhOnUserClientColumn.fetcher.fetch(userId).map(_.v)).map {
|
||||
case (Some(userVertex), Some(neighbors)) =>
|
||||
val engagements = mapUserVertexToEngagementAndFilter(userVertex)
|
||||
|
||||
val candidatesAndScores: Seq[(Long, Double, Seq[EngagementType])] =
|
||||
EngagementScorer.apply(engagements, engagementScoreMap = updatedEngagementScoreMap)
|
||||
|
||||
val directNeighbors = neighbors.candidates.map(_._1).toSet
|
||||
val (directFollows, nonDirectFollows) = candidatesAndScores
|
||||
.partition {
|
||||
case (id, _, _) => directNeighbors.contains(id)
|
||||
}
|
||||
|
||||
val candidates =
|
||||
(if (includeNonDirectFollowCandidates) nonDirectFollows else Seq.empty) ++
|
||||
(if (includeDirectFollowCandidates)
|
||||
directFollows.take(RealTimeRealGraphClient.MaxNumDirectFollow)
|
||||
else Seq.empty)
|
||||
|
||||
candidates.map {
|
||||
case (id, score, proof) =>
|
||||
CandidateUser(id, Some(score))
|
||||
}
|
||||
|
||||
case _ => Nil
|
||||
}
|
||||
}
|
||||
|
||||
def getRealGraphWeights(userId: Long): Stitch[Map[Long, Double]] =
|
||||
realGraphScoresMhOnUserClientColumn.fetcher
|
||||
.fetch(userId)
|
||||
.map(
|
||||
_.v
|
||||
.map(_.candidates.map(candidate => (candidate.userId, candidate.score)).toMap)
|
||||
.getOrElse(Map.empty[Long, Double]))
|
||||
}
|
||||
|
||||
object RealTimeRealGraphClient {
|
||||
private def toEngagement(interaction: Interaction): Option[Engagement] = {
|
||||
// We do not include SoftFollow since it's deprecated
|
||||
interaction match {
|
||||
case Interaction.Retweet(Retweet(timestamp)) =>
|
||||
Some(Engagement(EngagementType.Retweet, timestamp))
|
||||
case Interaction.Favorite(Favorite(timestamp)) =>
|
||||
Some(Engagement(EngagementType.Like, timestamp))
|
||||
case Interaction.Click(Click(timestamp)) => Some(Engagement(EngagementType.Click, timestamp))
|
||||
case Interaction.Mention(Mention(timestamp)) =>
|
||||
Some(Engagement(EngagementType.Mention, timestamp))
|
||||
case Interaction.ProfileView(ProfileView(timestamp)) =>
|
||||
Some(Engagement(EngagementType.ProfileView, timestamp))
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
|
||||
val MaxNumDirectFollow = 50
|
||||
val MaxEngagementAge: Duration = 14.days
|
||||
val MaxNewUserAge: Duration = 30.days
|
||||
val ProfileViewScore = 0.4
|
||||
val EngagementScoreMap = Map(
|
||||
EngagementType.Like -> 1.0,
|
||||
EngagementType.Retweet -> 1.0,
|
||||
EngagementType.Mention -> 1.0
|
||||
)
|
||||
val StrongEngagementScoreMap = Map(
|
||||
EngagementType.Like -> 1.0,
|
||||
EngagementType.Retweet -> 1.0,
|
||||
)
|
||||
}
|
||||
@ -1,26 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/github/nscala_time",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"escherbird/src/scala/com/twitter/escherbird/util/stitchcache",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"socialgraph/server/src/main/scala/com/twitter/socialgraph/util",
|
||||
"src/thrift/com/twitter/socialgraph:thrift-scala",
|
||||
"stitch/stitch-socialgraph",
|
||||
"strato/config/columns/onboarding/socialGraphService:socialGraphService-strato-client",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
"util/util-core:scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,421 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.socialgraph
|
||||
|
||||
import com.twitter.escherbird.util.stitchcache.StitchCache
|
||||
import com.twitter.finagle.stats.NullStatsReceiver
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.StatsUtil
|
||||
import com.twitter.follow_recommendations.common.models.FollowProof
|
||||
import com.twitter.follow_recommendations.common.models.UserIdWithTimestamp
|
||||
import com.twitter.inject.Logging
|
||||
import com.twitter.socialgraph.thriftscala.EdgesRequest
|
||||
import com.twitter.socialgraph.thriftscala.IdsRequest
|
||||
import com.twitter.socialgraph.thriftscala.IdsResult
|
||||
import com.twitter.socialgraph.thriftscala.LookupContext
|
||||
import com.twitter.socialgraph.thriftscala.OverCapacity
|
||||
import com.twitter.socialgraph.thriftscala.PageRequest
|
||||
import com.twitter.socialgraph.thriftscala.RelationshipType
|
||||
import com.twitter.socialgraph.thriftscala.SrcRelationship
|
||||
import com.twitter.socialgraph.util.ByteBufferUtil
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.socialgraph.SocialGraph
|
||||
import com.twitter.strato.client.Fetcher
|
||||
import com.twitter.strato.generated.client.onboarding.socialGraphService.IdsClientColumn
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Time
|
||||
import java.nio.ByteBuffer
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
case class RecentEdgesQuery(
|
||||
userId: Long,
|
||||
relations: Seq[RelationshipType],
|
||||
// prefer to default value to better utilize the caching function of stitch
|
||||
count: Option[Int] = Some(SocialGraphClient.MaxQuerySize),
|
||||
performUnion: Boolean = true,
|
||||
recentEdgesWindowOpt: Option[Duration] = None,
|
||||
targets: Option[Seq[Long]] = None)
|
||||
|
||||
case class EdgeRequestQuery(
|
||||
userId: Long,
|
||||
relation: RelationshipType,
|
||||
count: Option[Int] = Some(SocialGraphClient.MaxQuerySize),
|
||||
performUnion: Boolean = true,
|
||||
recentEdgesWindowOpt: Option[Duration] = None,
|
||||
targets: Option[Seq[Long]] = None)
|
||||
|
||||
@Singleton
|
||||
class SocialGraphClient @Inject() (
|
||||
socialGraph: SocialGraph,
|
||||
idsClientColumn: IdsClientColumn,
|
||||
statsReceiver: StatsReceiver = NullStatsReceiver)
|
||||
extends Logging {
|
||||
|
||||
private val stats = statsReceiver.scope(this.getClass.getSimpleName)
|
||||
private val cacheStats = stats.scope("cache")
|
||||
private val getIntersectionsStats = stats.scope("getIntersections")
|
||||
private val getIntersectionsFromCachedColumnStats =
|
||||
stats.scope("getIntersectionsFromCachedColumn")
|
||||
private val getRecentEdgesStats = stats.scope("getRecentEdges")
|
||||
private val getRecentEdgesCachedStats = stats.scope("getRecentEdgesCached")
|
||||
private val getRecentEdgesFromCachedColumnStats = stats.scope("getRecentEdgesFromCachedColumn")
|
||||
private val getRecentEdgesCachedInternalStats = stats.scope("getRecentEdgesCachedInternal")
|
||||
private val getRecentEdgesWithTimeStats = stats.scope("getRecentEdgesWithTime")
|
||||
|
||||
val sgsIdsFetcher: Fetcher[IdsRequest, Unit, IdsResult] = idsClientColumn.fetcher
|
||||
|
||||
private val recentEdgesCache = StitchCache[RecentEdgesQuery, Seq[Long]](
|
||||
maxCacheSize = SocialGraphClient.MaxCacheSize,
|
||||
ttl = SocialGraphClient.CacheTTL,
|
||||
statsReceiver = cacheStats,
|
||||
underlyingCall = getRecentEdges
|
||||
)
|
||||
|
||||
def getRecentEdgesCached(
|
||||
rq: RecentEdgesQuery,
|
||||
useCachedStratoColumn: Boolean = true
|
||||
): Stitch[Seq[Long]] = {
|
||||
getRecentEdgesCachedStats.counter("requests").incr()
|
||||
if (useCachedStratoColumn) {
|
||||
getRecentEdgesFromCachedColumn(rq)
|
||||
} else {
|
||||
StatsUtil.profileStitch(
|
||||
getRecentEdgesCachedInternal(rq),
|
||||
getRecentEdgesCachedInternalStats
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
def getRecentEdgesCachedInternal(rq: RecentEdgesQuery): Stitch[Seq[Long]] = {
|
||||
recentEdgesCache.readThrough(rq)
|
||||
}
|
||||
|
||||
def getRecentEdgesFromCachedColumn(rq: RecentEdgesQuery): Stitch[Seq[Long]] = {
|
||||
val pageRequest = rq.recentEdgesWindowOpt match {
|
||||
case Some(recentEdgesWindow) =>
|
||||
PageRequest(
|
||||
count = rq.count,
|
||||
cursor = Some(getEdgeCursor(recentEdgesWindow)),
|
||||
selectAll = Some(true)
|
||||
)
|
||||
case _ => PageRequest(count = rq.count)
|
||||
}
|
||||
val idsRequest = IdsRequest(
|
||||
rq.relations.map { relationshipType =>
|
||||
SrcRelationship(
|
||||
source = rq.userId,
|
||||
relationshipType = relationshipType,
|
||||
targets = rq.targets
|
||||
)
|
||||
},
|
||||
pageRequest = Some(pageRequest),
|
||||
context = Some(LookupContext(performUnion = Some(rq.performUnion)))
|
||||
)
|
||||
|
||||
val socialGraphStitch = sgsIdsFetcher
|
||||
.fetch(idsRequest, Unit)
|
||||
.map(_.v)
|
||||
.map { result =>
|
||||
result
|
||||
.map { idResult =>
|
||||
val userIds: Seq[Long] = idResult.ids
|
||||
getRecentEdgesFromCachedColumnStats.stat("num_edges").add(userIds.size)
|
||||
userIds
|
||||
}.getOrElse(Seq.empty)
|
||||
}
|
||||
.rescue {
|
||||
case e: Exception =>
|
||||
stats.counter(e.getClass.getSimpleName).incr()
|
||||
Stitch.Nil
|
||||
}
|
||||
|
||||
StatsUtil.profileStitch(
|
||||
socialGraphStitch,
|
||||
getRecentEdgesFromCachedColumnStats
|
||||
)
|
||||
}
|
||||
|
||||
def getRecentEdges(rq: RecentEdgesQuery): Stitch[Seq[Long]] = {
|
||||
val pageRequest = rq.recentEdgesWindowOpt match {
|
||||
case Some(recentEdgesWindow) =>
|
||||
PageRequest(
|
||||
count = rq.count,
|
||||
cursor = Some(getEdgeCursor(recentEdgesWindow)),
|
||||
selectAll = Some(true)
|
||||
)
|
||||
case _ => PageRequest(count = rq.count)
|
||||
}
|
||||
val socialGraphStitch = socialGraph
|
||||
.ids(
|
||||
IdsRequest(
|
||||
rq.relations.map { relationshipType =>
|
||||
SrcRelationship(
|
||||
source = rq.userId,
|
||||
relationshipType = relationshipType,
|
||||
targets = rq.targets
|
||||
)
|
||||
},
|
||||
pageRequest = Some(pageRequest),
|
||||
context = Some(LookupContext(performUnion = Some(rq.performUnion)))
|
||||
)
|
||||
)
|
||||
.map { idsResult =>
|
||||
val userIds: Seq[Long] = idsResult.ids
|
||||
getRecentEdgesStats.stat("num_edges").add(userIds.size)
|
||||
userIds
|
||||
}
|
||||
.rescue {
|
||||
case e: OverCapacity =>
|
||||
stats.counter(e.getClass.getSimpleName).incr()
|
||||
logger.warn("SGS Over Capacity", e)
|
||||
Stitch.Nil
|
||||
}
|
||||
StatsUtil.profileStitch(
|
||||
socialGraphStitch,
|
||||
getRecentEdgesStats
|
||||
)
|
||||
}
|
||||
|
||||
// This method return recent edges of (userId, timeInMs)
|
||||
def getRecentEdgesWithTime(rq: EdgeRequestQuery): Stitch[Seq[UserIdWithTimestamp]] = {
|
||||
val pageRequest = rq.recentEdgesWindowOpt match {
|
||||
case Some(recentEdgesWindow) =>
|
||||
PageRequest(
|
||||
count = rq.count,
|
||||
cursor = Some(getEdgeCursor(recentEdgesWindow)),
|
||||
selectAll = Some(true)
|
||||
)
|
||||
case _ => PageRequest(count = rq.count)
|
||||
}
|
||||
|
||||
val socialGraphStitch = socialGraph
|
||||
.edges(
|
||||
EdgesRequest(
|
||||
SrcRelationship(
|
||||
source = rq.userId,
|
||||
relationshipType = rq.relation,
|
||||
targets = rq.targets
|
||||
),
|
||||
pageRequest = Some(pageRequest),
|
||||
context = Some(LookupContext(performUnion = Some(rq.performUnion)))
|
||||
)
|
||||
)
|
||||
.map { edgesResult =>
|
||||
val userIds = edgesResult.edges.map { socialEdge =>
|
||||
UserIdWithTimestamp(socialEdge.target, socialEdge.updatedAt)
|
||||
}
|
||||
getRecentEdgesWithTimeStats.stat("num_edges").add(userIds.size)
|
||||
userIds
|
||||
}
|
||||
.rescue {
|
||||
case e: OverCapacity =>
|
||||
stats.counter(e.getClass.getSimpleName).incr()
|
||||
logger.warn("SGS Over Capacity", e)
|
||||
Stitch.Nil
|
||||
}
|
||||
StatsUtil.profileStitch(
|
||||
socialGraphStitch,
|
||||
getRecentEdgesWithTimeStats
|
||||
)
|
||||
}
|
||||
|
||||
// This method returns the cursor for a time duration, such that all the edges returned by SGS will be created
|
||||
// in the range (now-window, now)
|
||||
def getEdgeCursor(window: Duration): ByteBuffer = {
|
||||
val cursorInLong = (-(Time.now - window).inMilliseconds) << 20
|
||||
ByteBufferUtil.fromLong(cursorInLong)
|
||||
}
|
||||
|
||||
// notice that this is more expensive but more realtime than the GFS one
|
||||
def getIntersections(
|
||||
userId: Long,
|
||||
candidateIds: Seq[Long],
|
||||
numIntersectionIds: Int
|
||||
): Stitch[Map[Long, FollowProof]] = {
|
||||
val socialGraphStitch: Stitch[Map[Long, FollowProof]] = Stitch
|
||||
.collect(candidateIds.map { candidateId =>
|
||||
socialGraph
|
||||
.ids(
|
||||
IdsRequest(
|
||||
Seq(
|
||||
SrcRelationship(userId, RelationshipType.Following),
|
||||
SrcRelationship(candidateId, RelationshipType.FollowedBy)
|
||||
),
|
||||
pageRequest = Some(PageRequest(count = Some(numIntersectionIds)))
|
||||
)
|
||||
).map { idsResult =>
|
||||
getIntersectionsStats.stat("num_edges").add(idsResult.ids.size)
|
||||
(candidateId -> FollowProof(idsResult.ids, idsResult.ids.size))
|
||||
}
|
||||
}).map(_.toMap)
|
||||
.rescue {
|
||||
case e: OverCapacity =>
|
||||
stats.counter(e.getClass.getSimpleName).incr()
|
||||
logger.warn("social graph over capacity in hydrating social proof", e)
|
||||
Stitch.value(Map.empty)
|
||||
}
|
||||
StatsUtil.profileStitch(
|
||||
socialGraphStitch,
|
||||
getIntersectionsStats
|
||||
)
|
||||
}
|
||||
|
||||
def getIntersectionsFromCachedColumn(
|
||||
userId: Long,
|
||||
candidateIds: Seq[Long],
|
||||
numIntersectionIds: Int
|
||||
): Stitch[Map[Long, FollowProof]] = {
|
||||
val socialGraphStitch: Stitch[Map[Long, FollowProof]] = Stitch
|
||||
.collect(candidateIds.map { candidateId =>
|
||||
val idsRequest = IdsRequest(
|
||||
Seq(
|
||||
SrcRelationship(userId, RelationshipType.Following),
|
||||
SrcRelationship(candidateId, RelationshipType.FollowedBy)
|
||||
),
|
||||
pageRequest = Some(PageRequest(count = Some(numIntersectionIds)))
|
||||
)
|
||||
|
||||
sgsIdsFetcher
|
||||
.fetch(idsRequest, Unit)
|
||||
.map(_.v)
|
||||
.map { resultOpt =>
|
||||
resultOpt.map { idsResult =>
|
||||
getIntersectionsFromCachedColumnStats.stat("num_edges").add(idsResult.ids.size)
|
||||
candidateId -> FollowProof(idsResult.ids, idsResult.ids.size)
|
||||
}
|
||||
}
|
||||
}).map(_.flatten.toMap)
|
||||
.rescue {
|
||||
case e: Exception =>
|
||||
stats.counter(e.getClass.getSimpleName).incr()
|
||||
Stitch.value(Map.empty)
|
||||
}
|
||||
StatsUtil.profileStitch(
|
||||
socialGraphStitch,
|
||||
getIntersectionsFromCachedColumnStats
|
||||
)
|
||||
}
|
||||
|
||||
def getInvalidRelationshipUserIds(
|
||||
userId: Long,
|
||||
maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship
|
||||
): Stitch[Seq[Long]] = {
|
||||
getRecentEdges(
|
||||
RecentEdgesQuery(
|
||||
userId,
|
||||
SocialGraphClient.InvalidRelationshipTypes,
|
||||
Some(maxNumRelationship)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def getInvalidRelationshipUserIdsFromCachedColumn(
|
||||
userId: Long,
|
||||
maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship
|
||||
): Stitch[Seq[Long]] = {
|
||||
getRecentEdgesFromCachedColumn(
|
||||
RecentEdgesQuery(
|
||||
userId,
|
||||
SocialGraphClient.InvalidRelationshipTypes,
|
||||
Some(maxNumRelationship)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def getRecentFollowedUserIds(userId: Long): Stitch[Seq[Long]] = {
|
||||
getRecentEdges(
|
||||
RecentEdgesQuery(
|
||||
userId,
|
||||
Seq(RelationshipType.Following)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def getRecentFollowedUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] = {
|
||||
getRecentEdgesFromCachedColumn(
|
||||
RecentEdgesQuery(
|
||||
userId,
|
||||
Seq(RelationshipType.Following)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def getRecentFollowedUserIdsWithTime(userId: Long): Stitch[Seq[UserIdWithTimestamp]] = {
|
||||
getRecentEdgesWithTime(
|
||||
EdgeRequestQuery(
|
||||
userId,
|
||||
RelationshipType.Following
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def getRecentFollowedByUserIds(userId: Long): Stitch[Seq[Long]] = {
|
||||
getRecentEdges(
|
||||
RecentEdgesQuery(
|
||||
userId,
|
||||
Seq(RelationshipType.FollowedBy)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def getRecentFollowedByUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] = {
|
||||
getRecentEdgesFromCachedColumn(
|
||||
RecentEdgesQuery(
|
||||
userId,
|
||||
Seq(RelationshipType.FollowedBy)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def getRecentFollowedUserIdsWithTimeWindow(
|
||||
userId: Long,
|
||||
timeWindow: Duration
|
||||
): Stitch[Seq[Long]] = {
|
||||
getRecentEdges(
|
||||
RecentEdgesQuery(
|
||||
userId,
|
||||
Seq(RelationshipType.Following),
|
||||
recentEdgesWindowOpt = Some(timeWindow)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
object SocialGraphClient {
|
||||
|
||||
val MaxQuerySize: Int = 500
|
||||
val MaxCacheSize: Int = 5000000
|
||||
// Ref: src/thrift/com/twitter/socialgraph/social_graph_service.thrift
|
||||
val MaxNumInvalidRelationship: Int = 5000
|
||||
val CacheTTL: Duration = Duration.fromHours(24)
|
||||
|
||||
val InvalidRelationshipTypes: Seq[RelationshipType] = Seq(
|
||||
RelationshipType.HideRecommendations,
|
||||
RelationshipType.Blocking,
|
||||
RelationshipType.BlockedBy,
|
||||
RelationshipType.Muting,
|
||||
RelationshipType.MutedBy,
|
||||
RelationshipType.ReportedAsSpam,
|
||||
RelationshipType.ReportedAsSpamBy,
|
||||
RelationshipType.ReportedAsAbuse,
|
||||
RelationshipType.ReportedAsAbuseBy,
|
||||
RelationshipType.FollowRequestOutgoing,
|
||||
RelationshipType.Following,
|
||||
RelationshipType.UsedToFollow,
|
||||
)
|
||||
|
||||
/**
|
||||
*
|
||||
* Whether to call SGS to validate each candidate based on the number of invalid relationship users
|
||||
* prefetched during request building step. This aims to not omit any invalid candidates that are
|
||||
* not filtered out in previous steps.
|
||||
* If the number is 0, this might be a fail-opened SGS call.
|
||||
* If the number is larger or equal to 5000, this could hit SGS page size limit.
|
||||
* Both cases account for a small percentage of the total traffic (<5%).
|
||||
*
|
||||
* @param numInvalidRelationshipUsers number of invalid relationship users fetched from getInvalidRelationshipUserIds
|
||||
* @return whether to enable post-ranker SGS predicate
|
||||
*/
|
||||
def enablePostRankerSgsPredicate(numInvalidRelationshipUsers: Int): Boolean = {
|
||||
numInvalidRelationshipUsers == 0 || numInvalidRelationshipUsers >= MaxNumInvalidRelationship
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,25 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.socialgraph
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.finagle.ThriftMux
|
||||
import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient
|
||||
import com.twitter.follow_recommendations.common.clients.common.BaseClientModule
|
||||
import com.twitter.socialgraph.thriftscala.SocialGraphService
|
||||
import com.twitter.stitch.socialgraph.SocialGraph
|
||||
import javax.inject.Singleton
|
||||
|
||||
object SocialGraphModule
|
||||
extends BaseClientModule[SocialGraphService.MethodPerEndpoint]
|
||||
with MtlsClient {
|
||||
override val label = "social-graph-service"
|
||||
override val dest = "/s/socialgraph/socialgraph"
|
||||
|
||||
override def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client =
|
||||
client.withSessionQualifier.noFailFast
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesStitchClient(futureIface: SocialGraphService.MethodPerEndpoint): SocialGraph = {
|
||||
SocialGraph(futureIface)
|
||||
}
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants",
|
||||
"src/scala/com/twitter/onboarding/relevance/candidate_generation/utt/models",
|
||||
"src/thrift/com/twitter/core_workflows/user_model:user_model-scala",
|
||||
"src/thrift/com/twitter/frigate/data_pipeline:frigate-user-history-thrift-scala",
|
||||
"src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
|
||||
"src/thrift/com/twitter/hermit/pop_geo:hermit-pop-geo-scala",
|
||||
"src/thrift/com/twitter/onboarding/relevance/relatable_accounts:relatable_accounts-scala",
|
||||
"src/thrift/com/twitter/onboarding/relevance/store:store-scala",
|
||||
"src/thrift/com/twitter/recos/user_user_graph:user_user_graph-scala",
|
||||
"src/thrift/com/twitter/search/account_search/extended_network:extended_network_users-scala",
|
||||
"src/thrift/com/twitter/service/metastore/gen:thrift-scala",
|
||||
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
|
||||
"src/thrift/com/twitter/wtf/ml:wtf-ml-output-thrift-scala",
|
||||
"src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-scala",
|
||||
"src/thrift/com/twitter/wtf/triangular_loop:triangular_loop-scala",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,249 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.strato
|
||||
|
||||
import com.google.inject.name.Named
|
||||
import com.google.inject.Provides
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState
|
||||
import com.twitter.search.account_search.extended_network.thriftscala.ExtendedNetworkUserKey
|
||||
import com.twitter.search.account_search.extended_network.thriftscala.ExtendedNetworkUserVal
|
||||
import com.twitter.finagle.ThriftMux
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.thrift.Protocols
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants
|
||||
import com.twitter.follow_recommendations.common.constants.ServiceConstants._
|
||||
import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.LatestEvents
|
||||
import com.twitter.hermit.candidate.thriftscala.{Candidates => HermitCandidates}
|
||||
import com.twitter.hermit.pop_geo.thriftscala.PopUsersInPlace
|
||||
import com.twitter.onboarding.relevance.relatable_accounts.thriftscala.RelatableAccounts
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.onboarding.relevance.candidates.thriftscala.InterestBasedUserRecommendations
|
||||
import com.twitter.onboarding.relevance.candidates.thriftscala.UTTInterest
|
||||
import com.twitter.onboarding.relevance.store.thriftscala.WhoToFollowDismissEventDetails
|
||||
import com.twitter.recos.user_user_graph.thriftscala.RecommendUserRequest
|
||||
import com.twitter.recos.user_user_graph.thriftscala.RecommendUserResponse
|
||||
import com.twitter.service.metastore.gen.thriftscala.UserRecommendabilityFeatures
|
||||
import com.twitter.strato.catalog.Scan.Slice
|
||||
import com.twitter.strato.client.Strato.{Client => StratoClient}
|
||||
import com.twitter.strato.client.Client
|
||||
import com.twitter.strato.client.Fetcher
|
||||
import com.twitter.strato.client.Scanner
|
||||
import com.twitter.strato.thrift.ScroogeConvImplicits._
|
||||
import com.twitter.wtf.candidate.thriftscala.CandidateSeq
|
||||
import com.twitter.wtf.ml.thriftscala.CandidateFeatures
|
||||
import com.twitter.wtf.real_time_interaction_graph.thriftscala.Interaction
|
||||
import com.twitter.wtf.triangular_loop.thriftscala.{Candidates => TriangularLoopCandidates}
|
||||
import com.twitter.strato.opcontext.Attribution._
|
||||
|
||||
object StratoClientModule extends TwitterModule {
|
||||
|
||||
// column paths
|
||||
val CosineFollowPath = "recommendations/similarity/similarUsersByFollowGraph.User"
|
||||
val CosineListPath = "recommendations/similarity/similarUsersByListGraph.User"
|
||||
val CuratedCandidatesPath = "onboarding/curatedAccounts"
|
||||
val CuratedFilteredAccountsPath = "onboarding/filteredAccountsFromRecommendations"
|
||||
val PopUsersInPlacePath = "onboarding/userrecs/popUsersInPlace"
|
||||
val ProfileSidebarBlacklistPath = "recommendations/hermit/profile-sidebar-blacklist"
|
||||
val RealTimeInteractionsPath = "hmli/realTimeInteractions"
|
||||
val SimsPath = "recommendations/similarity/similarUsersBySims.User"
|
||||
val DBV2SimsPath = "onboarding/userrecs/newSims.User"
|
||||
val TriangularLoopsPath = "onboarding/userrecs/triangularLoops.User"
|
||||
val TwoHopRandomWalkPath = "onboarding/userrecs/twoHopRandomWalk.User"
|
||||
val UserRecommendabilityPath = "onboarding/userRecommendabilityWithLongKeys.User"
|
||||
val UTTAccountRecommendationsPath = "onboarding/userrecs/utt_account_recommendations"
|
||||
val UttSeedAccountsRecommendationPath = "onboarding/userrecs/utt_seed_accounts"
|
||||
val UserStatePath = "onboarding/userState.User"
|
||||
val WTFPostNuxFeaturesPath = "ml/featureStore/onboarding/wtfPostNuxFeatures.User"
|
||||
val ElectionCandidatesPath = "onboarding/electionAccounts"
|
||||
val UserUserGraphPath = "recommendations/userUserGraph"
|
||||
val WtfDissmissEventsPath = "onboarding/wtfDismissEvents"
|
||||
val RelatableAccountsPath = "onboarding/userrecs/relatableAccounts"
|
||||
val ExtendedNetworkCandidatesPath = "search/account_search/extendedNetworkCandidatesMH"
|
||||
val LabeledNotificationPath = "frigate/magicrecs/labeledPushRecsAggregated.User"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def stratoClient(serviceIdentifier: ServiceIdentifier): Client = {
|
||||
val timeoutBudget = 500.milliseconds
|
||||
StratoClient(
|
||||
ThriftMux.client
|
||||
.withRequestTimeout(timeoutBudget)
|
||||
.withProtocolFactory(Protocols.binaryFactory(
|
||||
stringLengthLimit = StringLengthLimit,
|
||||
containerLengthLimit = ContainerLengthLimit)))
|
||||
.withMutualTls(serviceIdentifier)
|
||||
.build()
|
||||
}
|
||||
|
||||
// add strato putters, fetchers, scanners below:
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.COSINE_FOLLOW_FETCHER)
|
||||
def cosineFollowFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] =
|
||||
stratoClient.fetcher[Long, Unit, HermitCandidates](CosineFollowPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.COSINE_LIST_FETCHER)
|
||||
def cosineListFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] =
|
||||
stratoClient.fetcher[Long, Unit, HermitCandidates](CosineListPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER)
|
||||
def curatedBlacklistedAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] =
|
||||
stratoClient.fetcher[String, Unit, Seq[Long]](CuratedFilteredAccountsPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.CURATED_CANDIDATES_FETCHER)
|
||||
def curatedCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] =
|
||||
stratoClient.fetcher[String, Unit, Seq[Long]](CuratedCandidatesPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER)
|
||||
def popUsersInPlaceFetcher(stratoClient: Client): Fetcher[String, Unit, PopUsersInPlace] =
|
||||
stratoClient.fetcher[String, Unit, PopUsersInPlace](PopUsersInPlacePath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.RELATABLE_ACCOUNTS_FETCHER)
|
||||
def relatableAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, RelatableAccounts] =
|
||||
stratoClient.fetcher[String, Unit, RelatableAccounts](RelatableAccountsPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.PROFILE_SIDEBAR_BLACKLIST_SCANNER)
|
||||
def profileSidebarBlacklistScanner(
|
||||
stratoClient: Client
|
||||
): Scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit] =
|
||||
stratoClient.scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit](ProfileSidebarBlacklistPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.REAL_TIME_INTERACTIONS_FETCHER)
|
||||
def realTimeInteractionsFetcher(
|
||||
stratoClient: Client
|
||||
): Fetcher[(Long, Long), Unit, Seq[Interaction]] =
|
||||
stratoClient.fetcher[(Long, Long), Unit, Seq[Interaction]](RealTimeInteractionsPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.SIMS_FETCHER)
|
||||
def simsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] =
|
||||
stratoClient.fetcher[Long, Unit, HermitCandidates](SimsPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.DBV2_SIMS_FETCHER)
|
||||
def dbv2SimsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] =
|
||||
stratoClient.fetcher[Long, Unit, HermitCandidates](DBV2SimsPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.TRIANGULAR_LOOPS_FETCHER)
|
||||
def triangularLoopsFetcher(stratoClient: Client): Fetcher[Long, Unit, TriangularLoopCandidates] =
|
||||
stratoClient.fetcher[Long, Unit, TriangularLoopCandidates](TriangularLoopsPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER)
|
||||
def twoHopRandomWalkFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateSeq] =
|
||||
stratoClient.fetcher[Long, Unit, CandidateSeq](TwoHopRandomWalkPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER)
|
||||
def userRecommendabilityFetcher(
|
||||
stratoClient: Client
|
||||
): Fetcher[Long, Unit, UserRecommendabilityFeatures] =
|
||||
stratoClient.fetcher[Long, Unit, UserRecommendabilityFeatures](UserRecommendabilityPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.USER_STATE_FETCHER)
|
||||
def userStateFetcher(stratoClient: Client): Fetcher[Long, Unit, CondensedUserState] =
|
||||
stratoClient.fetcher[Long, Unit, CondensedUserState](UserStatePath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.UTT_ACCOUNT_RECOMMENDATIONS_FETCHER)
|
||||
def uttAccountRecommendationsFetcher(
|
||||
stratoClient: Client
|
||||
): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] =
|
||||
stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations](
|
||||
UTTAccountRecommendationsPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.UTT_SEED_ACCOUNTS_FETCHER)
|
||||
def uttSeedAccountRecommendationsFetcher(
|
||||
stratoClient: Client
|
||||
): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] =
|
||||
stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations](
|
||||
UttSeedAccountsRecommendationPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.ELECTION_CANDIDATES_FETCHER)
|
||||
def electionCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] =
|
||||
stratoClient.fetcher[String, Unit, Seq[Long]](ElectionCandidatesPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER)
|
||||
def userUserGraphFetcher(
|
||||
stratoClient: Client
|
||||
): Fetcher[RecommendUserRequest, Unit, RecommendUserResponse] =
|
||||
stratoClient.fetcher[RecommendUserRequest, Unit, RecommendUserResponse](UserUserGraphPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.POST_NUX_WTF_FEATURES_FETCHER)
|
||||
def wtfPostNuxFeaturesFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateFeatures] = {
|
||||
val attribution = ManhattanAppId("starbuck", "wtf_starbuck")
|
||||
stratoClient
|
||||
.fetcher[Long, Unit, CandidateFeatures](WTFPostNuxFeaturesPath)
|
||||
.withAttribution(attribution)
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.EXTENDED_NETWORK)
|
||||
def extendedNetworkFetcher(
|
||||
stratoClient: Client
|
||||
): Fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal] = {
|
||||
stratoClient
|
||||
.fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal](ExtendedNetworkCandidatesPath)
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.DISMISS_STORE_SCANNER)
|
||||
def dismissStoreScanner(
|
||||
stratoClient: Client
|
||||
): Scanner[
|
||||
(Long, Slice[(Long, Long)]),
|
||||
Unit,
|
||||
(Long, (Long, Long)),
|
||||
WhoToFollowDismissEventDetails
|
||||
] =
|
||||
stratoClient.scanner[
|
||||
(Long, Slice[(Long, Long)]), // PKEY: userId, LKEY: (-ts, candidateId)
|
||||
Unit,
|
||||
(Long, (Long, Long)),
|
||||
WhoToFollowDismissEventDetails
|
||||
](WtfDissmissEventsPath)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.LABELED_NOTIFICATION_FETCHER)
|
||||
def labeledNotificationFetcher(
|
||||
stratoClient: Client
|
||||
): Fetcher[Long, Unit, LatestEvents] = {
|
||||
stratoClient
|
||||
.fetcher[Long, Unit, LatestEvents](LabeledNotificationPath)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders",
|
||||
"stitch/stitch-core",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
"user-signal-service/thrift/src/main/thrift:thrift-scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,83 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.clients.user_state
|
||||
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState
|
||||
import com.twitter.core_workflows.user_model.thriftscala.UserState
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.decider.RandomRecipient
|
||||
import com.twitter.finagle.Memcached.Client
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.util.DefaultTimer
|
||||
import com.twitter.follow_recommendations.common.base.StatsUtil
|
||||
import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient
|
||||
import com.twitter.follow_recommendations.common.clients.cache.ThriftEnumOptionBijection
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants
|
||||
import com.twitter.follow_recommendations.configapi.deciders.DeciderKey
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.strato.client.Fetcher
|
||||
import com.twitter.util.Duration
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
import java.lang.{Long => JLong}
|
||||
|
||||
@Singleton
|
||||
class UserStateClient @Inject() (
|
||||
@Named(GuiceNamedConstants.USER_STATE_FETCHER) userStateFetcher: Fetcher[
|
||||
Long,
|
||||
Unit,
|
||||
CondensedUserState
|
||||
],
|
||||
client: Client,
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider = Decider.False) {
|
||||
|
||||
private val stats: StatsReceiver = statsReceiver.scope("user_state_client")
|
||||
|
||||
// client to memcache cluster
|
||||
val bijection = new ThriftEnumOptionBijection[UserState](UserState.apply)
|
||||
val memcacheClient = MemcacheClient[Option[UserState]](
|
||||
client = client,
|
||||
dest = "/s/cache/follow_recos_service:twemcaches",
|
||||
valueBijection = bijection,
|
||||
ttl = UserStateClient.CacheTTL,
|
||||
statsReceiver = stats.scope("twemcache")
|
||||
)
|
||||
|
||||
def getUserState(userId: Long): Stitch[Option[UserState]] = {
|
||||
val deciderKey: String = DeciderKey.EnableDistributedCaching.toString
|
||||
val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient))
|
||||
val userStateStitch: Stitch[Option[UserState]] =
|
||||
enableDistributedCaching match {
|
||||
// read from memcache
|
||||
case true => memcacheClient.readThrough(
|
||||
// add a key prefix to address cache key collisions
|
||||
key = "UserStateClient" + userId.toString,
|
||||
underlyingCall = () => fetchUserState(userId)
|
||||
)
|
||||
case false => fetchUserState(userId)
|
||||
}
|
||||
val userStateStitchWithTimeout: Stitch[Option[UserState]] =
|
||||
userStateStitch
|
||||
// set a 150ms timeout limit for user state fetches
|
||||
.within(150.milliseconds)(DefaultTimer)
|
||||
.rescue {
|
||||
case e: Exception =>
|
||||
stats.scope("rescued").counter(e.getClass.getSimpleName).incr()
|
||||
Stitch(None)
|
||||
}
|
||||
// profile the latency of stitch call and return the result
|
||||
StatsUtil.profileStitch(
|
||||
userStateStitchWithTimeout,
|
||||
stats.scope("getUserState")
|
||||
)
|
||||
}
|
||||
|
||||
def fetchUserState(userId: JLong): Stitch[Option[UserState]] = {
|
||||
userStateFetcher.fetch(userId).map(_.v.flatMap(_.userState))
|
||||
}
|
||||
}
|
||||
|
||||
object UserStateClient {
|
||||
val CacheTTL: Duration = Duration.fromHours(6)
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,91 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.constants
|
||||
|
||||
import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap
|
||||
import com.twitter.hermit.model.Algorithm._
|
||||
import com.twitter.follow_recommendations.common.models.AlgorithmType
|
||||
|
||||
object CandidateAlgorithmTypeConstants {
|
||||
|
||||
/**
|
||||
* Each algorithm is based on one, or more, of the 4 types of information we have on users,
|
||||
* described in [[AlgorithmType]]. Assignment of algorithms to these categories are based on
|
||||
*/
|
||||
private val AlgorithmIdToType: Map[String, Set[AlgorithmType.Value]] = Map(
|
||||
// Activity Algorithms:
|
||||
AlgorithmToFeedbackTokenMap(NewFollowingSimilarUser).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(Sims).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(NewFollowingSimilarUserSalsa).toString -> Set(
|
||||
AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(RecentEngagementNonDirectFollow).toString -> Set(
|
||||
AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(RecentEngagementSimilarUser).toString -> Set(
|
||||
AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(RecentEngagementSarusOcCur).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(RecentSearchBasedRec).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(TwistlyTweetAuthors).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(Follow2VecNearestNeighbors).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(EmailTweetClick).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(RepeatedProfileVisits).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(GoodTweetClickEngagements).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(TweetShareEngagements).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(TweetSharerToShareRecipientEngagements).toString -> Set(
|
||||
AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(TweetAuthorToShareRecipientEngagements).toString -> Set(
|
||||
AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(LinearRegressionFollow2VecNearestNeighbors).toString -> Set(
|
||||
AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(NUXLOHistory).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(TrafficAttributionAccounts).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(RealGraphOonV2).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(MagicRecsRecentEngagements).toString -> Set(AlgorithmType.Activity),
|
||||
AlgorithmToFeedbackTokenMap(NotificationEngagement).toString -> Set(AlgorithmType.Activity),
|
||||
// Social Algorithms:
|
||||
AlgorithmToFeedbackTokenMap(TwoHopRandomWalk).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(RealTimeMutualFollow).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(ForwardPhoneBook).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(ForwardEmailBook).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(NewFollowingNewFollowingExpansion).toString -> Set(
|
||||
AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(NewFollowingSarusCoOccurSocialProof).toString -> Set(
|
||||
AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(ReverseEmailBookIbis).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(ReversePhoneBook).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(StrongTiePredictionRec).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(StrongTiePredictionRecWithSocialProof).toString -> Set(
|
||||
AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRec).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRecNoCaching).toString -> Set(
|
||||
AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(TriangularLoop).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(StrongTiePredictionPmi).toString -> Set(AlgorithmType.Social),
|
||||
AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRAB).toString -> Set(AlgorithmType.Social),
|
||||
// Geo Algorithms:
|
||||
AlgorithmToFeedbackTokenMap(PopCountryBackFill).toString -> Set(AlgorithmType.Geo),
|
||||
AlgorithmToFeedbackTokenMap(PopCountry).toString -> Set(AlgorithmType.Geo),
|
||||
AlgorithmToFeedbackTokenMap(PopGeohash).toString -> Set(AlgorithmType.Geo),
|
||||
// AlgorithmToFeedbackTokenMap(PopGeohashRealGraph).toString -> Set(AlgorithmType.Geo),
|
||||
AlgorithmToFeedbackTokenMap(EngagedFollowerRatio).toString -> Set(AlgorithmType.Geo),
|
||||
AlgorithmToFeedbackTokenMap(CrowdSearchAccounts).toString -> Set(AlgorithmType.Geo),
|
||||
AlgorithmToFeedbackTokenMap(OrganicFollowAccounts).toString -> Set(AlgorithmType.Geo),
|
||||
AlgorithmToFeedbackTokenMap(PopGeohashQualityFollow).toString -> Set(AlgorithmType.Geo),
|
||||
AlgorithmToFeedbackTokenMap(PPMILocaleFollow).toString -> Set(AlgorithmType.Geo),
|
||||
// Interest Algorithms:
|
||||
AlgorithmToFeedbackTokenMap(TttInterest).toString -> Set(AlgorithmType.Interest),
|
||||
AlgorithmToFeedbackTokenMap(UttInterestRelatedUsers).toString -> Set(AlgorithmType.Interest),
|
||||
AlgorithmToFeedbackTokenMap(UttSeedAccounts).toString -> Set(AlgorithmType.Interest),
|
||||
AlgorithmToFeedbackTokenMap(UttProducerExpansion).toString -> Set(AlgorithmType.Interest),
|
||||
// Hybrid (more than one type) Algorithms:
|
||||
AlgorithmToFeedbackTokenMap(UttProducerOfflineMbcgV1).toString -> Set(
|
||||
AlgorithmType.Interest,
|
||||
AlgorithmType.Geo),
|
||||
AlgorithmToFeedbackTokenMap(CuratedAccounts).toString -> Set(
|
||||
AlgorithmType.Interest,
|
||||
AlgorithmType.Geo),
|
||||
AlgorithmToFeedbackTokenMap(UserUserGraph).toString -> Set(
|
||||
AlgorithmType.Social,
|
||||
AlgorithmType.Activity),
|
||||
)
|
||||
def getAlgorithmTypes(algoId: String): Set[String] = {
|
||||
AlgorithmIdToType.get(algoId).map(_.map(_.toString)).getOrElse(Set.empty)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,43 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.constants
|
||||
|
||||
object GuiceNamedConstants {
|
||||
final val PRODUCER_SIDE_FEATURE_SWITCHES = "PRODUCER_SIDE_FEATURE_SWITCHES"
|
||||
final val CLIENT_EVENT_LOGGER = "CLIENT_EVENT_LOGGER"
|
||||
final val COSINE_FOLLOW_FETCHER = "cosine_follow_fetcher"
|
||||
final val COSINE_LIST_FETCHER = "cosine_list_fetcher"
|
||||
final val CURATED_CANDIDATES_FETCHER = "curated_candidates_fetcher"
|
||||
final val CURATED_COMPETITOR_ACCOUNTS_FETCHER = "curated_competitor_accounts_fetcher"
|
||||
final val POP_USERS_IN_PLACE_FETCHER = "pop_users_in_place_fetcher"
|
||||
final val PROFILE_SIDEBAR_BLACKLIST_SCANNER = "profile_sidebar_blacklist_scanner"
|
||||
final val REQUEST_LOGGER = "REQUEST_LOGGER"
|
||||
final val FLOW_LOGGER = "FLOW_LOGGER"
|
||||
final val REAL_TIME_INTERACTIONS_FETCHER = "real_time_interactions_fetcher"
|
||||
final val SIMS_FETCHER = "sims_fetcher"
|
||||
final val DBV2_SIMS_FETCHER = "dbv2_sims_fetcher"
|
||||
|
||||
final val TRIANGULAR_LOOPS_FETCHER = "triangular_loops_fetcher"
|
||||
final val TWO_HOP_RANDOM_WALK_FETCHER = "two_hop_random_walk_fetcher"
|
||||
final val USER_RECOMMENDABILITY_FETCHER = "user_recommendability_fetcher"
|
||||
final val USER_STATE_FETCHER = "user_state_fetcher"
|
||||
final val UTT_ACCOUNT_RECOMMENDATIONS_FETCHER = "utt_account_recomendations_fetcher"
|
||||
final val UTT_SEED_ACCOUNTS_FETCHER = "utt_seed_accounts_fetcher"
|
||||
|
||||
final val ELECTION_CANDIDATES_FETCHER = "election_candidates_fetcher"
|
||||
final val POST_NUX_WTF_FEATURES_FETCHER = "post_nux_wtf_features_fetcher"
|
||||
|
||||
final val USER_USER_GRAPH_FETCHER = "user_user_graph_fetcher"
|
||||
final val DISMISS_STORE_SCANNER = "dismiss_store_scanner"
|
||||
final val LABELED_NOTIFICATION_FETCHER = "labeled_notification_scanner"
|
||||
|
||||
final val STP_EP_SCORER = "stp_ep_scorer"
|
||||
final val STP_DBV2_SCORER = "stp_dbv2_scorer"
|
||||
final val STP_RAB_DBV2_SCORER = "stp_rab_dbv2_scorer"
|
||||
|
||||
final val EXTENDED_NETWORK = "extended_network_candidates"
|
||||
|
||||
// scoring client constants
|
||||
final val WTF_PROD_DEEPBIRDV2_CLIENT = "wtf_prod_deepbirdv2_client"
|
||||
|
||||
// ann clients
|
||||
final val RELATABLE_ACCOUNTS_FETCHER = "relatable_accounts_fetcher"
|
||||
}
|
||||
Binary file not shown.
@ -1,15 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.constants
|
||||
|
||||
import com.twitter.conversions.StorageUnitOps._
|
||||
|
||||
object ServiceConstants {
|
||||
|
||||
/** thrift client response size limits
|
||||
* these were estimated using monitoring dashboard
|
||||
* 3MB network usage per second / 25 rps ~ 120KB/req << 1MB
|
||||
* we give some buffer here in case some requests require more data than others
|
||||
*/
|
||||
val StringLengthLimit: Long =
|
||||
10.megabyte.inBytes
|
||||
val ContainerLengthLimit: Long = 1.megabyte.inBytes
|
||||
}
|
||||
@ -1,82 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":candidate-algorithm-adapter",
|
||||
":client-context-adapter",
|
||||
":post-nux-algorithm-adapter",
|
||||
":pre-fetched-feature-adapter",
|
||||
],
|
||||
)
|
||||
|
||||
target(
|
||||
name = "common",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils",
|
||||
"src/java/com/twitter/ml/api:api-base",
|
||||
"src/scala/com/twitter/ml/api/util",
|
||||
"src/scala/com/twitter/onboarding/relevance/util/metadata",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "candidate-algorithm-adapter",
|
||||
sources = [
|
||||
"CandidateAlgorithmAdapter.scala",
|
||||
],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":common",
|
||||
"hermit/hermit-core/src/main/scala/com/twitter/hermit/constants",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "client-context-adapter",
|
||||
sources = [
|
||||
"ClientContextAdapter.scala",
|
||||
],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":common",
|
||||
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "post-nux-algorithm-adapter",
|
||||
sources = [
|
||||
"PostNuxAlgorithmAdapter.scala",
|
||||
],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":common",
|
||||
"src/scala/com/twitter/ml/featurestore/catalog/features/customer_journey:post-nux-algorithm-aggregate",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "pre-fetched-feature-adapter",
|
||||
sources = [
|
||||
"PreFetchedFeatureAdapter.scala",
|
||||
],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":common",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,72 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.feature_hydration.adapters
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails
|
||||
import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap
|
||||
import com.twitter.hermit.model.Algorithm
|
||||
import com.twitter.hermit.model.Algorithm.Algorithm
|
||||
import com.twitter.hermit.model.Algorithm.UttProducerOfflineMbcgV1
|
||||
import com.twitter.hermit.model.Algorithm.UttProducerOnlineMbcgV1
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.ml.api.Feature.SparseBinary
|
||||
import com.twitter.ml.api.Feature.SparseContinuous
|
||||
import com.twitter.ml.api.FeatureContext
|
||||
import com.twitter.ml.api.IRecordOneToOneAdapter
|
||||
import com.twitter.ml.api.util.FDsl._
|
||||
|
||||
object CandidateAlgorithmAdapter
|
||||
extends IRecordOneToOneAdapter[Option[UserCandidateSourceDetails]] {
|
||||
|
||||
val CANDIDATE_ALGORITHMS: SparseBinary = new SparseBinary("candidate.source.algorithm_ids")
|
||||
val CANDIDATE_SOURCE_SCORES: SparseContinuous =
|
||||
new SparseContinuous("candidate.source.scores")
|
||||
val CANDIDATE_SOURCE_RANKS: SparseContinuous =
|
||||
new SparseContinuous("candidate.source.ranks")
|
||||
|
||||
override val getFeatureContext: FeatureContext = new FeatureContext(
|
||||
CANDIDATE_ALGORITHMS,
|
||||
CANDIDATE_SOURCE_SCORES,
|
||||
CANDIDATE_SOURCE_RANKS
|
||||
)
|
||||
|
||||
/** list of candidate source remaps to avoid creating different features for experimental sources.
|
||||
* the LHS should contain the experimental source, and the RHS should contain the prod source.
|
||||
*/
|
||||
def remapCandidateSource(a: Algorithm): Algorithm = a match {
|
||||
case UttProducerOnlineMbcgV1 => UttProducerOfflineMbcgV1
|
||||
case _ => a
|
||||
}
|
||||
|
||||
// add the list of algorithm feedback tokens (integers) as a sparse binary feature
|
||||
override def adaptToDataRecord(
|
||||
userCandidateSourceDetailsOpt: Option[UserCandidateSourceDetails]
|
||||
): DataRecord = {
|
||||
val dr = new DataRecord()
|
||||
userCandidateSourceDetailsOpt.foreach { userCandidateSourceDetails =>
|
||||
val scoreMap = for {
|
||||
(source, scoreOpt) <- userCandidateSourceDetails.candidateSourceScores
|
||||
score <- scoreOpt
|
||||
algo <- Algorithm.withNameOpt(source.name)
|
||||
algoId <- AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo))
|
||||
} yield algoId.toString -> score
|
||||
val rankMap = for {
|
||||
(source, rank) <- userCandidateSourceDetails.candidateSourceRanks
|
||||
algo <- Algorithm.withNameOpt(source.name)
|
||||
algoId <- AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo))
|
||||
} yield algoId.toString -> rank.toDouble
|
||||
|
||||
val algoIds = scoreMap.keys.toSet ++ rankMap.keys.toSet
|
||||
|
||||
// hydrate if not empty
|
||||
if (rankMap.nonEmpty) {
|
||||
dr.setFeatureValue(CANDIDATE_SOURCE_RANKS, rankMap)
|
||||
}
|
||||
if (scoreMap.nonEmpty) {
|
||||
dr.setFeatureValue(CANDIDATE_SOURCE_SCORES, scoreMap)
|
||||
}
|
||||
if (algoIds.nonEmpty) {
|
||||
dr.setFeatureValue(CANDIDATE_ALGORITHMS, algoIds)
|
||||
}
|
||||
}
|
||||
dr
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,79 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.feature_hydration.adapters
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.ml.api.Feature.Binary
|
||||
import com.twitter.ml.api.Feature.Continuous
|
||||
import com.twitter.ml.api.Feature.Discrete
|
||||
import com.twitter.ml.api.Feature.Text
|
||||
import com.twitter.ml.api.util.FDsl._
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.ml.api.FeatureContext
|
||||
import com.twitter.ml.api.IRecordOneToOneAdapter
|
||||
import com.twitter.onboarding.relevance.util.metadata.LanguageUtil
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.snowflake.id.SnowflakeId
|
||||
|
||||
object ClientContextAdapter extends IRecordOneToOneAdapter[(ClientContext, DisplayLocation)] {
|
||||
|
||||
// we name features with `user.account` for relatively static user-related features
|
||||
val USER_COUNTRY: Text = new Text("user.account.country")
|
||||
val USER_LANGUAGE: Text = new Text("user.account.language")
|
||||
// we name features with `user.context` for more dynamic user-related features
|
||||
val USER_LANGUAGE_PREFIX: Text = new Text("user.context.language_prefix")
|
||||
val USER_CLIENT: Discrete = new Discrete("user.context.client")
|
||||
val USER_AGE: Continuous = new Continuous("user.context.age")
|
||||
val USER_IS_RECENT: Binary = new Binary("user.is.recent")
|
||||
// we name features with `meta` for meta info about the WTF recommendation request
|
||||
val META_DISPLAY_LOCATION: Text = new Text("meta.display_location")
|
||||
val META_POSITION: Discrete = new Discrete("meta.position")
|
||||
// This indicates whether a data point is from a random serving policy
|
||||
val META_IS_RANDOM: Binary = new Binary("prediction.engine.is_random")
|
||||
|
||||
val RECENT_WIN_IN_DAYS: Int = 30
|
||||
val GOAL_META_POSITION: Long = 1L
|
||||
val GOAL_META_IS_RANDOM: Boolean = true
|
||||
|
||||
override val getFeatureContext: FeatureContext = new FeatureContext(
|
||||
USER_COUNTRY,
|
||||
USER_LANGUAGE,
|
||||
USER_AGE,
|
||||
USER_LANGUAGE_PREFIX,
|
||||
USER_CLIENT,
|
||||
USER_IS_RECENT,
|
||||
META_DISPLAY_LOCATION,
|
||||
META_POSITION,
|
||||
META_IS_RANDOM
|
||||
)
|
||||
|
||||
/**
|
||||
* we only want to set the relevant fields iff they exist to eliminate redundant information
|
||||
* we do some simple normalization on the language code
|
||||
* we set META_POSITION to 1 always
|
||||
* we set META_IS_RANDOM to true always to simulate a random serving distribution
|
||||
* @param record ClientContext and DisplayLocation from the request
|
||||
*/
|
||||
override def adaptToDataRecord(target: (ClientContext, DisplayLocation)): DataRecord = {
|
||||
val dr = new DataRecord()
|
||||
val cc = target._1
|
||||
val dl = target._2
|
||||
cc.countryCode.foreach(countryCode => dr.setFeatureValue(USER_COUNTRY, countryCode))
|
||||
cc.languageCode.foreach(rawLanguageCode => {
|
||||
val userLanguage = LanguageUtil.simplifyLanguage(rawLanguageCode)
|
||||
val userLanguagePrefix = userLanguage.take(2)
|
||||
dr.setFeatureValue(USER_LANGUAGE, userLanguage)
|
||||
dr.setFeatureValue(USER_LANGUAGE_PREFIX, userLanguagePrefix)
|
||||
})
|
||||
cc.appId.foreach(appId => dr.setFeatureValue(USER_CLIENT, appId))
|
||||
cc.userId.foreach(id =>
|
||||
SnowflakeId.timeFromIdOpt(id).map { signupTime =>
|
||||
val userAge = signupTime.untilNow.inMillis.toDouble
|
||||
dr.setFeatureValue(USER_AGE, userAge)
|
||||
dr.setFeatureValue(USER_IS_RECENT, signupTime.untilNow.inDays <= RECENT_WIN_IN_DAYS)
|
||||
signupTime.untilNow.inDays
|
||||
})
|
||||
dr.setFeatureValue(META_DISPLAY_LOCATION, dl.toFsName)
|
||||
dr.setFeatureValue(META_POSITION, GOAL_META_POSITION)
|
||||
dr.setFeatureValue(META_IS_RANDOM, GOAL_META_IS_RANDOM)
|
||||
dr
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,151 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.feature_hydration.adapters
|
||||
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.ml.api.Feature
|
||||
import com.twitter.ml.api.Feature.Continuous
|
||||
import com.twitter.ml.api.FeatureContext
|
||||
import com.twitter.ml.api.IRecordOneToOneAdapter
|
||||
import com.twitter.ml.api.util.FDsl._
|
||||
import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmFeatures
|
||||
import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmIdAggregateFeatureGroup
|
||||
import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmTypeAggregateFeatureGroup
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
object PostNuxAlgorithmIdAdapter extends PostNuxAlgorithmAdapter {
|
||||
override val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures =
|
||||
PostNuxAlgorithmIdAggregateFeatureGroup
|
||||
|
||||
// To keep the length of feature names reasonable, we remove the prefix added by FeatureStore.
|
||||
override val FeatureStorePrefix: String =
|
||||
"wtf_algorithm_id.customer_journey.post_nux_algorithm_id_aggregate_feature_group."
|
||||
}
|
||||
|
||||
object PostNuxAlgorithmTypeAdapter extends PostNuxAlgorithmAdapter {
|
||||
override val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures =
|
||||
PostNuxAlgorithmTypeAggregateFeatureGroup
|
||||
|
||||
// To keep the length of feature names reasonable, we remove the prefix added by FeatureStore.
|
||||
override val FeatureStorePrefix: String =
|
||||
"wtf_algorithm_type.customer_journey.post_nux_algorithm_type_aggregate_feature_group."
|
||||
}
|
||||
|
||||
trait PostNuxAlgorithmAdapter extends IRecordOneToOneAdapter[DataRecord] {
|
||||
|
||||
val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures
|
||||
|
||||
// The string that is attached to the feature name when it is fetched from feature store.
|
||||
val FeatureStorePrefix: String
|
||||
|
||||
/**
|
||||
*
|
||||
* This stores transformed aggregate features for PostNux algorithm aggregate features. The
|
||||
* transformation here is log-ratio, where ratio is the raw value divided by # of impressions.
|
||||
*/
|
||||
case class TransformedAlgorithmFeatures(
|
||||
ratioLog: Continuous) {
|
||||
def getFeatures: Seq[Continuous] = Seq(ratioLog)
|
||||
}
|
||||
|
||||
private def applyFeatureStorePrefix(feature: Continuous) = new Continuous(
|
||||
s"$FeatureStorePrefix${feature.getFeatureName}")
|
||||
|
||||
// The list of input features WITH the prefix assigned to them by FeatureStore.
|
||||
lazy val allInputFeatures: Seq[Seq[Continuous]] = Seq(
|
||||
PostNuxAlgorithmFeatureGroup.Aggregate7DayFeatures.map(applyFeatureStorePrefix),
|
||||
PostNuxAlgorithmFeatureGroup.Aggregate30DayFeatures.map(applyFeatureStorePrefix)
|
||||
)
|
||||
|
||||
// This is a list of the features WITHOUT the prefix assigned to them by FeatureStore.
|
||||
lazy val outputBaseFeatureNames: Seq[Seq[Continuous]] = Seq(
|
||||
PostNuxAlgorithmFeatureGroup.Aggregate7DayFeatures,
|
||||
PostNuxAlgorithmFeatureGroup.Aggregate30DayFeatures
|
||||
)
|
||||
|
||||
// We use backend impression to calculate ratio values.
|
||||
lazy val ratioDenominators: Seq[Continuous] = Seq(
|
||||
applyFeatureStorePrefix(PostNuxAlgorithmFeatureGroup.BackendImpressions7Days),
|
||||
applyFeatureStorePrefix(PostNuxAlgorithmFeatureGroup.BackendImpressions30Days)
|
||||
)
|
||||
|
||||
/**
|
||||
* A mapping from an original feature's ID to the corresponding set of transformed features.
|
||||
* This is used to compute the transformed features for each of the original ones.
|
||||
*/
|
||||
private lazy val TransformedFeaturesMap: Map[Continuous, TransformedAlgorithmFeatures] =
|
||||
outputBaseFeatureNames.flatten.map { feature =>
|
||||
(
|
||||
// The input feature would have the FeatureStore prefix attached to it.
|
||||
new Continuous(s"$FeatureStorePrefix${feature.getFeatureName}"),
|
||||
// We don't keep the FeatureStore prefix to keep the length of feature names reasonable.
|
||||
TransformedAlgorithmFeatures(
|
||||
new Continuous(s"${feature.getFeatureName}-ratio-log")
|
||||
))
|
||||
}.toMap
|
||||
|
||||
/**
|
||||
* Given a denominator, number of impressions, this function returns another function that adds
|
||||
* transformed features (log1p and ratio) of an input feature to a DataRecord.
|
||||
*/
|
||||
private def addTransformedFeaturesToDataRecordFunc(
|
||||
originalDr: DataRecord,
|
||||
numImpressions: Double,
|
||||
): (DataRecord, Continuous) => DataRecord = { (record: DataRecord, feature: Continuous) =>
|
||||
{
|
||||
Option(originalDr.getFeatureValue(feature)) foreach { featureValue =>
|
||||
TransformedFeaturesMap.get(feature).foreach { transformedFeatures =>
|
||||
record.setFeatureValue(
|
||||
transformedFeatures.ratioLog,
|
||||
// We don't use log1p here since the values are ratios and adding 1 to the _ratio_ would
|
||||
// lead to logarithm of values between 1 and 2, essentially making all values the same.
|
||||
math.log((featureValue + 1) / numImpressions)
|
||||
)
|
||||
}
|
||||
}
|
||||
record
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param record: The input record whose PostNuxAlgorithm aggregates are to be transformed.
|
||||
* @return the input [[DataRecord]] with transformed aggregates added.
|
||||
*/
|
||||
override def adaptToDataRecord(record: DataRecord): DataRecord = {
|
||||
if (record.continuousFeatures == null) {
|
||||
// There are no base features available, and hence no transformations.
|
||||
record
|
||||
} else {
|
||||
|
||||
/**
|
||||
* The `foldLeft` below goes through pairs of (1) Feature groups, such as those calculated over
|
||||
* 7 days or 30 days, and (2) the number of impressions for each of these groups, which is the
|
||||
* denominator when ratio is calculated.
|
||||
*/
|
||||
ratioDenominators
|
||||
.zip(allInputFeatures).foldLeft( /* initial empty DataRecord */ record)(
|
||||
(
|
||||
/* DataRecord with transformed features up to here */ transformedRecord,
|
||||
/* A tuple with the denominator (#impressions) and features to be transformed */ numImpressionsAndFeatures
|
||||
) => {
|
||||
val (numImpressionsFeature, features) = numImpressionsAndFeatures
|
||||
Option(record.getFeatureValue(numImpressionsFeature)) match {
|
||||
case Some(numImpressions) if numImpressions > 0.0 =>
|
||||
/**
|
||||
* With the number of impressions fixed, we generate a function that adds log-ratio
|
||||
* for each feature in the current [[DataRecord]]. The `foldLeft` goes through all
|
||||
* such features and applies that function while updating the kept DataRecord.
|
||||
*/
|
||||
features.foldLeft(transformedRecord)(
|
||||
addTransformedFeaturesToDataRecordFunc(record, numImpressions))
|
||||
case _ =>
|
||||
transformedRecord
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
def getFeatures: Seq[Feature[_]] = TransformedFeaturesMap.values.flatMap(_.getFeatures).toSeq
|
||||
|
||||
override def getFeatureContext: FeatureContext =
|
||||
new FeatureContext()
|
||||
.addFeatures(this.getFeatures.asJava)
|
||||
}
|
||||
Binary file not shown.
@ -1,91 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.feature_hydration.adapters
|
||||
|
||||
import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.ml.api.Feature.Continuous
|
||||
import com.twitter.ml.api.util.FDsl._
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.ml.api.FeatureContext
|
||||
import com.twitter.ml.api.IRecordOneToOneAdapter
|
||||
import com.twitter.util.Time
|
||||
|
||||
/**
|
||||
* This adapter mimics UserRecentWTFImpressionsAndFollowsAdapter (for user) and
|
||||
* RecentWTFImpressionsFeatureAdapter (for candidate) for extracting recent impression
|
||||
* and follow features. This adapter extracts user, candidate, and pair-wise features.
|
||||
*/
|
||||
object PreFetchedFeatureAdapter
|
||||
extends IRecordOneToOneAdapter[
|
||||
(HasPreFetchedFeature, CandidateUser)
|
||||
] {
|
||||
|
||||
// impression features
|
||||
val USER_NUM_RECENT_IMPRESSIONS: Continuous = new Continuous(
|
||||
"user.prefetch.num_recent_impressions"
|
||||
)
|
||||
val USER_LAST_IMPRESSION_DURATION: Continuous = new Continuous(
|
||||
"user.prefetch.last_impression_duration"
|
||||
)
|
||||
val CANDIDATE_NUM_RECENT_IMPRESSIONS: Continuous = new Continuous(
|
||||
"user-candidate.prefetch.num_recent_impressions"
|
||||
)
|
||||
val CANDIDATE_LAST_IMPRESSION_DURATION: Continuous = new Continuous(
|
||||
"user-candidate.prefetch.last_impression_duration"
|
||||
)
|
||||
// follow features
|
||||
val USER_NUM_RECENT_FOLLOWERS: Continuous = new Continuous(
|
||||
"user.prefetch.num_recent_followers"
|
||||
)
|
||||
val USER_NUM_RECENT_FOLLOWED_BY: Continuous = new Continuous(
|
||||
"user.prefetch.num_recent_followed_by"
|
||||
)
|
||||
val USER_NUM_RECENT_MUTUAL_FOLLOWS: Continuous = new Continuous(
|
||||
"user.prefetch.num_recent_mutual_follows"
|
||||
)
|
||||
// impression + follow features
|
||||
val USER_NUM_RECENT_FOLLOWED_IMPRESSIONS: Continuous = new Continuous(
|
||||
"user.prefetch.num_recent_followed_impression"
|
||||
)
|
||||
val USER_LAST_FOLLOWED_IMPRESSION_DURATION: Continuous = new Continuous(
|
||||
"user.prefetch.last_followed_impression_duration"
|
||||
)
|
||||
|
||||
override def adaptToDataRecord(
|
||||
record: (HasPreFetchedFeature, CandidateUser)
|
||||
): DataRecord = {
|
||||
val (target, candidate) = record
|
||||
val dr = new DataRecord()
|
||||
val t = Time.now
|
||||
// set impression features for user, optionally for candidate
|
||||
dr.setFeatureValue(USER_NUM_RECENT_IMPRESSIONS, target.numWtfImpressions.toDouble)
|
||||
dr.setFeatureValue(
|
||||
USER_LAST_IMPRESSION_DURATION,
|
||||
(t - target.latestImpressionTime).inMillis.toDouble)
|
||||
target.getCandidateImpressionCounts(candidate.id).foreach { counts =>
|
||||
dr.setFeatureValue(CANDIDATE_NUM_RECENT_IMPRESSIONS, counts.toDouble)
|
||||
}
|
||||
target.getCandidateLatestTime(candidate.id).foreach { latestTime: Time =>
|
||||
dr.setFeatureValue(CANDIDATE_LAST_IMPRESSION_DURATION, (t - latestTime).inMillis.toDouble)
|
||||
}
|
||||
// set recent follow features for user
|
||||
dr.setFeatureValue(USER_NUM_RECENT_FOLLOWERS, target.numRecentFollowedUserIds.toDouble)
|
||||
dr.setFeatureValue(USER_NUM_RECENT_FOLLOWED_BY, target.numRecentFollowedByUserIds.toDouble)
|
||||
dr.setFeatureValue(USER_NUM_RECENT_MUTUAL_FOLLOWS, target.numRecentMutualFollows.toDouble)
|
||||
dr.setFeatureValue(USER_NUM_RECENT_FOLLOWED_IMPRESSIONS, target.numFollowedImpressions.toDouble)
|
||||
dr.setFeatureValue(
|
||||
USER_LAST_FOLLOWED_IMPRESSION_DURATION,
|
||||
target.lastFollowedImpressionDurationMs.getOrElse(Long.MaxValue).toDouble)
|
||||
dr
|
||||
}
|
||||
override def getFeatureContext: FeatureContext = new FeatureContext(
|
||||
USER_NUM_RECENT_IMPRESSIONS,
|
||||
USER_LAST_IMPRESSION_DURATION,
|
||||
CANDIDATE_NUM_RECENT_IMPRESSIONS,
|
||||
CANDIDATE_LAST_IMPRESSION_DURATION,
|
||||
USER_NUM_RECENT_FOLLOWERS,
|
||||
USER_NUM_RECENT_FOLLOWED_BY,
|
||||
USER_NUM_RECENT_MUTUAL_FOLLOWS,
|
||||
USER_NUM_RECENT_FOLLOWED_IMPRESSIONS,
|
||||
USER_LAST_FOLLOWED_IMPRESSION_DURATION,
|
||||
)
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils",
|
||||
"src/java/com/twitter/ml/api:api-base",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,23 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.feature_hydration.common
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.HasDisplayLocation
|
||||
import com.twitter.follow_recommendations.common.models.HasSimilarToContext
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.ml.api.FeatureContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
|
||||
trait FeatureSource {
|
||||
def id: FeatureSourceId
|
||||
def featureContext: FeatureContext
|
||||
def hydrateFeatures(
|
||||
target: HasClientContext
|
||||
with HasPreFetchedFeature
|
||||
with HasParams
|
||||
with HasSimilarToContext
|
||||
with HasDisplayLocation,
|
||||
candidates: Seq[CandidateUser]
|
||||
): Stitch[Map[CandidateUser, DataRecord]]
|
||||
}
|
||||
Binary file not shown.
@ -1,19 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.feature_hydration.common
|
||||
|
||||
sealed trait FeatureSourceId
|
||||
|
||||
object FeatureSourceId {
|
||||
object CandidateAlgorithmSourceId extends FeatureSourceId
|
||||
object ClientContextSourceId extends FeatureSourceId
|
||||
object FeatureStoreSourceId extends FeatureSourceId
|
||||
object FeatureStoreTimelinesAuthorSourceId extends FeatureSourceId
|
||||
object FeatureStoreGizmoduckSourceId extends FeatureSourceId
|
||||
object FeatureStoreUserMetricCountsSourceId extends FeatureSourceId
|
||||
object FeatureStoreNotificationSourceId extends FeatureSourceId
|
||||
|
||||
object FeatureStorePrecomputedNotificationSourceId extends FeatureSourceId
|
||||
object FeatureStorePostNuxAlgorithmSourceId extends FeatureSourceId
|
||||
@deprecated object StratoFeatureHydrationSourceId extends FeatureSourceId
|
||||
object PreFetchedFeatureSourceId extends FeatureSourceId
|
||||
object UserScoringFeatureSourceId extends FeatureSourceId
|
||||
}
|
||||
Binary file not shown.
@ -1,25 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.feature_hydration.common
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.HasMutualFollowedUserIds
|
||||
import com.twitter.follow_recommendations.common.models.HasWtfImpressions
|
||||
import com.twitter.follow_recommendations.common.models.WtfImpression
|
||||
import com.twitter.util.Time
|
||||
|
||||
trait HasPreFetchedFeature extends HasMutualFollowedUserIds with HasWtfImpressions {
|
||||
|
||||
lazy val followedImpressions: Seq[WtfImpression] = {
|
||||
for {
|
||||
wtfImprList <- wtfImpressions.toSeq
|
||||
wtfImpr <- wtfImprList
|
||||
if recentFollowedUserIds.exists(_.contains(wtfImpr.candidateId))
|
||||
} yield wtfImpr
|
||||
}
|
||||
|
||||
lazy val numFollowedImpressions: Int = followedImpressions.size
|
||||
|
||||
lazy val lastFollowedImpressionDurationMs: Option[Long] = {
|
||||
if (followedImpressions.nonEmpty) {
|
||||
Some((Time.now - followedImpressions.map(_.latestTime).max).inMillis)
|
||||
} else None
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user