2151 lines
80 KiB
Scala
2151 lines
80 KiB
Scala
package com.twitter.frigate.pushservice.config
|
|
|
|
import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest
|
|
import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse
|
|
import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest
|
|
import com.twitter.bijection.scrooge.BinaryScalaCodec
|
|
import com.twitter.bijection.scrooge.CompactScalaCodec
|
|
import com.twitter.channels.common.thriftscala.ApiList
|
|
import com.twitter.channels.common.thriftscala.ApiListDisplayLocation
|
|
import com.twitter.channels.common.thriftscala.ApiListView
|
|
import com.twitter.content_mixer.thriftscala.ContentMixer
|
|
import com.twitter.conversions.DurationOps._
|
|
import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService
|
|
import com.twitter.cr_mixer.thriftscala.CrMixer
|
|
import com.twitter.datatools.entityservice.entities.sports.thriftscala.BaseballGameLiveUpdate
|
|
import com.twitter.datatools.entityservice.entities.sports.thriftscala.BasketballGameLiveUpdate
|
|
import com.twitter.datatools.entityservice.entities.sports.thriftscala.CricketMatchLiveUpdate
|
|
import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate
|
|
import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate
|
|
import com.twitter.discovery.common.configapi.ConfigParamsBuilder
|
|
import com.twitter.discovery.common.configapi.FeatureContextBuilder
|
|
import com.twitter.discovery.common.environment.{Environment => NotifEnvironment}
|
|
import com.twitter.escherbird.common.thriftscala.Domains
|
|
import com.twitter.escherbird.common.thriftscala.QualifiedId
|
|
import com.twitter.escherbird.metadata.thriftscala.EntityMegadata
|
|
import com.twitter.escherbird.metadata.thriftscala.MetadataService
|
|
import com.twitter.escherbird.util.metadatastitch.MetadataStitchClient
|
|
import com.twitter.escherbird.util.uttclient
|
|
import com.twitter.escherbird.util.uttclient.CacheConfigV2
|
|
import com.twitter.escherbird.util.uttclient.CachedUttClientV2
|
|
import com.twitter.escherbird.utt.strato.thriftscala.Environment
|
|
import com.twitter.eventbus.client.EventBusPublisherBuilder
|
|
import com.twitter.events.recos.thriftscala.EventsRecosService
|
|
import com.twitter.explore_ranker.thriftscala.ExploreRanker
|
|
import com.twitter.featureswitches.v2.FeatureSwitches
|
|
import com.twitter.finagle.Memcached
|
|
import com.twitter.finagle.ThriftMux
|
|
import com.twitter.finagle.client.BackupRequestFilter
|
|
import com.twitter.finagle.client.ClientRegistry
|
|
import com.twitter.finagle.loadbalancer.Balancers
|
|
import com.twitter.finagle.memcached.Client
|
|
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
|
import com.twitter.finagle.mtls.client.MtlsStackClient._
|
|
import com.twitter.finagle.mux.transport.OpportunisticTls
|
|
import com.twitter.finagle.service.Retries
|
|
import com.twitter.finagle.service.RetryPolicy
|
|
import com.twitter.finagle.stats.StatsReceiver
|
|
import com.twitter.finagle.thrift.ClientId
|
|
import com.twitter.finagle.thrift.RichClientParam
|
|
import com.twitter.finagle.util.DefaultTimer
|
|
import com.twitter.flockdb.client._
|
|
import com.twitter.flockdb.client.thriftscala.FlockDB
|
|
import com.twitter.frigate.common.base.RandomRanker
|
|
import com.twitter.frigate.common.candidate._
|
|
import com.twitter.frigate.common.config.RateLimiterGenerator
|
|
import com.twitter.frigate.common.entity_graph_client.RecommendedTweetEntitiesStore
|
|
import com.twitter.frigate.common.filter.DynamicRequestMeterFilter
|
|
import com.twitter.frigate.common.history._
|
|
import com.twitter.frigate.common.ml.feature._
|
|
import com.twitter.frigate.common.store._
|
|
import com.twitter.frigate.common.store.deviceinfo.DeviceInfoStore
|
|
import com.twitter.frigate.common.store.deviceinfo.MobileSdkStore
|
|
import com.twitter.frigate.common.store.interests._
|
|
import com.twitter.frigate.common.store.strato.StratoFetchableStore
|
|
import com.twitter.frigate.common.store.strato.StratoScannableStore
|
|
import com.twitter.frigate.common.util.Finagle.readOnlyThriftService
|
|
import com.twitter.frigate.common.util._
|
|
import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil
|
|
import com.twitter.frigate.data_pipeline.features_common._
|
|
import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryKey
|
|
import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue
|
|
import com.twitter.frigate.dau_model.thriftscala.DauProbability
|
|
import com.twitter.frigate.magic_events.thriftscala.FanoutEvent
|
|
import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory
|
|
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
import com.twitter.frigate.pushservice.adaptor.LoggedOutPushCandidateSourceGenerator
|
|
import com.twitter.frigate.pushservice.adaptor.PushCandidateSourceGenerator
|
|
import com.twitter.frigate.pushservice.config.mlconfig.DeepbirdV2ModelConfig
|
|
import com.twitter.frigate.pushservice.ml._
|
|
import com.twitter.frigate.pushservice.params._
|
|
import com.twitter.frigate.pushservice.rank.LoggedOutRanker
|
|
import com.twitter.frigate.pushservice.rank.RFPHLightRanker
|
|
import com.twitter.frigate.pushservice.rank.RFPHRanker
|
|
import com.twitter.frigate.pushservice.rank.SubscriptionCreatorRanker
|
|
import com.twitter.frigate.pushservice.refresh_handler._
|
|
import com.twitter.frigate.pushservice.refresh_handler.cross.CandidateCopyExpansion
|
|
import com.twitter.frigate.pushservice.send_handler.SendHandlerPushCandidateHydrator
|
|
import com.twitter.frigate.pushservice.store._
|
|
import com.twitter.frigate.pushservice.take.CandidateNotifier
|
|
import com.twitter.frigate.pushservice.take.NotificationSender
|
|
import com.twitter.frigate.pushservice.take.NotificationServiceRequest
|
|
import com.twitter.frigate.pushservice.take.NotificationServiceSender
|
|
import com.twitter.frigate.pushservice.take.NtabOnlyChannelSelector
|
|
import com.twitter.frigate.pushservice.take.history.EventBusWriter
|
|
import com.twitter.frigate.pushservice.take.history.HistoryWriter
|
|
import com.twitter.frigate.pushservice.take.sender.Ibis2Sender
|
|
import com.twitter.frigate.pushservice.take.sender.NtabSender
|
|
import com.twitter.frigate.pushservice.take.LoggedOutRefreshForPushNotifier
|
|
import com.twitter.frigate.pushservice.util.RFPHTakeStepUtil
|
|
import com.twitter.frigate.pushservice.util.SendHandlerPredicateUtil
|
|
import com.twitter.frigate.scribe.thriftscala.NotificationScribe
|
|
import com.twitter.frigate.thriftscala._
|
|
import com.twitter.frigate.user_states.thriftscala.MRUserHmmState
|
|
import com.twitter.geoduck.backend.hydration.thriftscala.Hydration
|
|
import com.twitter.geoduck.common.thriftscala.PlaceQueryFields
|
|
import com.twitter.geoduck.common.thriftscala.PlaceType
|
|
import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation}
|
|
import com.twitter.geoduck.service.common.clientmodules.GeoduckUserLocate
|
|
import com.twitter.geoduck.service.common.clientmodules.GeoduckUserLocateModule
|
|
import com.twitter.geoduck.service.thriftscala.LocationResponse
|
|
import com.twitter.geoduck.thriftscala.LocationService
|
|
import com.twitter.gizmoduck.context.thriftscala.ReadConfig
|
|
import com.twitter.gizmoduck.context.thriftscala.TestUserConfig
|
|
import com.twitter.gizmoduck.testusers.client.TestUserClientBuilder
|
|
import com.twitter.gizmoduck.thriftscala.LookupContext
|
|
import com.twitter.gizmoduck.thriftscala.QueryFields
|
|
import com.twitter.gizmoduck.thriftscala.User
|
|
import com.twitter.gizmoduck.thriftscala.UserService
|
|
import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace
|
|
import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate
|
|
import com.twitter.hermit.predicate.tweetypie.PerspectiveReadableStore
|
|
import com.twitter.hermit.store._
|
|
import com.twitter.hermit.store.common._
|
|
import com.twitter.hermit.store.gizmoduck.GizmoduckUserStore
|
|
import com.twitter.hermit.store.metastore.UserCountryStore
|
|
import com.twitter.hermit.store.metastore.UserLanguagesStore
|
|
import com.twitter.hermit.store.scarecrow.ScarecrowCheckEventStore
|
|
import com.twitter.hermit.store.semantic_core.MetaDataReadableStore
|
|
import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery
|
|
import com.twitter.hermit.store.timezone.GizmoduckUserUtcOffsetStore
|
|
import com.twitter.hermit.store.timezone.UtcOffsetStore
|
|
import com.twitter.hermit.store.tweetypie.TweetyPieStore
|
|
import com.twitter.hermit.store.tweetypie.UserTweet
|
|
import com.twitter.hermit.store.user_htl_session_store.UserHTLLastVisitReadableStore
|
|
import com.twitter.hermit.stp.thriftscala.STPResult
|
|
import com.twitter.hss.api.thriftscala.UserHealthSignal
|
|
import com.twitter.hss.api.thriftscala.UserHealthSignal._
|
|
import com.twitter.hss.api.thriftscala.UserHealthSignalResponse
|
|
import com.twitter.interests.thriftscala.InterestId
|
|
import com.twitter.interests.thriftscala.InterestsThriftService
|
|
import com.twitter.interests.thriftscala.{UserInterests => Interests}
|
|
import com.twitter.interests_discovery.thriftscala.InterestsDiscoveryService
|
|
import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists
|
|
import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest
|
|
import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse
|
|
import com.twitter.kujaku.domain.thriftscala.MachineTranslationResponse
|
|
import com.twitter.livevideo.timeline.client.v2.LiveVideoTimelineClient
|
|
import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent}
|
|
import com.twitter.livevideo.timeline.thrift.thriftscala.TimelineService
|
|
import com.twitter.logging.Logger
|
|
import com.twitter.ml.api.thriftscala.{DataRecord => ThriftDataRecord}
|
|
import com.twitter.ml.featurestore.catalog.entities.core.{Author => TweetAuthorEntity}
|
|
import com.twitter.ml.featurestore.catalog.entities.core.{User => TargetUserEntity}
|
|
import com.twitter.ml.featurestore.catalog.entities.core.{UserAuthor => UserAuthorEntity}
|
|
import com.twitter.ml.featurestore.catalog.entities.magicrecs.{SocialContext => SocialContextEntity}
|
|
import com.twitter.ml.featurestore.catalog.entities.magicrecs.{UserSocialContext => TargetUserSocialContextEntity}
|
|
import com.twitter.ml.featurestore.timelines.thriftscala.TimelineScorerScoreView
|
|
import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest
|
|
import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue
|
|
import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore
|
|
import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStoreBuilder
|
|
import com.twitter.notificationservice.scribe.manhattan.FeedbackSignalManhattanClient
|
|
import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest
|
|
import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails
|
|
import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest
|
|
import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse
|
|
import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest
|
|
import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey
|
|
import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient
|
|
import com.twitter.nrel.heavyranker.CandidateFeatureHydrator
|
|
import com.twitter.nrel.heavyranker.FeatureHydrator
|
|
import com.twitter.nrel.heavyranker.{PushPredictionServiceStore => RelevancePushPredictionServiceStore}
|
|
import com.twitter.nrel.heavyranker.{TargetFeatureHydrator => RelevanceTargetFeatureHydrator}
|
|
import com.twitter.nrel.lightranker.MagicRecsServeDataRecordLightRanker
|
|
import com.twitter.nrel.lightranker.{Config => LightRankerConfig}
|
|
import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment
|
|
import com.twitter.periscope.api.thriftscala.AudioSpacesLookupContext
|
|
import com.twitter.permissions_storage.thriftscala.AppPermission
|
|
import com.twitter.recommendation.interests.discovery.core.config.{DeployConfig => InterestDeployConfig}
|
|
import com.twitter.recommendation.interests.discovery.popgeo.deploy.PopGeoInterestProvider
|
|
import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph
|
|
import com.twitter.recos.user_user_graph.thriftscala.UserUserGraph
|
|
import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty
|
|
import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWProducer
|
|
import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation
|
|
import com.twitter.search.earlybird.thriftscala.EarlybirdService
|
|
import com.twitter.service.gen.scarecrow.thriftscala.ScarecrowService
|
|
import com.twitter.service.metastore.gen.thriftscala.Location
|
|
import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities
|
|
import com.twitter.socialgraph.thriftscala.SocialGraphService
|
|
import com.twitter.spam.rtf.thriftscala.SafetyLevel
|
|
import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult
|
|
import com.twitter.storage.client.manhattan.kv.Guarantee
|
|
import com.twitter.storage.client.manhattan.kv.ManhattanKVClient
|
|
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
|
|
import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint
|
|
import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder
|
|
import com.twitter.storehaus.ReadableStore
|
|
import com.twitter.storehaus_internal.manhattan.Apollo
|
|
import com.twitter.storehaus_internal.manhattan.Athena
|
|
import com.twitter.storehaus_internal.manhattan.Dataset
|
|
import com.twitter.storehaus_internal.manhattan.ManhattanStore
|
|
import com.twitter.storehaus_internal.manhattan.Nash
|
|
import com.twitter.storehaus_internal.manhattan.Omega
|
|
import com.twitter.storehaus_internal.memcache.MemcacheStore
|
|
import com.twitter.storehaus_internal.util.ClientName
|
|
import com.twitter.storehaus_internal.util.ZkEndPoint
|
|
import com.twitter.strato.catalog.Scan.Slice
|
|
import com.twitter.strato.client.Strato
|
|
import com.twitter.strato.client.UserId
|
|
import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata
|
|
import com.twitter.strato.columns.notifications.thriftscala.SourceDestUserRequest
|
|
import com.twitter.strato.generated.client.geo.user.FrequentSoftUserLocationClientColumn
|
|
import com.twitter.strato.generated.client.ml.featureStore.TimelineScorerTweetScoresV1ClientColumn
|
|
import com.twitter.strato.generated.client.notifications.space_device_follow_impl.SpaceDeviceFollowingClientColumn
|
|
import com.twitter.strato.generated.client.periscope.CoreOnAudioSpaceClientColumn
|
|
import com.twitter.strato.generated.client.periscope.ParticipantsOnAudioSpaceClientColumn
|
|
import com.twitter.strato.generated.client.rux.TargetingPropertyOnUserClientColumn
|
|
import com.twitter.strato.generated.client.socialgraph.graphs.creatorSubscriptionTimeline.{CountEdgesBySourceClientColumn => CreatorSubscriptionNumTweetsColumn}
|
|
import com.twitter.strato.generated.client.translation.service.IsTweetTranslatableClientColumn
|
|
import com.twitter.strato.generated.client.translation.service.platform.MachineTranslateTweetClientColumn
|
|
import com.twitter.strato.generated.client.trends.trip.TripTweetsAirflowProdClientColumn
|
|
import com.twitter.strato.thrift.ScroogeConvImplicits._
|
|
import com.twitter.taxi.common.AppId
|
|
import com.twitter.taxi.deploy.Cluster
|
|
import com.twitter.taxi.deploy.Env
|
|
import com.twitter.topiclisting.TopicListing
|
|
import com.twitter.topiclisting.TopicListingBuilder
|
|
import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain
|
|
import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets
|
|
import com.twitter.tsp.thriftscala.TopicSocialProofRequest
|
|
import com.twitter.tsp.thriftscala.TopicSocialProofResponse
|
|
import com.twitter.tweetypie.thriftscala.GetTweetOptions
|
|
import com.twitter.tweetypie.thriftscala.Tweet.VisibleTextRangeField
|
|
import com.twitter.tweetypie.thriftscala.TweetService
|
|
import com.twitter.ubs.thriftscala.AudioSpace
|
|
import com.twitter.ubs.thriftscala.Participants
|
|
import com.twitter.ubs.thriftscala.SellerApplicationState
|
|
import com.twitter.user_session_store.thriftscala.UserSession
|
|
import com.twitter.util.Duration
|
|
import com.twitter.util.Future
|
|
import com.twitter.util.Timer
|
|
import com.twitter.util.tunable.TunableMap
|
|
import com.twitter.wtf.scalding.common.thriftscala.UserFeatures
|
|
import org.apache.thrift.protocol.TCompactProtocol
|
|
import com.twitter.timelinescorer.thriftscala.v1.ScoredTweet
|
|
import com.twitter.ubs.thriftscala.SellerTrack
|
|
import com.twitter.wtf.candidate.thriftscala.CandidateSeq
|
|
|
|
trait DeployConfig extends Config {
|
|
// Any finagle clients should not be defined as lazy. If defined lazy,
|
|
// ClientRegistry.expAllRegisteredClientsResolved() call in init will not ensure that the clients
|
|
// are active before thrift endpoint is active. We want the clients to be active, because zookeeper
|
|
// resolution triggered by first request(s) might result in the request(s) failing.
|
|
|
|
def serviceIdentifier: ServiceIdentifier
|
|
|
|
def tunableMap: TunableMap
|
|
|
|
def featureSwitches: FeatureSwitches
|
|
|
|
override val isProd: Boolean =
|
|
serviceIdentifier.environment == PushConstants.ServiceProdEnvironmentName
|
|
|
|
def shardParams: ShardParams
|
|
|
|
def log: Logger
|
|
|
|
implicit def statsReceiver: StatsReceiver
|
|
|
|
implicit val timer: Timer = DefaultTimer
|
|
|
|
def notifierThriftClientId: ClientId
|
|
|
|
def loggedOutNotifierThriftClientId: ClientId
|
|
|
|
def pushserviceThriftClientId: ClientId
|
|
|
|
def deepbirdv2PredictionServiceDest: String
|
|
|
|
def featureStoreUtil: FeatureStoreUtil
|
|
|
|
def targetLevelFeaturesConfig: PushFeaturesConfig
|
|
|
|
private val manhattanClientMtlsParams = ManhattanKVClientMtlsParams(
|
|
serviceIdentifier = serviceIdentifier,
|
|
opportunisticTls = OpportunisticTls.Required
|
|
)
|
|
|
|
// Commonly used clients
|
|
val gizmoduckClient = {
|
|
|
|
val client = ThriftMux.client
|
|
.withMutualTls(serviceIdentifier)
|
|
.withClientId(pushserviceThriftClientId)
|
|
.build[UserService.MethodPerEndpoint](
|
|
dest = "/s/gizmoduck/gizmoduck"
|
|
)
|
|
|
|
/**
|
|
* RequestContext test user config to allow reading test user accounts on pushservice for load
|
|
* testing
|
|
*/
|
|
val GizmoduckTestUserConfig = TestUserConfig(
|
|
clientId = Some(pushserviceThriftClientId.name),
|
|
readConfig = Some(ReadConfig(includeTestUsers = true))
|
|
)
|
|
|
|
TestUserClientBuilder[UserService.MethodPerEndpoint]
|
|
.withClient(client)
|
|
.withConfig(GizmoduckTestUserConfig)
|
|
.build()
|
|
}
|
|
|
|
val sgsClient = {
|
|
val service = readOnlyThriftService(
|
|
"",
|
|
"/s/socialgraph/socialgraph",
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
)
|
|
new SocialGraphService.FinagledClient(service)
|
|
}
|
|
|
|
val tweetyPieClient = {
|
|
val service = readOnlyThriftService(
|
|
"",
|
|
"/s/tweetypie/tweetypie",
|
|
statsReceiver,
|
|
notifierThriftClientId,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
)
|
|
new TweetService.FinagledClient(service)
|
|
}
|
|
|
|
lazy val geoduckHydrationClient: Hydration.MethodPerEndpoint = {
|
|
val servicePerEndpoint = ThriftMux.client
|
|
.withLabel("geoduck_hydration")
|
|
.withClientId(pushserviceThriftClientId)
|
|
.withMutualTls(serviceIdentifier)
|
|
.methodBuilder("/s/geo/hydration")
|
|
.withTimeoutPerRequest(10.seconds)
|
|
.withTimeoutTotal(10.seconds)
|
|
.idempotent(maxExtraLoad = 0.0)
|
|
.servicePerEndpoint[Hydration.ServicePerEndpoint]
|
|
Hydration.MethodPerEndpoint(servicePerEndpoint)
|
|
}
|
|
|
|
lazy val geoduckLocationClient: LocationService.MethodPerEndpoint = {
|
|
val servicePerEndpoint = ThriftMux.client
|
|
.withLabel("geoduck_location")
|
|
.withClientId(pushserviceThriftClientId)
|
|
.withMutualTls(serviceIdentifier)
|
|
.methodBuilder("/s/geo/geoduck_locationservice")
|
|
.withTimeoutPerRequest(10.seconds)
|
|
.withTimeoutTotal(10.seconds)
|
|
.idempotent(maxExtraLoad = 0.0)
|
|
.servicePerEndpoint[LocationService.ServicePerEndpoint]
|
|
LocationService.MethodPerEndpoint(servicePerEndpoint)
|
|
}
|
|
|
|
override val geoDuckV2Store: ReadableStore[Long, LocationResponse] = {
|
|
val geoduckLocate: GeoduckUserLocate = GeoduckUserLocateModule.providesGeoduckUserLocate(
|
|
locationServiceClient = geoduckLocationClient,
|
|
hydrationClient = geoduckHydrationClient,
|
|
unscopedStatsReceiver = statsReceiver
|
|
)
|
|
|
|
val store: ReadableStore[Long, LocationResponse] = ReadableStore
|
|
.convert[GeoduckRequest, Long, LocationResponse, LocationResponse](
|
|
GeoduckStoreV2(geoduckLocate))({ userId: Long =>
|
|
GeoduckRequest(
|
|
userId,
|
|
placeTypes = Set(
|
|
PlaceType.City,
|
|
PlaceType.Metro,
|
|
PlaceType.Country,
|
|
PlaceType.ZipCode,
|
|
PlaceType.Admin0,
|
|
PlaceType.Admin1),
|
|
placeFields = Set(PlaceQueryFields.PlaceNames),
|
|
includeCountryCode = true
|
|
)
|
|
})({ locationResponse: LocationResponse => Future.value(locationResponse) })
|
|
|
|
val _cacheName = "geoduckv2_in_memory_cache"
|
|
ObservedCachedReadableStore.from(
|
|
store,
|
|
ttl = 20.seconds,
|
|
maxKeys = 1000,
|
|
cacheName = _cacheName,
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope(_cacheName))
|
|
}
|
|
|
|
private val deepbirdServiceBase = ThriftMux.client
|
|
.withClientId(pushserviceThriftClientId)
|
|
.withMutualTls(serviceIdentifier)
|
|
.withLoadBalancer(Balancers.p2c())
|
|
.newService(deepbirdv2PredictionServiceDest, "DeepbirdV2PredictionService")
|
|
val deepbirdPredictionServiceClient = new DeepbirdPredictionService.ServiceToClient(
|
|
Finagle
|
|
.retryReadFilter(
|
|
tries = 3,
|
|
statsReceiver = statsReceiver.scope("DeepbirdV2PredictionService"))
|
|
.andThen(Finagle.timeoutFilter(timeout = 10.seconds))
|
|
.andThen(deepbirdServiceBase),
|
|
RichClientParam(serviceName = "DeepbirdV2PredictionService", clientStats = statsReceiver)
|
|
)
|
|
|
|
val manhattanStarbuckAppId = "frigate_pushservice_starbuck"
|
|
val metastoreLocationAppId = "frigate_notifier_metastore_location"
|
|
val manhattanMetastoreAppId = "frigate_pushservice_penguin"
|
|
|
|
def pushServiceMHCacheDest: String
|
|
def pushServiceCoreSvcsCacheDest: String
|
|
def poptartImpressionsCacheDest: String = "/srv#/prod/local/cache/poptart_impressions"
|
|
def entityGraphCacheDest: String
|
|
|
|
val pushServiceCacheClient: Client = MemcacheStore.memcachedClient(
|
|
name = ClientName("memcache-pushservice"),
|
|
dest = ZkEndPoint(pushServiceMHCacheDest),
|
|
statsReceiver = statsReceiver,
|
|
timeout = 2.seconds,
|
|
serviceIdentifier = serviceIdentifier
|
|
)
|
|
|
|
val pushServiceCoreSvcsCacheClient: Client =
|
|
MemcacheStore.memcachedClient(
|
|
name = ClientName("memcache-pushservice-core-svcs"),
|
|
dest = ZkEndPoint(pushServiceCoreSvcsCacheDest),
|
|
statsReceiver = statsReceiver,
|
|
serviceIdentifier = serviceIdentifier,
|
|
timeout = 2.seconds,
|
|
)
|
|
|
|
val poptartImpressionsCacheClient: Client =
|
|
MemcacheStore.memcachedClient(
|
|
name = ClientName("memcache-pushservice-poptart-impressions"),
|
|
dest = ZkEndPoint(poptartImpressionsCacheDest),
|
|
statsReceiver = statsReceiver,
|
|
serviceIdentifier = serviceIdentifier,
|
|
timeout = 2.seconds
|
|
)
|
|
|
|
val entityGraphCacheClient: Client = MemcacheStore.memcachedClient(
|
|
name = ClientName("memcache-pushservice-entity-graph"),
|
|
dest = ZkEndPoint(entityGraphCacheDest),
|
|
statsReceiver = statsReceiver,
|
|
serviceIdentifier = serviceIdentifier,
|
|
timeout = 2.seconds
|
|
)
|
|
|
|
val stratoClient = {
|
|
val pushserviceThriftClient = ThriftMux.client.withClientId(pushserviceThriftClientId)
|
|
val baseBuilder = Strato
|
|
.Client(pushserviceThriftClient)
|
|
.withMutualTls(serviceIdentifier)
|
|
val finalBuilder = if (isServiceLocal) {
|
|
baseBuilder.withRequestTimeout(Duration.fromSeconds(15))
|
|
} else {
|
|
baseBuilder.withRequestTimeout(Duration.fromSeconds(3))
|
|
}
|
|
finalBuilder.build()
|
|
}
|
|
|
|
val interestThriftServiceClient = ThriftMux.client
|
|
.withClientId(pushserviceThriftClientId)
|
|
.withMutualTls(serviceIdentifier)
|
|
.withRequestTimeout(3.seconds)
|
|
.configured(Retries.Policy(RetryPolicy.tries(1)))
|
|
.configured(BackupRequestFilter.Configured(maxExtraLoad = 0.0, sendInterrupts = false))
|
|
.withStatsReceiver(statsReceiver)
|
|
.build[InterestsThriftService.MethodPerEndpoint](
|
|
dest = "/s/interests-thrift-service/interests-thrift-service",
|
|
label = "interests-lookup"
|
|
)
|
|
|
|
def memcacheCASDest: String
|
|
|
|
override val casLock: CasLock = {
|
|
val magicrecsCasMemcacheClient = Memcached.client
|
|
.withMutualTls(serviceIdentifier)
|
|
.withLabel("mr-cas-memcache-client")
|
|
.withRequestTimeout(3.seconds)
|
|
.withStatsReceiver(statsReceiver)
|
|
.configured(Retries.Policy(RetryPolicy.tries(3)))
|
|
.newTwemcacheClient(memcacheCASDest)
|
|
.withStrings
|
|
|
|
MemcacheCasLock(magicrecsCasMemcacheClient)
|
|
}
|
|
|
|
override val pushInfoStore: ReadableStore[Long, UserForPushTargeting] = {
|
|
StratoFetchableStore.withUnitView[Long, UserForPushTargeting](
|
|
stratoClient,
|
|
"frigate/magicrecs/pushRecsTargeting.User")
|
|
}
|
|
|
|
override val loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] = {
|
|
StratoFetchableStore.withUnitView[Long, LOWebNotificationMetadata](
|
|
stratoClient,
|
|
"frigate/magicrecs/web/loggedOutWebUserStoreMh"
|
|
)
|
|
}
|
|
|
|
// Setting up model stores
|
|
override val dauProbabilityStore: ReadableStore[Long, DauProbability] = {
|
|
StratoFetchableStore
|
|
.withUnitView[Long, DauProbability](stratoClient, "frigate/magicrecs/dauProbability.User")
|
|
}
|
|
|
|
override val nsfwConsumerStore = {
|
|
StratoFetchableStore.withUnitView[Long, NSFWUserSegmentation](
|
|
stratoClient,
|
|
"frigate/nsfw-user-segmentation/nsfwUserSegmentation.User")
|
|
}
|
|
|
|
override val nsfwProducerStore = {
|
|
StratoFetchableStore.withUnitView[Long, NSFWProducer](
|
|
stratoClient,
|
|
"frigate/nsfw-user-segmentation/nsfwProducer.User"
|
|
)
|
|
}
|
|
|
|
override val idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] = {
|
|
val service = Finagle.readOnlyThriftService(
|
|
name = "interests-discovery-service",
|
|
dest = "/s/interests_discovery/interests_discovery",
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
requestTimeout = 4.seconds,
|
|
tries = 2,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
)
|
|
val client = new InterestsDiscoveryService.FinagledClient(
|
|
service = service,
|
|
RichClientParam(serviceName = "interests-discovery-service")
|
|
)
|
|
|
|
InterestDiscoveryStore(client)
|
|
}
|
|
|
|
override val popGeoLists = {
|
|
StratoFetchableStore.withUnitView[String, NonPersonalizedRecommendedLists](
|
|
stratoClient,
|
|
column = "recommendations/interests_discovery/recommendations_mh/OrganicPopgeoLists"
|
|
)
|
|
}
|
|
|
|
override val listAPIStore = {
|
|
val fetcher = stratoClient
|
|
.fetcher[Long, ApiListView, ApiList]("channels/hydration/apiList.List")
|
|
StratoFetchableStore.withView[Long, ApiListView, ApiList](
|
|
fetcher,
|
|
ApiListView(ApiListDisplayLocation.Recommendations)
|
|
)
|
|
}
|
|
|
|
override val reactivatedUserInfoStore = {
|
|
val stratoFetchableStore = StratoFetchableStore
|
|
.withUnitView[Long, String](stratoClient, "ml/featureStore/recentReactivationTime.User")
|
|
|
|
ObservedReadableStore(
|
|
stratoFetchableStore
|
|
)(statsReceiver.scope("RecentReactivationTime"))
|
|
}
|
|
|
|
override val openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]] = {
|
|
StratoFetchableStore
|
|
.withUnitView[Long, Map[Int, Int]](
|
|
stratoClient,
|
|
"frigate/magicrecs/opendPushByHourAggregated.User")
|
|
}
|
|
|
|
private val lexClient: LiveVideoTimelineClient = {
|
|
val lexService =
|
|
new TimelineService.FinagledClient(
|
|
readOnlyThriftService(
|
|
name = "lex",
|
|
dest = lexServiceDest,
|
|
statsReceiver = statsReceiver.scope("lex-service"),
|
|
thriftClientId = pushserviceThriftClientId,
|
|
requestTimeout = 5.seconds,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
),
|
|
clientParam = RichClientParam(serviceName = "lex")
|
|
)
|
|
new LiveVideoTimelineClient(lexService)
|
|
}
|
|
|
|
override val lexServiceStore = {
|
|
ObservedCachedReadableStore.from[EventRequest, LiveEvent](
|
|
buildStore(LexServiceStore(lexClient), "lexServiceStore"),
|
|
ttl = 1.hour,
|
|
maxKeys = 1000,
|
|
cacheName = "lexServiceStore_cache",
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope("lexServiceStore_cache"))
|
|
}
|
|
|
|
val inferredEntitiesFromInterestedInKeyedByClusterColumn =
|
|
"recommendations/simclusters_v2/inferred_entities/inferredEntitiesFromInterestedInKeyedByCluster"
|
|
override val simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities] = {
|
|
val store = StratoFetchableStore
|
|
.withUnitView[Int, SimClustersInferredEntities](
|
|
stratoClient,
|
|
inferredEntitiesFromInterestedInKeyedByClusterColumn)
|
|
ObservedCachedReadableStore.from[Int, SimClustersInferredEntities](
|
|
buildStore(store, "simcluster_entity_store_cache"),
|
|
ttl = 6.hours,
|
|
maxKeys = 1000,
|
|
cacheName = "simcluster_entity_store_cache",
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope("simcluster_entity_store_cache"))
|
|
}
|
|
|
|
def fanoutMetadataColumn: String
|
|
|
|
override val fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent] = {
|
|
val store = StratoFetchableStore
|
|
.withUnitView[(Long, Long), FanoutEvent](stratoClient, fanoutMetadataColumn)
|
|
ObservedCachedReadableStore.from[(Long, Long), FanoutEvent](
|
|
buildStore(store, "fanoutMetadataStore"),
|
|
ttl = 10.minutes,
|
|
maxKeys = 1000,
|
|
cacheName = "fanoutMetadataStore_cache",
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope("fanoutMetadataStore_cache"))
|
|
}
|
|
|
|
/**
|
|
* PostRanking Feature Store Client
|
|
*/
|
|
override def postRankingFeatureStoreClient = {
|
|
val clientStats = statsReceiver.scope("post_ranking_feature_store_client")
|
|
val clientConfig =
|
|
FeatureStoreClientBuilder.getClientConfig(PostRankingFeaturesConfig(), featureStoreUtil)
|
|
|
|
FeatureStoreClientBuilder.getDynamicFeatureStoreClient(clientConfig, clientStats)
|
|
}
|
|
|
|
/**
|
|
* Interests lookup store
|
|
*/
|
|
override val interestsWithLookupContextStore = {
|
|
ObservedCachedReadableStore.from[InterestsLookupRequestWithContext, Interests](
|
|
buildStore(
|
|
new InterestsWithLookupContextStore(interestThriftServiceClient, statsReceiver),
|
|
"InterestsWithLookupContextStore"
|
|
),
|
|
ttl = 1.minute,
|
|
maxKeys = 1000,
|
|
cacheName = "interestsWithLookupContextStore_cache",
|
|
windowSize = 10000L
|
|
)
|
|
}
|
|
|
|
/**
|
|
* OptOutInterestsStore
|
|
*/
|
|
override lazy val optOutUserInterestsStore: ReadableStore[Long, Seq[InterestId]] = {
|
|
buildStore(
|
|
InterestsOptOutwithLookUpContextStore(interestThriftServiceClient),
|
|
"InterestsOptOutStore"
|
|
)
|
|
}
|
|
|
|
override val topicListing: TopicListing =
|
|
if (isServiceLocal) {
|
|
new TopicListingBuilder(statsReceiver.scope("topiclisting"), Some(localConfigRepoPath)).build
|
|
} else {
|
|
new TopicListingBuilder(statsReceiver.scope("topiclisting"), None).build
|
|
}
|
|
|
|
val cachedUttClient = {
|
|
val DefaultUttCacheConfig = CacheConfigV2(capacity = 100)
|
|
val uttClientCacheConfigs = uttclient.UttClientCacheConfigsV2(
|
|
DefaultUttCacheConfig,
|
|
DefaultUttCacheConfig,
|
|
DefaultUttCacheConfig,
|
|
DefaultUttCacheConfig
|
|
)
|
|
new CachedUttClientV2(stratoClient, Environment.Prod, uttClientCacheConfigs, statsReceiver)
|
|
}
|
|
|
|
override val uttEntityHydrationStore =
|
|
new UttEntityHydrationStore(cachedUttClient, statsReceiver, log)
|
|
|
|
private lazy val dbv2PredictionServiceScoreStore: RelevancePushPredictionServiceStore =
|
|
DeepbirdV2ModelConfig.buildPredictionServiceScoreStore(
|
|
deepbirdPredictionServiceClient,
|
|
"deepbirdv2_magicrecs"
|
|
)
|
|
|
|
// Customized model to PredictionServiceStoreMap
|
|
// It is used to specify the predictionServiceStore for the models not in the default dbv2PredictionServiceScoreStore
|
|
private lazy val modelToPredictionServiceStoreMap: Map[
|
|
WeightedOpenOrNtabClickModel.ModelNameType,
|
|
RelevancePushPredictionServiceStore
|
|
] = Map()
|
|
|
|
override lazy val weightedOpenOrNtabClickModelScorer = new PushMLModelScorer(
|
|
PushMLModel.WeightedOpenOrNtabClickProbability,
|
|
modelToPredictionServiceStoreMap,
|
|
dbv2PredictionServiceScoreStore,
|
|
statsReceiver.scope("weighted_oonc_scoring")
|
|
)
|
|
|
|
override lazy val optoutModelScorer = new PushMLModelScorer(
|
|
PushMLModel.OptoutProbability,
|
|
Map.empty,
|
|
dbv2PredictionServiceScoreStore,
|
|
statsReceiver.scope("optout_scoring")
|
|
)
|
|
|
|
override lazy val filteringModelScorer = new PushMLModelScorer(
|
|
PushMLModel.FilteringProbability,
|
|
Map.empty,
|
|
dbv2PredictionServiceScoreStore,
|
|
statsReceiver.scope("filtering_scoring")
|
|
)
|
|
|
|
private val queryFields: Set[QueryFields] = Set(
|
|
QueryFields.Profile,
|
|
QueryFields.Account,
|
|
QueryFields.Roles,
|
|
QueryFields.Discoverability,
|
|
QueryFields.Safety,
|
|
QueryFields.Takedowns,
|
|
QueryFields.Labels,
|
|
QueryFields.Counts,
|
|
QueryFields.ExtendedProfile
|
|
)
|
|
|
|
// Setting up safeUserStore
|
|
override val safeUserStore =
|
|
// in-memory cache
|
|
ObservedCachedReadableStore.from[Long, User](
|
|
ObservedReadableStore(
|
|
GizmoduckUserStore.safeStore(
|
|
client = gizmoduckClient,
|
|
queryFields = queryFields,
|
|
safetyLevel = SafetyLevel.FilterNone,
|
|
statsReceiver = statsReceiver
|
|
)
|
|
)(statsReceiver.scope("SafeUserStore")),
|
|
ttl = 1.minute,
|
|
maxKeys = 5e4.toInt,
|
|
cacheName = "safeUserStore_cache",
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope("safeUserStore_cache"))
|
|
|
|
val mobileSdkStore = MobileSdkStore(
|
|
"frigate_mobile_sdk_version_apollo",
|
|
"mobile_sdk_versions_scalding",
|
|
manhattanClientMtlsParams,
|
|
Apollo
|
|
)
|
|
|
|
val deviceUserStore = ObservedReadableStore(
|
|
GizmoduckUserStore(
|
|
client = gizmoduckClient,
|
|
queryFields = Set(QueryFields.Devices),
|
|
context = LookupContext(includeSoftUsers = true),
|
|
statsReceiver = statsReceiver
|
|
)
|
|
)(statsReceiver.scope("devicesUserStore"))
|
|
|
|
override val deviceInfoStore = DeviceInfoStore(
|
|
ObservedMemcachedReadableStore.fromCacheClient(
|
|
backingStore = ObservedReadableStore(
|
|
mobileSdkStore
|
|
)(statsReceiver.scope("uncachedMobileSdkVersionsStore")),
|
|
cacheClient = pushServiceCacheClient,
|
|
ttl = 12.hours
|
|
)(
|
|
valueInjection = BinaryScalaCodec(SdkVersionValue),
|
|
statsReceiver = statsReceiver.scope("MobileSdkVersionsStore"),
|
|
keyToString = {
|
|
case SdkVersionKey(Some(userId), Some(clientId)) =>
|
|
s"DeviceInfoStore/$userId/$clientId"
|
|
case SdkVersionKey(Some(userId), None) => s"DeviceInfoStore/$userId/_"
|
|
case SdkVersionKey(None, Some(clientId)) =>
|
|
s"DeviceInfoStore/_/$clientId"
|
|
case SdkVersionKey(None, None) => s"DeviceInfoStore/_"
|
|
}
|
|
),
|
|
deviceUserStore
|
|
)
|
|
|
|
// Setting up edgeStore
|
|
override val edgeStore = SocialGraphPredicate.buildEdgeStore(sgsClient)
|
|
|
|
override val socialGraphServiceProcessStore = SocialGraphServiceProcessStore(edgeStore)
|
|
|
|
def userTweetEntityGraphDest: String
|
|
def userUserGraphDest: String
|
|
def lexServiceDest: String
|
|
|
|
// Setting up the history store
|
|
def frigateHistoryCacheDest: String
|
|
|
|
val notificationHistoryStore: NotificationHistoryStore = {
|
|
|
|
val manhattanStackBasedClient = ThriftMux.client
|
|
.withClientId(notifierThriftClientId)
|
|
.withOpportunisticTls(OpportunisticTls.Required)
|
|
.withMutualTls(
|
|
serviceIdentifier
|
|
)
|
|
|
|
val manhattanHistoryMethodBuilder = manhattanStackBasedClient
|
|
.withLabel("manhattan_history_v2")
|
|
.withRequestTimeout(10.seconds)
|
|
.withStatsReceiver(statsReceiver)
|
|
.methodBuilder(Omega.wilyName)
|
|
.withMaxRetries(3)
|
|
|
|
NotificationHistoryStore.build(
|
|
"frigate_notifier",
|
|
"frigate_notifications_v2",
|
|
manhattanHistoryMethodBuilder,
|
|
maxRetryCount = 3
|
|
)
|
|
}
|
|
|
|
val emailNotificationHistoryStore: ReadOnlyHistoryStore = {
|
|
val client = ManhattanKVClient(
|
|
appId = "frigate_email_history",
|
|
dest = "/s/manhattan/omega.native-thrift",
|
|
mtlsParams = ManhattanKVClientMtlsParams(
|
|
serviceIdentifier = serviceIdentifier,
|
|
opportunisticTls = OpportunisticTls.Required
|
|
)
|
|
)
|
|
val endpoint = ManhattanKVEndpointBuilder(client)
|
|
.defaultGuarantee(Guarantee.SoftDcReadMyWrites)
|
|
.statsReceiver(statsReceiver)
|
|
.build()
|
|
|
|
ReadOnlyHistoryStore(ManhattanKVHistoryStore(endpoint, dataset = "frigate_email_history"))(
|
|
statsReceiver)
|
|
}
|
|
|
|
val manhattanKVLoggedOutHistoryStoreEndpoint: ManhattanKVEndpoint = {
|
|
val mhClient = ManhattanKVClient(
|
|
"frigate_notification_logged_out_history",
|
|
Nash.wilyName,
|
|
manhattanClientMtlsParams)
|
|
ManhattanKVEndpointBuilder(mhClient)
|
|
.defaultGuarantee(Guarantee.SoftDcReadMyWrites)
|
|
.defaultMaxTimeout(5.seconds)
|
|
.maxRetryCount(3)
|
|
.statsReceiver(statsReceiver)
|
|
.build()
|
|
}
|
|
|
|
val manhattanKVNtabHistoryStoreEndpoint: ManhattanKVEndpoint = {
|
|
val mhClient = ManhattanKVClient("frigate_ntab", Omega.wilyName, manhattanClientMtlsParams)
|
|
ManhattanKVEndpointBuilder(mhClient)
|
|
.defaultGuarantee(Guarantee.SoftDcReadMyWrites)
|
|
.defaultMaxTimeout(5.seconds)
|
|
.maxRetryCount(3)
|
|
.statsReceiver(statsReceiver)
|
|
.build()
|
|
}
|
|
|
|
val nTabHistoryStore: ReadableWritableStore[(Long, String), GenericNotificationOverrideKey] = {
|
|
ObservedReadableWritableStore(
|
|
NTabHistoryStore(manhattanKVNtabHistoryStoreEndpoint, "frigate_ntab_generic_notif_history")
|
|
)(statsReceiver.scope("NTabHistoryStore"))
|
|
}
|
|
|
|
override lazy val ocfFatigueStore: ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] =
|
|
new OCFPromptHistoryStore(
|
|
manhattanAppId = "frigate_pushservice_ocf_fatigue_store",
|
|
dataset = "fatigue_v1",
|
|
manhattanClientMtlsParams
|
|
)
|
|
|
|
def historyStore: PushServiceHistoryStore
|
|
|
|
def emailHistoryStore: PushServiceHistoryStore
|
|
|
|
def loggedOutHistoryStore: PushServiceHistoryStore
|
|
|
|
override val hydratedLabeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue] = {
|
|
val labeledHistoryMemcacheClient = {
|
|
MemcacheStore.memcachedClient(
|
|
name = ClientName("history-memcache"),
|
|
dest = ZkEndPoint(frigateHistoryCacheDest),
|
|
statsReceiver = statsReceiver,
|
|
timeout = 2.seconds,
|
|
serviceIdentifier = serviceIdentifier
|
|
)
|
|
}
|
|
|
|
implicit val keyCodec = CompactScalaCodec(UserHistoryKey)
|
|
implicit val valueCodec = CompactScalaCodec(UserHistoryValue)
|
|
val dataset: Dataset[UserHistoryKey, UserHistoryValue] =
|
|
Dataset(
|
|
"",
|
|
"frigate_data_pipeline_pushservice",
|
|
"labeled_push_recs_aggregated_hydrated",
|
|
Athena
|
|
)
|
|
ObservedMemcachedReadableStore.fromCacheClient(
|
|
backingStore = ObservedReadableStore(buildManhattanStore(dataset))(
|
|
statsReceiver.scope("UncachedHydratedLabeledPushRecsStore")
|
|
),
|
|
cacheClient = labeledHistoryMemcacheClient,
|
|
ttl = 6.hours
|
|
)(
|
|
valueInjection = valueCodec,
|
|
statsReceiver = statsReceiver.scope("HydratedLabeledPushRecsStore"),
|
|
keyToString = {
|
|
case UserHistoryKey.UserId(userId) => s"HLPRS/$userId"
|
|
case unknownKey =>
|
|
throw new IllegalArgumentException(s"Unknown userHistoryStore cache key $unknownKey")
|
|
}
|
|
)
|
|
}
|
|
|
|
override val realTimeClientEventStore: RealTimeClientEventStore = {
|
|
val client = ManhattanKVClient(
|
|
"frigate_eventstream",
|
|
"/s/manhattan/omega.native-thrift",
|
|
manhattanClientMtlsParams
|
|
)
|
|
val endpoint =
|
|
ManhattanKVEndpointBuilder(client)
|
|
.defaultGuarantee(Guarantee.SoftDcReadMyWrites)
|
|
.defaultMaxTimeout(3.seconds)
|
|
.statsReceiver(statsReceiver)
|
|
.build()
|
|
|
|
ManhattanRealTimeClientEventStore(endpoint, "realtime_client_events", statsReceiver, None)
|
|
}
|
|
|
|
override val onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue] = {
|
|
OnlineUserHistoryStore(realTimeClientEventStore)
|
|
}
|
|
|
|
override val userMediaRepresentationStore = UserMediaRepresentationStore(
|
|
"user_media_representation",
|
|
"user_media_representation_dataset",
|
|
manhattanClientMtlsParams
|
|
)
|
|
|
|
override val producerMediaRepresentationStore = ObservedMemcachedReadableStore.fromCacheClient(
|
|
backingStore = UserMediaRepresentationStore(
|
|
"user_media_representation",
|
|
"producer_media_representation_dataset",
|
|
manhattanClientMtlsParams
|
|
)(statsReceiver.scope("UncachedProducerMediaRepStore")),
|
|
cacheClient = pushServiceCacheClient,
|
|
ttl = 4.hours
|
|
)(
|
|
valueInjection = BinaryScalaCodec(UserMediaRepresentation),
|
|
keyToString = { k: Long => s"ProducerMediaRepStore/$k" },
|
|
statsReceiver.scope("ProducerMediaRepStore")
|
|
)
|
|
|
|
override val mrUserStatePredictionStore = {
|
|
StratoFetchableStore.withUnitView[Long, MRUserHmmState](
|
|
stratoClient,
|
|
"frigate/magicrecs/mrUserStatePrediction.User")
|
|
}
|
|
|
|
override val userHTLLastVisitStore =
|
|
UserHTLLastVisitReadableStore(
|
|
"pushservice_htl_user_session",
|
|
"tls_user_session_store",
|
|
statsReceiver.scope("userHTLLastVisitStore"),
|
|
manhattanClientMtlsParams
|
|
)
|
|
|
|
val crMixerClient: CrMixer.MethodPerEndpoint = new CrMixer.FinagledClient(
|
|
readOnlyThriftService(
|
|
"cr-mixer",
|
|
"/s/cr-mixer/cr-mixer-plus",
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
requestTimeout = 5.seconds,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
),
|
|
clientParam = RichClientParam(serviceName = "cr-mixer")
|
|
)
|
|
|
|
val crMixerStore = CrMixerTweetStore(crMixerClient)(statsReceiver.scope("CrMixerTweetStore"))
|
|
|
|
val contentMixerClient: ContentMixer.MethodPerEndpoint = new ContentMixer.FinagledClient(
|
|
readOnlyThriftService(
|
|
"content-mixer",
|
|
"/s/corgi-shared/content-mixer",
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
requestTimeout = 5.seconds,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
),
|
|
clientParam = RichClientParam(serviceName = "content-mixer")
|
|
)
|
|
|
|
val exploreRankerClient: ExploreRanker.MethodPerEndpoint =
|
|
new ExploreRanker.FinagledClient(
|
|
readOnlyThriftService(
|
|
"explore-ranker",
|
|
"/s/explore-ranker/explore-ranker",
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
requestTimeout = 5.seconds,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
),
|
|
clientParam = RichClientParam(serviceName = "explore-ranker")
|
|
)
|
|
|
|
val contentMixerStore = {
|
|
ObservedReadableStore(ContentMixerStore(contentMixerClient))(
|
|
statsReceiver.scope("ContentMixerStore"))
|
|
}
|
|
|
|
val exploreRankerStore = {
|
|
ObservedReadableStore(ExploreRankerStore(exploreRankerClient))(
|
|
statsReceiver.scope("ExploreRankerStore")
|
|
)
|
|
}
|
|
|
|
val gizmoduckUtcOffsetStore = ObservedReadableStore(
|
|
GizmoduckUserUtcOffsetStore.fromUserStore(safeUserStore)
|
|
)(statsReceiver.scope("GizmoUserUtcOffsetStore"))
|
|
|
|
override val userUtcOffsetStore =
|
|
UtcOffsetStore
|
|
.makeMemcachedUtcOffsetStore(
|
|
gizmoduckUtcOffsetStore,
|
|
pushServiceCoreSvcsCacheClient,
|
|
ReadableStore.empty,
|
|
manhattanStarbuckAppId,
|
|
manhattanClientMtlsParams
|
|
)(statsReceiver)
|
|
.mapValues(Duration.fromSeconds)
|
|
|
|
override val cachedTweetyPieStoreV2 = {
|
|
val getTweetOptions = Some(
|
|
GetTweetOptions(
|
|
safetyLevel = Some(SafetyLevel.MagicRecsV2),
|
|
includeRetweetCount = true,
|
|
includeReplyCount = true,
|
|
includeFavoriteCount = true,
|
|
includeQuotedTweet = true,
|
|
additionalFieldIds = Seq(VisibleTextRangeField.id)
|
|
)
|
|
)
|
|
buildCachedTweetyPieStore(getTweetOptions, "tp_v2")
|
|
}
|
|
|
|
override val cachedTweetyPieStoreV2NoVF = {
|
|
val getTweetOptions = Some(
|
|
GetTweetOptions(
|
|
safetyLevel = Some(SafetyLevel.FilterDefault),
|
|
includeRetweetCount = true,
|
|
includeReplyCount = true,
|
|
includeFavoriteCount = true,
|
|
includeQuotedTweet = true,
|
|
additionalFieldIds = Seq(VisibleTextRangeField.id),
|
|
)
|
|
)
|
|
buildCachedTweetyPieStore(getTweetOptions, "tp_v2_noVF")
|
|
}
|
|
|
|
override val safeCachedTweetyPieStoreV2 = {
|
|
val getTweetOptions = Some(
|
|
GetTweetOptions(
|
|
safetyLevel = Some(SafetyLevel.MagicRecsAggressiveV2),
|
|
includeRetweetCount = true,
|
|
includeReplyCount = true,
|
|
includeFavoriteCount = true,
|
|
includeQuotedTweet = true,
|
|
additionalFieldIds = Seq(VisibleTextRangeField.id)
|
|
)
|
|
)
|
|
buildCachedTweetyPieStore(getTweetOptions, "sftp_v2")
|
|
}
|
|
|
|
override val userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult] = {
|
|
val getTweetOptions = Some(
|
|
GetTweetOptions(
|
|
safetyLevel = Some(SafetyLevel.MagicRecsV2),
|
|
includeRetweetCount = true,
|
|
includeReplyCount = true,
|
|
includeFavoriteCount = true,
|
|
includeQuotedTweet = true,
|
|
additionalFieldIds = Seq(VisibleTextRangeField.id)
|
|
)
|
|
)
|
|
TweetyPieStore.buildUserTweetStore(
|
|
client = tweetyPieClient,
|
|
options = getTweetOptions
|
|
)
|
|
}
|
|
|
|
override val safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult] = {
|
|
val getTweetOptions = Some(
|
|
GetTweetOptions(
|
|
safetyLevel = Some(SafetyLevel.MagicRecsAggressiveV2),
|
|
includeRetweetCount = true,
|
|
includeReplyCount = true,
|
|
includeFavoriteCount = true,
|
|
includeQuotedTweet = true,
|
|
additionalFieldIds = Seq(VisibleTextRangeField.id)
|
|
)
|
|
)
|
|
TweetyPieStore.buildUserTweetStore(
|
|
client = tweetyPieClient,
|
|
options = getTweetOptions
|
|
)
|
|
}
|
|
|
|
override val tweetContentFeatureCacheStore: ReadableStore[Long, ThriftDataRecord] = {
|
|
ObservedMemcachedReadableStore.fromCacheClient(
|
|
backingStore = TweetContentFeatureReadableStore(stratoClient),
|
|
cacheClient = poptartImpressionsCacheClient,
|
|
ttl = 12.hours
|
|
)(
|
|
valueInjection = BinaryScalaCodec(ThriftDataRecord),
|
|
statsReceiver = statsReceiver.scope("TweetContentFeaturesCacheStore"),
|
|
keyToString = { k: Long => s"tcf/$k" }
|
|
)
|
|
}
|
|
|
|
lazy val tweetTranslationStore: ReadableStore[
|
|
TweetTranslationStore.Key,
|
|
TweetTranslationStore.Value
|
|
] = {
|
|
val isTweetTranslatableStore =
|
|
StratoFetchableStore
|
|
.withUnitView[IsTweetTranslatableClientColumn.Key, Boolean](
|
|
fetcher = new IsTweetTranslatableClientColumn(stratoClient).fetcher
|
|
)
|
|
|
|
val translateTweetStore =
|
|
StratoFetchableStore
|
|
.withUnitView[MachineTranslateTweetClientColumn.Key, MachineTranslationResponse](
|
|
fetcher = new MachineTranslateTweetClientColumn(stratoClient).fetcher
|
|
)
|
|
|
|
ObservedReadableStore(
|
|
TweetTranslationStore(translateTweetStore, isTweetTranslatableStore, statsReceiver)
|
|
)(statsReceiver.scope("tweetTranslationStore"))
|
|
}
|
|
|
|
val scarecrowClient = new ScarecrowService.FinagledClient(
|
|
readOnlyThriftService(
|
|
"",
|
|
"/s/abuse/scarecrow",
|
|
statsReceiver,
|
|
notifierThriftClientId,
|
|
requestTimeout = 5.second,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
),
|
|
clientParam = RichClientParam(serviceName = "")
|
|
)
|
|
|
|
// Setting up scarecrow store
|
|
override val scarecrowCheckEventStore = {
|
|
ScarecrowCheckEventStore(scarecrowClient)
|
|
}
|
|
|
|
// setting up the perspective store
|
|
override val userTweetPerspectiveStore = {
|
|
val service = new DynamicRequestMeterFilter(
|
|
tunableMap(PushServiceTunableKeys.TweetPerspectiveStoreQpsLimit),
|
|
RateLimiterGenerator.asTuple(_, shardParams.numShards, 40),
|
|
PushQPSLimitConstants.PerspectiveStoreQPS)(timer)
|
|
.andThen(
|
|
readOnlyThriftService(
|
|
"tweetypie_perspective_service",
|
|
"/s/tweetypie/tweetypie",
|
|
statsReceiver,
|
|
notifierThriftClientId,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
)
|
|
)
|
|
|
|
val client = new TweetService.FinagledClient(
|
|
service,
|
|
clientParam = RichClientParam(serviceName = "tweetypie_perspective_client"))
|
|
ObservedReadableStore(
|
|
PerspectiveReadableStore(client)
|
|
)(statsReceiver.scope("TweetPerspectiveStore"))
|
|
}
|
|
|
|
//user country code store, used in RecsWithheldContentPredicate - wrapped by memcache based cache
|
|
override val userCountryStore =
|
|
ObservedMemcachedReadableStore.fromCacheClient(
|
|
backingStore = ObservedReadableStore(
|
|
UserCountryStore(metastoreLocationAppId, manhattanClientMtlsParams)
|
|
)(statsReceiver.scope("userCountryStore")),
|
|
cacheClient = pushServiceCacheClient,
|
|
ttl = 12.hours
|
|
)(
|
|
valueInjection = BinaryScalaCodec(Location),
|
|
statsReceiver = statsReceiver.scope("UserCountryStore"),
|
|
keyToString = { k: Long => s"UserCountryStore/$k" }
|
|
)
|
|
|
|
override val audioSpaceParticipantsStore: ReadableStore[String, Participants] = {
|
|
val store = StratoFetchableStore
|
|
.DefaultStratoFetchableStore(
|
|
fetcher = new ParticipantsOnAudioSpaceClientColumn(stratoClient).fetcher
|
|
).composeKeyMapping[String](broadcastId =>
|
|
(broadcastId, AudioSpacesLookupContext(forUserId = None)))
|
|
|
|
ObservedCachedReadableStore
|
|
.from(
|
|
store = buildStore(store, "AudioSpaceParticipantsStore"),
|
|
ttl = 20.seconds,
|
|
maxKeys = 200,
|
|
cacheName = "AudioSpaceParticipantsStore",
|
|
windowSize = 200
|
|
)
|
|
|
|
}
|
|
|
|
override val topicSocialProofServiceStore: ReadableStore[
|
|
TopicSocialProofRequest,
|
|
TopicSocialProofResponse
|
|
] = {
|
|
StratoFetchableStore.withUnitView[TopicSocialProofRequest, TopicSocialProofResponse](
|
|
stratoClient,
|
|
"topic-signals/tsp/topic-social-proof")
|
|
}
|
|
|
|
override val spaceDeviceFollowStore: ReadableStore[SourceDestUserRequest, Boolean] = {
|
|
StratoFetchableStore.withUnitView(
|
|
fetcher = new SpaceDeviceFollowingClientColumn(stratoClient).fetcher
|
|
)
|
|
}
|
|
|
|
override val audioSpaceStore: ReadableStore[String, AudioSpace] = {
|
|
val store = StratoFetchableStore
|
|
.DefaultStratoFetchableStore(
|
|
fetcher = new CoreOnAudioSpaceClientColumn(stratoClient).fetcher
|
|
).composeKeyMapping[String] { broadcastId =>
|
|
(broadcastId, AudioSpacesLookupContext(forUserId = None))
|
|
}
|
|
|
|
ObservedCachedReadableStore
|
|
.from(
|
|
store = buildStore(store, "AudioSpaceVisibilityStore"),
|
|
ttl = 1.minute,
|
|
maxKeys = 5000,
|
|
cacheName = "AudioSpaceVisibilityStore",
|
|
windowSize = 10000L)
|
|
}
|
|
|
|
override val userLanguagesStore = UserLanguagesStore(
|
|
manhattanMetastoreAppId,
|
|
manhattanClientMtlsParams,
|
|
statsReceiver.scope("user_languages_store")
|
|
)
|
|
|
|
val tflockClient: TFlockClient = new TFlockClient(
|
|
new FlockDB.FinagledClient(
|
|
readOnlyThriftService(
|
|
"tflockClient",
|
|
"/s/tflock/tflock",
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
),
|
|
serviceName = "tflock",
|
|
stats = statsReceiver
|
|
),
|
|
defaultPageSize = 1000
|
|
)
|
|
|
|
val rawFlockClient = ThriftMux.client
|
|
.withClientId(pushserviceThriftClientId)
|
|
.withMutualTls(serviceIdentifier)
|
|
.build[FlockDB.MethodPerEndpoint]("/s/flock/flock")
|
|
|
|
val flockClient: FlockClient = new FlockClient(
|
|
rawFlockClient,
|
|
defaultPageSize = 100
|
|
)
|
|
|
|
override val recentFollowsStore: FlockFollowStore = {
|
|
val dStats = statsReceiver.scope("FlockRecentFollowsStore")
|
|
FlockFollowStore(flockClient, dStats)
|
|
}
|
|
|
|
def notificationServiceClient: NotificationService$FinagleClient
|
|
|
|
def notificationServiceSend(
|
|
target: Target,
|
|
request: CreateGenericNotificationRequest
|
|
): Future[CreateGenericNotificationResponse]
|
|
|
|
def notificationServiceDelete(
|
|
request: DeleteGenericNotificationRequest
|
|
): Future[Unit]
|
|
|
|
def notificationServiceDeleteTimeline(
|
|
request: DeleteCurrentTimelineForUserRequest
|
|
): Future[Unit]
|
|
|
|
override val notificationServiceSender: ReadableStore[
|
|
NotificationServiceRequest,
|
|
CreateGenericNotificationResponse
|
|
] = {
|
|
new NotificationServiceSender(
|
|
notificationServiceSend,
|
|
PushParams.EnableWritesToNotificationServiceParam,
|
|
PushParams.EnableWritesToNotificationServiceForAllEmployeesParam,
|
|
PushParams.EnableWritesToNotificationServiceForEveryoneParam
|
|
)
|
|
}
|
|
|
|
val eventRecosServiceClient = {
|
|
val dest = "/s/events-recos/events-recos-service"
|
|
new EventsRecosService.FinagledClient(
|
|
readOnlyThriftService(
|
|
"EventRecosService",
|
|
dest,
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
),
|
|
clientParam = RichClientParam(serviceName = "EventRecosService")
|
|
)
|
|
}
|
|
|
|
lazy val recommendedTrendsCandidateSource = RecommendedTrendsCandidateSource(
|
|
TrendsRecommendationStore(eventRecosServiceClient, statsReceiver))
|
|
|
|
override val softUserGeoLocationStore: ReadableStore[Long, GeoLocation] =
|
|
StratoFetchableStore.withUnitView[Long, GeoLocation](fetcher =
|
|
new FrequentSoftUserLocationClientColumn(stratoClient).fetcher)
|
|
|
|
lazy val candidateSourceGenerator = new PushCandidateSourceGenerator(
|
|
earlybirdCandidateSource,
|
|
userTweetEntityGraphCandidates,
|
|
cachedTweetyPieStoreV2,
|
|
safeCachedTweetyPieStoreV2,
|
|
userTweetTweetyPieStore,
|
|
safeUserTweetTweetyPieStore,
|
|
cachedTweetyPieStoreV2NoVF,
|
|
edgeStore,
|
|
interestsWithLookupContextStore,
|
|
uttEntityHydrationStore,
|
|
geoDuckV2Store,
|
|
topTweetsByGeoStore,
|
|
topTweetsByGeoV2VersionedStore,
|
|
ruxTweetImpressionsStore,
|
|
recommendedTrendsCandidateSource,
|
|
recentTweetsByAuthorsStore,
|
|
topicSocialProofServiceStore,
|
|
crMixerStore,
|
|
contentMixerStore,
|
|
exploreRankerStore,
|
|
softUserGeoLocationStore,
|
|
tripTweetCandidateStore,
|
|
popGeoLists,
|
|
idsStore
|
|
)
|
|
|
|
lazy val loCandidateSourceGenerator = new LoggedOutPushCandidateSourceGenerator(
|
|
tripTweetCandidateStore,
|
|
geoDuckV2Store,
|
|
safeCachedTweetyPieStoreV2,
|
|
cachedTweetyPieStoreV2NoVF,
|
|
cachedTweetyPieStoreV2,
|
|
contentMixerStore,
|
|
softUserGeoLocationStore,
|
|
topTweetsByGeoStore,
|
|
topTweetsByGeoV2VersionedStore
|
|
)
|
|
|
|
lazy val rfphStatsRecorder = new RFPHStatsRecorder()
|
|
|
|
lazy val rfphRestrictStep = new RFPHRestrictStep()
|
|
|
|
lazy val rfphTakeStepUtil = new RFPHTakeStepUtil()(statsReceiver)
|
|
|
|
lazy val rfphPrerankFilter = new RFPHPrerankFilter()(statsReceiver)
|
|
|
|
lazy val rfphLightRanker = new RFPHLightRanker(lightRanker, statsReceiver)
|
|
|
|
lazy val sendHandlerPredicateUtil = new SendHandlerPredicateUtil()(statsReceiver)
|
|
|
|
lazy val ntabSender =
|
|
new NtabSender(
|
|
notificationServiceSender,
|
|
nTabHistoryStore,
|
|
notificationServiceDelete,
|
|
notificationServiceDeleteTimeline
|
|
)
|
|
|
|
lazy val ibis2Sender = new Ibis2Sender(pushIbisV2Store, tweetTranslationStore, statsReceiver)
|
|
|
|
lazy val historyWriter = new HistoryWriter(historyStore, statsReceiver)
|
|
|
|
lazy val loggedOutHistoryWriter = new HistoryWriter(loggedOutHistoryStore, statsReceiver)
|
|
|
|
lazy val eventBusWriter = new EventBusWriter(pushSendEventBusPublisher, statsReceiver)
|
|
|
|
lazy val ntabOnlyChannelSelector = new NtabOnlyChannelSelector
|
|
|
|
lazy val notificationSender =
|
|
new NotificationSender(
|
|
ibis2Sender,
|
|
ntabSender,
|
|
statsReceiver,
|
|
notificationScribe
|
|
)
|
|
|
|
lazy val candidateNotifier =
|
|
new CandidateNotifier(
|
|
notificationSender,
|
|
casLock = casLock,
|
|
historyWriter = historyWriter,
|
|
eventBusWriter = eventBusWriter,
|
|
ntabOnlyChannelSelector = ntabOnlyChannelSelector
|
|
)(statsReceiver)
|
|
|
|
lazy val loggedOutCandidateNotifier = new CandidateNotifier(
|
|
notificationSender,
|
|
casLock = casLock,
|
|
historyWriter = loggedOutHistoryWriter,
|
|
eventBusWriter = null,
|
|
ntabOnlyChannelSelector = ntabOnlyChannelSelector
|
|
)(statsReceiver)
|
|
|
|
lazy val rfphNotifier =
|
|
new RefreshForPushNotifier(rfphStatsRecorder, candidateNotifier)(statsReceiver)
|
|
|
|
lazy val loRfphNotifier =
|
|
new LoggedOutRefreshForPushNotifier(rfphStatsRecorder, loggedOutCandidateNotifier)(
|
|
statsReceiver)
|
|
|
|
lazy val rfphRanker = {
|
|
val randomRanker = RandomRanker[Target, PushCandidate]()
|
|
val subscriptionCreatorRanker =
|
|
new SubscriptionCreatorRanker(superFollowEligibilityUserStore, statsReceiver)
|
|
new RFPHRanker(
|
|
randomRanker,
|
|
weightedOpenOrNtabClickModelScorer,
|
|
subscriptionCreatorRanker,
|
|
userHealthSignalStore,
|
|
producerMediaRepresentationStore,
|
|
statsReceiver
|
|
)
|
|
}
|
|
|
|
lazy val rfphFeatureHydrator = new RFPHFeatureHydrator(featureHydrator)
|
|
lazy val loggedOutRFPHRanker = new LoggedOutRanker(cachedTweetyPieStoreV2, statsReceiver)
|
|
|
|
override val userFeaturesStore: ReadableStore[Long, UserFeatures] = {
|
|
implicit val valueCodec = new BinaryScalaCodec(UserFeatures)
|
|
val dataset: Dataset[Long, UserFeatures] =
|
|
Dataset(
|
|
"",
|
|
"user_features_pushservice_apollo",
|
|
"recommendations_user_features_apollo",
|
|
Apollo)
|
|
|
|
ObservedMemcachedReadableStore.fromCacheClient(
|
|
backingStore = ObservedReadableStore(buildManhattanStore(dataset))(
|
|
statsReceiver.scope("UncachedUserFeaturesStore")
|
|
),
|
|
cacheClient = pushServiceCacheClient,
|
|
ttl = 24.hours
|
|
)(
|
|
valueInjection = valueCodec,
|
|
statsReceiver = statsReceiver.scope("UserFeaturesStore"),
|
|
keyToString = { k: Long => s"ufts/$k" }
|
|
)
|
|
}
|
|
|
|
override def htlScoreStore(userId: Long): ReadableStore[Long, ScoredTweet] = {
|
|
val fetcher = new TimelineScorerTweetScoresV1ClientColumn(stratoClient).fetcher
|
|
val htlStore = buildStore(
|
|
StratoFetchableStore.withView[Long, TimelineScorerScoreView, ScoredTweet](
|
|
fetcher,
|
|
TimelineScorerScoreView(Some(userId))
|
|
),
|
|
"htlScoreStore"
|
|
)
|
|
htlStore
|
|
}
|
|
|
|
override val userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty] = {
|
|
val name = "userTargetingPropertyStore"
|
|
val store = StratoFetchableStore
|
|
.withUnitView(new TargetingPropertyOnUserClientColumn(stratoClient).fetcher)
|
|
buildStore(store, name)
|
|
}
|
|
|
|
override val timelinesUserSessionStore: ReadableStore[Long, UserSession] = {
|
|
implicit val valueCodec = new CompactScalaCodec(UserSession)
|
|
val dataset: Dataset[Long, UserSession] = Dataset[Long, UserSession](
|
|
"",
|
|
"frigate_realgraph",
|
|
"real_graph_user_features",
|
|
Apollo
|
|
)
|
|
|
|
ObservedMemcachedReadableStore.fromCacheClient(
|
|
backingStore = ObservedReadableStore(buildManhattanStore(dataset))(
|
|
statsReceiver.scope("UncachedTimelinesUserSessionStore")
|
|
),
|
|
cacheClient = pushServiceCacheClient,
|
|
ttl = 6.hours
|
|
)(
|
|
valueInjection = valueCodec,
|
|
statsReceiver = statsReceiver.scope("timelinesUserSessionStore"),
|
|
keyToString = { k: Long => s"tluss/$k" }
|
|
)
|
|
}
|
|
|
|
lazy val recentTweetsFromTflockStore: ReadableStore[Long, Seq[Long]] =
|
|
ObservedReadableStore(
|
|
RecentTweetsByAuthorsStore.usingRecentTweetsConfig(
|
|
tflockClient,
|
|
RecentTweetsConfig(maxResults = 1, maxAge = 3.days)
|
|
)
|
|
)(statsReceiver.scope("RecentTweetsFromTflockStore"))
|
|
|
|
lazy val recentTweetsByAuthorsStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]] =
|
|
ObservedReadableStore(
|
|
RecentTweetsByAuthorsStore(tflockClient)
|
|
)(statsReceiver.scope("RecentTweetsByAuthorsStore"))
|
|
|
|
val jobConfig = PopGeoInterestProvider
|
|
.getPopularTweetsJobConfig(
|
|
InterestDeployConfig(
|
|
AppId("PopularTweetsByInterestProd"),
|
|
Cluster.ATLA,
|
|
Env.Prod,
|
|
serviceIdentifier,
|
|
manhattanClientMtlsParams
|
|
))
|
|
.withManhattanAppId("frigate_pop_by_geo_tweets")
|
|
|
|
override val topTweetsByGeoStore = TopTweetsStore.withMemCache(
|
|
jobConfig,
|
|
pushServiceCacheClient,
|
|
10.seconds
|
|
)(statsReceiver)
|
|
|
|
override val topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace] = {
|
|
StratoFetchableStore.withUnitView[String, PopTweetsInPlace](
|
|
stratoClient,
|
|
"recommendations/popgeo/popGeoTweetsVersioned")
|
|
}
|
|
|
|
override lazy val pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory] = {
|
|
StratoFetchableStore.withUnitView[Long, PushcapUserHistory](
|
|
stratoClient,
|
|
"frigate/magicrecs/pushcapDynamicPrediction.User")
|
|
}
|
|
|
|
override val tweetAuthorLocationFeatureBuilder =
|
|
UserLocationFeatureBuilder(Some("TweetAuthor"))
|
|
.withStats()
|
|
|
|
override val tweetAuthorLocationFeatureBuilderById =
|
|
UserLocationFeatureBuilderById(
|
|
userCountryStore,
|
|
tweetAuthorLocationFeatureBuilder
|
|
).withStats()
|
|
|
|
override val socialContextActionsFeatureBuilder =
|
|
SocialContextActionsFeatureBuilder().withStats()
|
|
|
|
override val tweetContentFeatureBuilder =
|
|
TweetContentFeatureBuilder(tweetContentFeatureCacheStore).withStats()
|
|
|
|
override val tweetAuthorRecentRealGraphFeatureBuilder =
|
|
RecentRealGraphFeatureBuilder(
|
|
stratoClient,
|
|
UserAuthorEntity,
|
|
TargetUserEntity,
|
|
TweetAuthorEntity,
|
|
TweetAuthorRecentRealGraphFeatures(statsReceiver.scope("TweetAuthorRecentRealGraphFeatures"))
|
|
).withStats()
|
|
|
|
override val socialContextRecentRealGraphFeatureBuilder =
|
|
SocialContextRecentRealGraphFeatureBuilder(
|
|
RecentRealGraphFeatureBuilder(
|
|
stratoClient,
|
|
TargetUserSocialContextEntity,
|
|
TargetUserEntity,
|
|
SocialContextEntity,
|
|
SocialContextRecentRealGraphFeatures(
|
|
statsReceiver.scope("SocialContextRecentRealGraphFeatures"))
|
|
)(statsReceiver
|
|
.scope("SocialContextRecentRealGraphFeatureBuilder").scope("RecentRealGraphFeatureBuilder"))
|
|
).withStats()
|
|
|
|
override val tweetSocialProofFeatureBuilder =
|
|
TweetSocialProofFeatureBuilder(Some("TargetUser")).withStats()
|
|
|
|
override val targetUserFullRealGraphFeatureBuilder =
|
|
TargetFullRealGraphFeatureBuilder(Some("TargetUser")).withStats()
|
|
|
|
override val postProcessingFeatureBuilder: PostProcessingFeatureBuilder =
|
|
PostProcessingFeatureBuilder()
|
|
|
|
override val mrOfflineUserCandidateSparseAggregatesFeatureBuilder =
|
|
MrOfflineUserCandidateSparseAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats()
|
|
|
|
override val mrOfflineUserAggregatesFeatureBuilder =
|
|
MrOfflineUserAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats()
|
|
|
|
override val mrOfflineUserCandidateAggregatesFeatureBuilder =
|
|
MrOfflineUserCandidateAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats()
|
|
|
|
override val tweetAnnotationsFeatureBuilder =
|
|
TweetAnnotationsFeatureBuilder(stratoClient).withStats()
|
|
|
|
override val targetUserMediaRepresentationFeatureBuilder =
|
|
UserMediaRepresentationFeatureBuilder(userMediaRepresentationStore).withStats()
|
|
|
|
override val targetLevelFeatureBuilder =
|
|
TargetLevelFeatureBuilder(featureStoreUtil, targetLevelFeaturesConfig).withStats()
|
|
|
|
override val candidateLevelFeatureBuilder =
|
|
CandidateLevelFeatureBuilder(featureStoreUtil).withStats()
|
|
|
|
override lazy val targetFeatureHydrator = RelevanceTargetFeatureHydrator(
|
|
targetUserFullRealGraphFeatureBuilder,
|
|
postProcessingFeatureBuilder,
|
|
targetUserMediaRepresentationFeatureBuilder,
|
|
targetLevelFeatureBuilder
|
|
)
|
|
|
|
override lazy val featureHydrator =
|
|
FeatureHydrator(targetFeatureHydrator, candidateFeatureHydrator)
|
|
|
|
val pushServiceLightRankerConfig: LightRankerConfig = new LightRankerConfig(
|
|
pushserviceThriftClientId,
|
|
serviceIdentifier,
|
|
statsReceiver.scope("lightRanker"),
|
|
deepbirdv2PredictionServiceDest,
|
|
"DeepbirdV2PredictionService"
|
|
)
|
|
val lightRanker: MagicRecsServeDataRecordLightRanker =
|
|
pushServiceLightRankerConfig.lightRanker
|
|
|
|
override val tweetImpressionStore: ReadableStore[Long, Seq[Long]] = {
|
|
val name = "htl_impression_store"
|
|
val store = buildStore(
|
|
HtlTweetImpressionStore.createStoreWithTweetIds(
|
|
requestTimeout = 6.seconds,
|
|
label = "htl_tweet_impressions",
|
|
serviceIdentifier = serviceIdentifier,
|
|
statsReceiver = statsReceiver
|
|
),
|
|
name
|
|
)
|
|
val numTweetsReturned =
|
|
statsReceiver.scope(name).stat("num_tweets_returned_per_user")
|
|
new TransformedReadableStore(store)((userId: Long, tweetIds: Seq[Long]) => {
|
|
numTweetsReturned.add(tweetIds.size)
|
|
Future.value(Some(tweetIds))
|
|
})
|
|
}
|
|
|
|
val ruxTweetImpressionsStore = new TweetImpressionsStore(stratoClient)
|
|
|
|
override val strongTiesStore: ReadableStore[Long, STPResult] = {
|
|
implicit val valueCodec = new BinaryScalaCodec(STPResult)
|
|
val strongTieScoringDataset: Dataset[Long, STPResult] =
|
|
Dataset("", "frigate_stp", "stp_result_rerank", Athena)
|
|
buildManhattanStore(strongTieScoringDataset)
|
|
}
|
|
|
|
override lazy val earlybirdFeatureStore = ObservedReadableStore(
|
|
EarlybirdFeatureStore(
|
|
clientId = pushserviceThriftClientId.name,
|
|
earlybirdSearchStore = earlybirdSearchStore
|
|
)
|
|
)(statsReceiver.scope("EarlybirdFeatureStore"))
|
|
|
|
override lazy val earlybirdFeatureBuilder = EarlybirdFeatureBuilder(earlybirdFeatureStore)
|
|
|
|
override lazy val earlybirdSearchStore = {
|
|
val earlybirdClientName: String = "earlybird"
|
|
val earlybirdSearchStoreName: String = "EarlybirdSearchStore"
|
|
|
|
val earlybirdClient = new EarlybirdService.FinagledClient(
|
|
readOnlyThriftService(
|
|
earlybirdClientName,
|
|
earlybirdSearchDest,
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
tries = 1,
|
|
requestTimeout = 3.seconds,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
),
|
|
clientParam = RichClientParam(protocolFactory = new TCompactProtocol.Factory)
|
|
)
|
|
|
|
ObservedReadableStore(
|
|
EarlybirdSearchStore(earlybirdClient)(statsReceiver.scope(earlybirdSearchStoreName))
|
|
)(statsReceiver.scope(earlybirdSearchStoreName))
|
|
}
|
|
|
|
override lazy val earlybirdCandidateSource: EarlybirdCandidateSource = EarlybirdCandidateSource(
|
|
clientId = pushserviceThriftClientId.name,
|
|
earlybirdSearchStore = earlybirdSearchStore
|
|
)
|
|
|
|
override val realGraphScoresTop500InStore: RealGraphScoresTop500InStore = {
|
|
val stratoRealGraphInStore =
|
|
StratoFetchableStore
|
|
.withUnitView[Long, CandidateSeq](
|
|
stratoClient,
|
|
"frigate/magicrecs/fanoutCoi500pRealGraphV2")
|
|
|
|
RealGraphScoresTop500InStore(
|
|
ObservedMemcachedReadableStore.fromCacheClient(
|
|
backingStore = stratoRealGraphInStore,
|
|
cacheClient = entityGraphCacheClient,
|
|
ttl = 24.hours
|
|
)(
|
|
valueInjection = BinaryScalaCodec(CandidateSeq),
|
|
statsReceiver = statsReceiver.scope("CachedRealGraphScoresTop500InStore"),
|
|
keyToString = { k: Long => s"500p_test/$k" }
|
|
)
|
|
)
|
|
}
|
|
|
|
override val tweetEntityGraphStore = {
|
|
val tweetEntityGraphClient = new UserTweetEntityGraph.FinagledClient(
|
|
Finagle.readOnlyThriftService(
|
|
"user_tweet_entity_graph",
|
|
userTweetEntityGraphDest,
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
requestTimeout = 5.seconds,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
)
|
|
)
|
|
ObservedReadableStore(
|
|
RecommendedTweetEntitiesStore(
|
|
tweetEntityGraphClient,
|
|
statsReceiver.scope("RecommendedTweetEntitiesStore")
|
|
)
|
|
)(statsReceiver.scope("RecommendedTweetEntitiesStore"))
|
|
}
|
|
|
|
override val userUserGraphStore = {
|
|
val userUserGraphClient = new UserUserGraph.FinagledClient(
|
|
Finagle.readOnlyThriftService(
|
|
"user_user_graph",
|
|
userUserGraphDest,
|
|
statsReceiver,
|
|
pushserviceThriftClientId,
|
|
requestTimeout = 5.seconds,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
),
|
|
clientParam = RichClientParam(serviceName = "user_user_graph")
|
|
)
|
|
ObservedReadableStore(
|
|
UserUserGraphStore(userUserGraphClient, statsReceiver.scope("UserUserGraphStore"))
|
|
)(statsReceiver.scope("UserUserGraphStore"))
|
|
}
|
|
|
|
override val ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[
|
|
CaretFeedbackDetails
|
|
]] = {
|
|
val client = ManhattanKVClient(
|
|
"pushservice_ntab_caret_feedback_omega",
|
|
Omega.wilyName,
|
|
manhattanClientMtlsParams
|
|
)
|
|
val endpoint = ManhattanKVEndpointBuilder(client)
|
|
.defaultGuarantee(Guarantee.SoftDcReadMyWrites)
|
|
.defaultMaxTimeout(3.seconds)
|
|
.maxRetryCount(2)
|
|
.statsReceiver(statsReceiver)
|
|
.build()
|
|
|
|
val feedbackSignalManhattanClient =
|
|
FeedbackSignalManhattanClient(endpoint, statsReceiver.scope("FeedbackSignalManhattanClient"))
|
|
NtabCaretFeedbackStore(feedbackSignalManhattanClient)
|
|
}
|
|
|
|
override val genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[
|
|
FeedbackPromptValue
|
|
]] = {
|
|
FeedbackStore(
|
|
GenericFeedbackStoreBuilder.build(
|
|
manhattanKVClientAppId = "frigate_pushservice_ntabfeedback_prompt",
|
|
environment = NotifEnvironment.apply(serviceIdentifier.environment),
|
|
svcIdentifier = serviceIdentifier,
|
|
statsReceiver = statsReceiver
|
|
))
|
|
}
|
|
|
|
override val genericNotificationFeedbackStore: GenericFeedbackStore = {
|
|
|
|
GenericFeedbackStoreBuilder.build(
|
|
manhattanKVClientAppId = "frigate_pushservice_ntabfeedback_prompt",
|
|
environment = NotifEnvironment.apply(serviceIdentifier.environment),
|
|
svcIdentifier = serviceIdentifier,
|
|
statsReceiver = statsReceiver
|
|
)
|
|
}
|
|
|
|
override val earlybirdSearchDest = "/s/earlybird-root-superroot/root-superroot"
|
|
|
|
// low latency as compared to default `semanticCoreMetadataClient`
|
|
private val lowLatencySemanticCoreMetadataClient: MetadataService.MethodPerEndpoint =
|
|
new MetadataService.FinagledClient(
|
|
Finagle.readOnlyThriftService(
|
|
name = "semantic_core_metadata_service",
|
|
dest = "/s/escherbird/metadataservice",
|
|
statsReceiver = statsReceiver,
|
|
thriftClientId = pushserviceThriftClientId,
|
|
tries = 2, // total number of tries. number of retries = tries - 1
|
|
requestTimeout = 2.seconds,
|
|
mTLSServiceIdentifier = Some(serviceIdentifier)
|
|
)
|
|
)
|
|
|
|
private val semanticCoreMetadataStitchClient = new MetadataStitchClient(
|
|
lowLatencySemanticCoreMetadataClient
|
|
)
|
|
|
|
override val semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] = {
|
|
val name = "semantic_core_megadata_store_cached"
|
|
val store = MetaDataReadableStore.getMegadataReadableStore(
|
|
metadataStitchClient = semanticCoreMetadataStitchClient,
|
|
typedMetadataDomains = Some(Set(Domains.EventsEntityService))
|
|
)
|
|
ObservedCachedReadableStore
|
|
.from(
|
|
store = ObservedReadableStore(store)(
|
|
statsReceiver
|
|
.scope("store")
|
|
.scope("semantic_core_megadata_store")
|
|
),
|
|
ttl = 1.hour,
|
|
maxKeys = 1000,
|
|
cacheName = "semantic_core_megadata_cache",
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope("store", name))
|
|
}
|
|
|
|
override val basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate] = {
|
|
StratoFetchableStore.withUnitView[QualifiedId, BasketballGameLiveUpdate](
|
|
stratoClient,
|
|
"semanticCore/basketballGameScore.Entity")
|
|
}
|
|
|
|
override val baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate] = {
|
|
StratoFetchableStore.withUnitView[QualifiedId, BaseballGameLiveUpdate](
|
|
stratoClient,
|
|
"semanticCore/baseballGameScore.Entity")
|
|
}
|
|
|
|
override val cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate] = {
|
|
StratoFetchableStore.withUnitView[QualifiedId, CricketMatchLiveUpdate](
|
|
stratoClient,
|
|
"semanticCore/cricketMatchScore.Entity")
|
|
}
|
|
|
|
override val soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate] = {
|
|
ObservedCachedReadableStore
|
|
.from(
|
|
store = StratoFetchableStore.withUnitView[QualifiedId, SoccerMatchLiveUpdate](
|
|
stratoClient,
|
|
"semanticCore/soccerMatchScore.Entity"),
|
|
ttl = 10.seconds,
|
|
maxKeys = 100,
|
|
cacheName = "SoccerMatchCachedStore",
|
|
windowSize = 100L
|
|
)(statsReceiver.scope("SoccerMatchCachedStore"))
|
|
|
|
}
|
|
|
|
override val nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate] = {
|
|
ObservedCachedReadableStore
|
|
.from(
|
|
store = StratoFetchableStore.withUnitView[QualifiedId, NflFootballGameLiveUpdate](
|
|
stratoClient,
|
|
"semanticCore/nflFootballGameScore.Entity"),
|
|
ttl = 10.seconds,
|
|
maxKeys = 100,
|
|
cacheName = "NFLMatchCachedStore",
|
|
windowSize = 100L
|
|
)(statsReceiver.scope("NFLMatchCachedStore"))
|
|
|
|
}
|
|
|
|
override val userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] = {
|
|
val userHealthSignalFetcher =
|
|
stratoClient.fetcher[Long, Seq[UserHealthSignal], UserHealthSignalResponse](
|
|
"hss/user_signals/api/healthSignals.User"
|
|
)
|
|
|
|
val store = buildStore(
|
|
StratoFetchableStore.withView[Long, Seq[UserHealthSignal], UserHealthSignalResponse](
|
|
userHealthSignalFetcher,
|
|
Seq(
|
|
AgathaRecentAbuseStrikeDouble,
|
|
AgathaCalibratedNsfwDouble,
|
|
AgathaCseDouble,
|
|
NsfwTextUserScoreDouble,
|
|
NsfwConsumerScoreDouble)),
|
|
"UserHealthSignalFetcher"
|
|
)
|
|
if (!inMemCacheOff) {
|
|
ObservedCachedReadableStore
|
|
.from(
|
|
store = ObservedReadableStore(store)(
|
|
statsReceiver.scope("store").scope("user_health_model_score_store")),
|
|
ttl = 12.hours,
|
|
maxKeys = 16777215,
|
|
cacheName = "user_health_model_score_store_cache",
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope("store", "user_health_model_score_store_cached"))
|
|
} else {
|
|
store
|
|
}
|
|
}
|
|
|
|
override val tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] = {
|
|
val tweetHealthScoreFetcher =
|
|
stratoClient.fetcher[TweetScoringRequest, Unit, TweetScoringResponse](
|
|
"abuse/detection/tweetHealthModelScore"
|
|
)
|
|
|
|
val store = buildStore(
|
|
StratoFetchableStore.withUnitView(tweetHealthScoreFetcher),
|
|
"TweetHealthScoreFetcher"
|
|
)
|
|
|
|
ObservedCachedReadableStore
|
|
.from(
|
|
store = ObservedReadableStore(store)(
|
|
statsReceiver.scope("store").scope("tweet_health_model_score_store")),
|
|
ttl = 30.minutes,
|
|
maxKeys = 1000,
|
|
cacheName = "tweet_health_model_score_store_cache",
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope("store", "tweet_health_model_score_store_cached"))
|
|
}
|
|
|
|
override val appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] = {
|
|
val store = StratoFetchableStore
|
|
.withUnitView[(Long, (String, String)), AppPermission](
|
|
stratoClient,
|
|
"clients/permissionsState")
|
|
ObservedCachedReadableStore.from[(Long, (String, String)), AppPermission](
|
|
buildStore(store, "mr_app_permission_store"),
|
|
ttl = 30.minutes,
|
|
maxKeys = 1000,
|
|
cacheName = "mr_app_permission_store_cache",
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope("mr_app_permission_store_cached"))
|
|
}
|
|
|
|
def pushSendEventStreamName: String
|
|
|
|
override val pushSendEventBusPublisher = EventBusPublisherBuilder()
|
|
.clientId("frigate_pushservice")
|
|
.streamName(pushSendEventStreamName)
|
|
.thriftStruct(NotificationScribe)
|
|
.statsReceiver(statsReceiver.scope("push_send_eventbus"))
|
|
.build()
|
|
|
|
override lazy val candidateFeatureHydrator: CandidateFeatureHydrator =
|
|
CandidateFeatureHydrator(
|
|
socialContextActionsFeatureBuilder = Some(socialContextActionsFeatureBuilder),
|
|
tweetSocialProofFeatureBuilder = Some(tweetSocialProofFeatureBuilder),
|
|
earlybirdFeatureBuilder = Some(earlybirdFeatureBuilder),
|
|
tweetContentFeatureBuilder = Some(tweetContentFeatureBuilder),
|
|
tweetAuthorRecentRealGraphFeatureBuilder = Some(tweetAuthorRecentRealGraphFeatureBuilder),
|
|
socialContextRecentRealGraphFeatureBuilder = Some(socialContextRecentRealGraphFeatureBuilder),
|
|
tweetAnnotationsFeatureBuilder = Some(tweetAnnotationsFeatureBuilder),
|
|
mrOfflineUserCandidateSparseAggregatesFeatureBuilder =
|
|
Some(mrOfflineUserCandidateSparseAggregatesFeatureBuilder),
|
|
candidateLevelFeatureBuilder = Some(candidateLevelFeatureBuilder)
|
|
)(statsReceiver.scope("push_feature_hydrator"))
|
|
|
|
private val candidateCopyCross =
|
|
new CandidateCopyExpansion(statsReceiver.scope("refresh_handler/cross"))
|
|
|
|
override lazy val candidateHydrator: PushCandidateHydrator =
|
|
PushCandidateHydrator(
|
|
this.socialGraphServiceProcessStore,
|
|
safeUserStore,
|
|
listAPIStore,
|
|
candidateCopyCross)(
|
|
statsReceiver.scope("push_candidate_hydrator"),
|
|
weightedOpenOrNtabClickModelScorer)
|
|
|
|
override lazy val sendHandlerCandidateHydrator: SendHandlerPushCandidateHydrator =
|
|
SendHandlerPushCandidateHydrator(
|
|
lexServiceStore,
|
|
fanoutMetadataStore,
|
|
semanticCoreMegadataStore,
|
|
safeUserStore,
|
|
simClusterToEntityStore,
|
|
audioSpaceStore,
|
|
interestsWithLookupContextStore,
|
|
uttEntityHydrationStore,
|
|
superFollowCreatorTweetCountStore
|
|
)(
|
|
statsReceiver.scope("push_candidate_hydrator"),
|
|
weightedOpenOrNtabClickModelScorer
|
|
)
|
|
|
|
def mrRequestScriberNode: String
|
|
def loggedOutMrRequestScriberNode: String
|
|
|
|
override lazy val configParamsBuilder: ConfigParamsBuilder = ConfigParamsBuilder(
|
|
config = overridesConfig,
|
|
featureContextBuilder = FeatureContextBuilder(featureSwitches),
|
|
statsReceiver = statsReceiver
|
|
)
|
|
|
|
def buildStore[K, V](store: ReadableStore[K, V], name: String): ReadableStore[K, V] = {
|
|
ObservedReadableStore(store)(statsReceiver.scope("store").scope(name))
|
|
}
|
|
|
|
def buildManhattanStore[K, V](dataset: Dataset[K, V]): ReadableStore[K, V] = {
|
|
val manhattanKVClientParams = ManhattanKVClientMtlsParams(
|
|
serviceIdentifier = serviceIdentifier,
|
|
opportunisticTls = OpportunisticTls.Required
|
|
)
|
|
ManhattanStore
|
|
.fromDatasetWithMtls[K, V](
|
|
dataset,
|
|
mtlsParams = manhattanKVClientParams,
|
|
statsReceiver = statsReceiver.scope(dataset.datasetName))
|
|
}
|
|
|
|
def buildCachedTweetyPieStore(
|
|
getTweetOptions: Option[GetTweetOptions],
|
|
keyPrefix: String
|
|
): ReadableStore[Long, TweetyPieResult] = {
|
|
def discardAdditionalMediaInfo(tweetypieResult: TweetyPieResult) = {
|
|
val updatedMedia = tweetypieResult.tweet.media.map { mediaSeq =>
|
|
mediaSeq.map { media => media.copy(additionalMetadata = None, sizes = Nil.toSet) }
|
|
}
|
|
val updatedTweet = tweetypieResult.tweet.copy(media = updatedMedia)
|
|
tweetypieResult.copy(tweet = updatedTweet)
|
|
}
|
|
|
|
val tweetypieStoreWithoutAdditionalMediaInfo = TweetyPieStore(
|
|
tweetyPieClient,
|
|
getTweetOptions,
|
|
transformTweetypieResult = discardAdditionalMediaInfo
|
|
)(statsReceiver.scope("tweetypie_without_additional_media_info"))
|
|
|
|
ObservedMemcachedReadableStore.fromCacheClient(
|
|
backingStore = tweetypieStoreWithoutAdditionalMediaInfo,
|
|
cacheClient = pushServiceCoreSvcsCacheClient,
|
|
ttl = 12.hours
|
|
)(
|
|
valueInjection = TweetyPieResultInjection,
|
|
statsReceiver = statsReceiver.scope("TweetyPieStore"),
|
|
keyToString = { k: Long => s"$keyPrefix/$k" }
|
|
)
|
|
}
|
|
|
|
override def init(): Future[Unit] =
|
|
ClientRegistry.expAllRegisteredClientsResolved().map { clients =>
|
|
log.info("Done resolving clients: " + clients.mkString("[", ", ", "]"))
|
|
}
|
|
|
|
val InlineActionsMhColumn =
|
|
"frigate/magicrecs/inlineActionsMh"
|
|
|
|
override val inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]] =
|
|
StratoScannableStore
|
|
.withUnitView[(Long, Slice[Long]), (Long, Long), String](stratoClient, InlineActionsMhColumn)
|
|
.composeKeyMapping[Long] { userId =>
|
|
(userId, Slice[Long](from = None, to = None, limit = None))
|
|
}.mapValues { response =>
|
|
response.map {
|
|
case (key, value) => (key._2, value)
|
|
}
|
|
}
|
|
|
|
override val tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets] = {
|
|
StratoFetchableStore
|
|
.withUnitView[TripDomain, TripTweets](
|
|
new TripTweetsAirflowProdClientColumn(stratoClient).fetcher)
|
|
}
|
|
|
|
override val softUserFollowingStore: ReadableStore[User, Seq[Long]] = new SoftUserFollowingStore(
|
|
stratoClient)
|
|
|
|
override val superFollowEligibilityUserStore: ReadableStore[Long, Boolean] = {
|
|
StratoFetchableStore.withUnitView[Long, Boolean](
|
|
stratoClient,
|
|
"audiencerewards/audienceRewardsService/getSuperFollowEligibility.User")
|
|
}
|
|
|
|
override val superFollowCreatorTweetCountStore: ReadableStore[UserId, Int] = {
|
|
ObservedCachedReadableStore
|
|
.from(
|
|
store = StratoFetchableStore
|
|
.withUnitView[UserId, Int](new CreatorSubscriptionNumTweetsColumn(stratoClient).fetcher),
|
|
ttl = 5.minutes,
|
|
maxKeys = 1000,
|
|
cacheName = "SuperFollowCreatorTweetCountStore",
|
|
windowSize = 10000L
|
|
)(statsReceiver.scope("SuperFollowCreatorTweetCountStore"))
|
|
|
|
}
|
|
|
|
override val hasSuperFollowingRelationshipStore: ReadableStore[
|
|
HasSuperFollowingRelationshipRequest,
|
|
Boolean
|
|
] = {
|
|
StratoFetchableStore.withUnitView[HasSuperFollowingRelationshipRequest, Boolean](
|
|
stratoClient,
|
|
"audiencerewards/superFollows/hasSuperFollowingRelationshipV2")
|
|
}
|
|
|
|
override val superFollowApplicationStatusStore: ReadableStore[
|
|
(Long, SellerTrack),
|
|
SellerApplicationState
|
|
] = {
|
|
StratoFetchableStore.withUnitView[(Long, SellerTrack), SellerApplicationState](
|
|
stratoClient,
|
|
"periscope/eligibility/applicationStatus")
|
|
}
|
|
|
|
def historyStoreMemcacheDest: String
|
|
|
|
override lazy val recentHistoryCacheClient = {
|
|
RecentHistoryCacheClient.build(historyStoreMemcacheDest, serviceIdentifier, statsReceiver)
|
|
}
|
|
|
|
override val openAppUserStore: ReadableStore[Long, Boolean] = {
|
|
buildStore(OpenAppUserStore(stratoClient), "OpenAppUserStore")
|
|
}
|
|
}
|