Mirrors
/
the-algorithm
mirror of https://github.com/twitter/the-algorithm.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[opensource] Update home mixer with latest changes

This commit is contained in:
twitter-team 2023-07-07 14:01:54 -07:00
parent fb54d8b549
commit 72eda9a24f
393 changed files with 9334 additions and 3850 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
home-mixer/README.md
View File

@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
- Fetch Tweet Candidates
- ScoredTweetsInNetworkCandidatePipelineConfig
- ScoredTweetsCrMixerCandidatePipelineConfig
- ScoredTweetsTweetMixerCandidatePipelineConfig
- ScoredTweetsUtegCandidatePipelineConfig
- ScoredTweetsFrsCandidatePipelineConfig
- Feature Hydration and Scoring
@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer.
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
- ListTweetsAdsCandidatePipelineConfig (fetch ads)

5
home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel
View File

@ -21,6 +21,7 @@ scala_library(
"finatra/inject/inject-utils/src/main/scala",
"home-mixer/server/src/main/resources",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
@ -31,6 +32,10 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"stringcenter/client",
"stringcenter/client/src/main/java",
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala
View File

@ -2,7 +2,7 @@ package com.twitter.home_mixer
import com.twitter.finatra.http.routing.HttpWarmup
import com.twitter.finatra.httpclient.RequestBuilder._
import com.twitter.inject.Logging
import com.twitter.util.logging.Logging
import com.twitter.inject.utils.Handler
import com.twitter.util.Try
import javax.inject.Inject

34
home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala
View File

@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer
import com.twitter.finatra.thrift.filters._
import com.twitter.finatra.thrift.routing.ThriftRouter
import com.twitter.home_mixer.controller.HomeThriftController
import com.twitter.home_mixer.federated.HomeMixerColumn
import com.twitter.home_mixer.module._
import com.twitter.home_mixer.param.GlobalParamConfigModule
import com.twitter.home_mixer.product.HomeMixerProductModule
import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
import com.twitter.product_mixer.component_library.module.CrMixerClientModule
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
import com.twitter.product_mixer.component_library.module.EarlybirdModule
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
import com.twitter.product_mixer.core.controllers.ProductMixerController
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
import com.twitter.product_mixer.core.module.ProductMixerModule
import com.twitter.product_mixer.core.module.StratoClientModule
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.fed.server.StratoFedServer
object HomeMixerServerMain extends HomeMixerServer
class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
class HomeMixerServer
extends StratoFedServer
with ThriftServer
with Mtls
with HttpServer
with HttpMtls {
override val name = "home-mixer-server"
override val modules: Seq[Module] = Seq(
AccountRecommendationsMixerModule,
AdvertiserBrandSafetySettingsStoreModule,
BlenderClientModule,
ClientSentImpressionsPublisherModule,
ConversationServiceModule,
CrMixerClientModule,
EarlybirdModule,
ExploreRankerClientModule,
FeedbackHistoryClientModule,
GizmoduckClientModule,
GlobalParamConfigModule,
HomeAdsCandidateSourceModule,
HomeMixerFlagsModule,
HomeMixerProductModule,
HomeMixerResourcesModule,
HomeNaviModelClientModule,
ImpressionBloomFilterModule,
InjectionHistoryClientModule,
FeedbackHistoryClientModule,
ManhattanClientsModule,
ManhattanFeatureRepositoryModule,
ManhattanTweetImpressionStoreModule,
MemcachedFeatureRepositoryModule,
NaviModelClientModule,
OnboardingTaskServiceModule,
OptimizedStratoClientModule,
PeopleDiscoveryServiceModule,
@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
SimClustersRecentEngagementsClientModule,
SocialGraphServiceModule,
StaleTweetsCacheModule,
StratoClientModule,
ThriftFeatureRepositoryModule,
TimelineMixerClientModule,
TimelineRankerClientModule,
TimelineScorerClientModule,
TimelineServiceClientModule,
TimelinesPersistenceStoreClientModule,
TopicSocialProofClientModule,
TweetImpressionStoreModule,
TweetyPieClientModule,
TweetMixerClientModule,
TweetypieClientModule,
TweetypieStaticEntitiesCacheClientModule,
UserMetadataStoreModule,
UserSessionStoreModule,
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
new ProductScopeStringCenterModule()
)
def configureThrift(router: ThriftRouter): Unit = {
override def configureThrift(router: ThriftRouter): Unit = {
router
.filter[LoggingMDCFilter]
.filter[TraceIdMDCFilter]
@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
this.injector,
st.HomeMixer.ExecutePipeline))
override val dest: String = "/s/home-mixer/home-mixer:strato"
override val columns: Seq[Class[_ <: StratoFed.Column]] =
Seq(classOf[HomeMixerColumn])
override protected def warmup(): Unit = {
handle[HomeMixerThriftServerWarmupHandler]()
handle[HomeMixerHttpServerWarmupHandler]()

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala
View File

@ -3,7 +3,7 @@ package com.twitter.home_mixer
import com.twitter.finagle.thrift.ClientId
import com.twitter.finatra.thrift.routing.ThriftWarmup
import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.inject.Logging
import com.twitter.util.logging.Logging
import com.twitter.inject.utils.Handler
import com.twitter.product_mixer.core.{thriftscala => pt}
import com.twitter.scrooge.Request

10
home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel
View File

@ -5,19 +5,13 @@ scala_library(
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
@ -25,10 +19,6 @@ scala_library(
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
],
exports = [
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",

25
home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala
View File

@ -1,18 +1,23 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
import com.twitter.product_mixer.component_library.filter.FeatureFilter
import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
override val gates: Seq[BaseGate[Query]],
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
extends DependentCandidatePipelineConfig[
@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
val tweetsWithConversationMetadata = candidates.map { candidate =>
TweetWithConversationMetadata(
tweetId = candidate.candidateIdLong,
userId = None,
sourceTweetId = None,
sourceUserId = None,
inReplyToTweetId = None,
userId = candidate.features.getOrElse(AuthorIdFeature, None),
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
conversationId = None,
ancestors = Seq.empty
)
@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
override val preFilterFeatureHydrationPhase1: Seq[
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
] = Seq(
tweetypieFeatureHydrator,
InNetworkFeatureHydrator,
)
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
RetweetDeduplicationFilter,
@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
FilterIdentifier(QuotedTweetDroppedFilterId),
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
),
invalidSubscriptionTweetFilter,
InvalidConversationModuleFilter
)

8
home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala
View File

@ -1,9 +1,9 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
@ -15,7 +15,7 @@ import javax.inject.Singleton
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
namesFeatureHydrator: NamesFeatureHydrator) {
def build(
@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
new ConversationServiceCandidatePipelineConfig(
conversationServiceCandidateSource,
tweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator,
namesFeatureHydrator,
invalidSubscriptionTweetFilter,
gates,
decorator
)

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala
View File

@ -1,7 +1,7 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
import com.twitter.home_mixer.service.HomeMixerAlertConfig

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel
View File

@ -13,8 +13,5 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
"snowflake/src/main/scala/com/twitter/snowflake/id",
"src/thrift/com/twitter/context:twitter-context-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"twitter-context/src/main/scala",
],
)

1
home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala
View File

@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService
import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.controllers.DebugTwitterContext
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
import com.twitter.product_mixer.core.service.urt.UrtService
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch

24
home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel Normal file
View File

@ -0,0 +1,24 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"stitch/stitch-repo/src/main/scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
"strato/src/main/scala/com/twitter/strato/callcontext",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
],
)

217
home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala Normal file
View File

@ -0,0 +1,217 @@
package com.twitter.home_mixer.federated
import com.twitter.gizmoduck.{thriftscala => gd}
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
import com.twitter.home_mixer.model.request.HomeMixerRequest
import com.twitter.home_mixer.{thriftscala => hm}
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
import com.twitter.product_mixer.core.{thriftscala => pm}
import com.twitter.stitch.Arrow
import com.twitter.stitch.Stitch
import com.twitter.strato.callcontext.CallContext
import com.twitter.strato.catalog.OpMetadata
import com.twitter.strato.config._
import com.twitter.strato.data._
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
import com.twitter.strato.graphql.timelines.{thriftscala => gql}
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.timelines.render.{thriftscala => tr}
import com.twitter.util.Try
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class HomeMixerColumn @Inject() (
homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn,
paramsBuilder: ParamsBuilder,
productPipelineRegistry: ProductPipelineRegistry)
extends StratoFed.Column(HomeMixerColumn.Path)
with StratoFed.Fetch.Arrow {
override val contactInfo: ContactInfo = ContactInfo(
contactEmail = "",
ldapGroup = "",
slackRoomId = ""
)
override val metadata: OpMetadata =
OpMetadata(
lifecycle = Some(Lifecycle.Production),
description =
Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
)
private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
ServiceIdentifierPattern(
role = "",
service = "",
env = "",
zone = Seq(""))
)
override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
override type Key = gql.TimelineKey
override type View = gql.HomeTimelineView
override type Value = tr.Timeline
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
private def createHomeMixerRequestArrow(
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn
): Arrow[(Key, View), hm.HomeMixerRequest] = {
val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
(gd.LookupContext(), Set(gd.QueryFields.Roles))
val populateUserRoles = Arrow
.flatMap[(Key, View), Option[Set[String]]] { _ =>
Stitch.collect {
CallContext.twitterUserId.map { userId =>
compositeOnUserClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch(userId, gizmoduckView).map(_.v)
.map {
_.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
}
}
}
}
val populateIpAddress = Arrow
.flatMap[(Key, View), Option[String]](_ =>
auditIpClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch((), ()).map(_.v))
Arrow.join(
populateUserRoles,
populateIpAddress
)
}
Arrow.zipWithArg(populateUserRolesAndIp).map {
case ((key, view), (roles, ipAddress)) =>
val deviceContextOpt = Some(
hm.DeviceContext(
isPolling = CallContext.isPolling,
requestContext = view.requestContext,
latestControlAvailable = view.latestControlAvailable,
autoplayEnabled = view.autoplayEnabled
))
val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
val (product, productContext) = key match {
case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
(
hm.Product.ForYou,
hm.ProductContext.ForYou(
hm.ForYou(
deviceContextOpt,
seenTweetIds,
view.dspClientContext,
view.pushToHomeTweetId
)
))
case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
(
hm.Product.Following,
hm.ProductContext.Following(
hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
(
hm.Product.Subscribed,
hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
}
val clientContext = pm.ClientContext(
userId = CallContext.twitterUserId,
guestId = CallContext.guestId,
guestIdAds = CallContext.guestIdAds,
guestIdMarketing = CallContext.guestIdMarketing,
appId = CallContext.clientApplicationId,
ipAddress = ipAddress,
userAgent = CallContext.userAgent,
countryCode = CallContext.requestCountryCode,
languageCode = CallContext.requestLanguageCode,
isTwoffice = CallContext.isInternalOrTwoffice,
userRoles = roles,
deviceId = CallContext.deviceId,
mobileDeviceId = CallContext.mobileDeviceId,
mobileDeviceAdId = CallContext.adId,
limitAdTracking = CallContext.limitAdTracking
)
hm.HomeMixerRequest(
clientContext = clientContext,
product = product,
productContext = Some(productContext),
maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
cursor = view.cursor.filter(_.nonEmpty)
)
}
}
override val fetch: Arrow[(Key, View), Result[Value]] = {
val transformThriftIntoPipelineRequest: Arrow[
(Key, View),
ProductPipelineRequest[HomeMixerRequest]
] = {
Arrow
.identity[(Key, View)]
.andThen {
createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
}
.map {
case thriftRequest =>
val request = homeMixerRequestUnmarshaller(thriftRequest)
val params = paramsBuilder.build(
clientContext = request.clientContext,
product = request.product,
featureOverrides =
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
)
ProductPipelineRequest(request, params)
}
}
val underlyingProduct: Arrow[
ProductPipelineRequest[HomeMixerRequest],
ProductPipelineResult[tr.TimelineResponse]
] = Arrow
.identity[ProductPipelineRequest[HomeMixerRequest]]
.map { pipelineRequest =>
val pipelineArrow = productPipelineRegistry
.getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
pipelineRequest.request.product)
.arrow
(pipelineArrow, pipelineRequest)
}.applyArrow
transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
_.result match {
case Some(result) => found(result.timeline)
case _ => missing
}
}
}
}
object HomeMixerColumn {
val Path = "home-mixer/homeMixer.Timeline"
private val FetchCallstack = s"$Path:fetch"
private val MaxCount: Option[Int] = Some(100)
}

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel
View File

@ -10,11 +10,8 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"stitch/stitch-timelineservice/src/main/scala",
"strato/config/columns/recommendations/similarity:similarity-strato-client",
"strato/src/main/scala/com/twitter/strato/client",
],
exports = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",

6
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel
View File

@ -6,13 +6,11 @@ scala_library(
dependencies = [
"finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
@ -25,8 +23,6 @@ scala_library(
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"stringcenter/client",
"stringcenter/client/src/main/java",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
],
)

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala
View File

@ -1,6 +1,8 @@
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator

5
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel
View File

@ -8,7 +8,9 @@ scala_library(
"finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
@ -18,6 +20,7 @@ scala_library(
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
],
)

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeAdsClientEventDetailsBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.finagle.tracing.Trace
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeClientEventDetailsBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.bijection.Base64String
import com.twitter.bijection.scrooge.BinaryScalaCodec

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTimelinesScoreInfoBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient

163
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetTypePredicates.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala
View File

@ -1,8 +1,10 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
import com.twitter.tweetypie.{thriftscala => tpt}
@ -25,74 +27,76 @@ object HomeTweetTypePredicates {
(
"has_exclusive_conversation_author_id",
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)),
("is_eligible_for_connect_boost", _ => false),
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("get_initial", _.getOrElse(GetInitialFeature, false)),
("get_newer", _.getOrElse(GetNewerFeature, false)),
("get_middle", _.getOrElse(GetMiddleFeature, false)),
("get_older", _.getOrElse(GetOlderFeature, false)),
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
("polling", _.getOrElse(PollingFeature, false)),
("tls_size_20_plus", _ => false),
("near_empty", _ => false),
("ranked_request", _ => false),
("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
(
"less_than_10_mins_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
(
"is_signup_request",
candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("empty_request", _ => false),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
"conversation_module_has_2_displayed_tweets",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)),
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
(
"served_size_between_50_and_100",
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
(
"account_age_less_than_30_minutes",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
(
"account_age_less_than_1_day",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
(
"account_age_less_than_7_days",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
("root_user_is_in_first_degree", _ => false),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
(
"part_of_utt",
_.getOrElse(EarlybirdFeature, None)
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
(
"has_home_latest_request_past_week",
_.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("viewer_is_employee", _ => false),
("viewer_is_timelines_employee", _ => false),
("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
(
"has_ancestor_authored_by_viewer",
candidate =>
@ -100,11 +104,6 @@ object HomeTweetTypePredicates {
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
(
"deep_reply",
candidate =>
@ -119,23 +118,22 @@ object HomeTweetTypePredicates {
"tweet_age_less_than_15_seconds",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 15.seconds)),
("is_followed_topic_tweet", _ => false),
("is_recommended_topic_tweet", _ => false),
("is_topic_tweet", _ => false),
("preferred_language_matches_tweet_language", _ => false),
(
"less_than_1_hour_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)),
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
(
"device_language_matches_tweet_language",
candidate =>
candidate.getOrElse(TweetLanguageFeature, None) ==
candidate.getOrElse(DeviceLanguageFeature, None)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
("in_network", _.getOrElse(FromInNetworkSourceFeature, true)),
("viewer_follows_original_author", _ => false),
("has_account_follow_prompt", _ => false),
("has_relevance_prompt", _ => false),
("has_topic_annotation_haug_prompt", _ => false),
("has_topic_annotation_random_precision_prompt", _ => false),
("has_topic_annotation_prompt", _ => false),
("in_network", _.getOrElse(InNetworkFeature, true)),
(
"has_political_annotation",
_.getOrElse(EarlybirdFeature, None).exists(
@ -153,9 +151,14 @@ object HomeTweetTypePredicates {
_.getOrElse(EarlybirdFeature, None)
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
("is_viewer_not_invited_to_reply", _ => false),
("is_viewer_invited_to_reply", _ => false),
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
(
"is_followed_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == BasicTopicContextFunctionalityType)),
(
"is_recommended_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == RecommendationTopicContextFunctionalityType)),
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
(
@ -164,8 +167,6 @@ object HomeTweetTypePredicates {
(
"has_gte_100k_favs",
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
("above_neighbor_is_topic_tweet", _ => false),
("is_topic_tweet_with_neighbor_below", _ => false),
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
(
@ -187,6 +188,7 @@ object HomeTweetTypePredicates {
(
"has_toxicity_score_above_threshold",
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
(
"text_only",
candidate =>
@ -204,23 +206,50 @@ object HomeTweetTypePredicates {
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
("3_or_more_consecutive_not_in_network", _ => false),
("2_or_more_consecutive_not_in_network", _ => false),
("5_out_of_7_not_in_network", _ => false),
("7_out_of_7_not_in_network", _ => false),
("5_out_of_5_not_in_network", _ => false),
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
("has_liked_by_social_context", _ => false),
("has_followed_by_social_context", _ => false),
("has_topic_social_context", _ => false),
("timeline_entry_has_banner", _ => false),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
(
"conversation_module_has_2_displayed_tweets",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
("served_in_recap_tweet_candidate_module_injection", _ => false),
("served_in_threaded_conversation_module", _ => false)
"has_liked_by_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
.exists(candidateFeatures
.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
(
"has_followed_by_social_context",
_.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
(
"has_topic_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(TopicIdSocialContextFeature, None)
.isDefined &&
candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
(
"video_bt_10_60_sec",
_.getOrElse(VideoDurationMsFeature, None).exists(duration =>
duration > 10000 && duration <= 60000)),
("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
(
"tweet_age_lte_30_minutes",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 30.minutes)),
(
"tweet_age_lte_1_hour",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 1.hour)),
(
"tweet_age_lte_6_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 6.hours)),
(
"tweet_age_lte_12_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 12.hours)),
(
"tweet_age_gte_24_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow >= 24.hours)),
)
val PredicateMap = CandidatePredicates.toMap

4
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala
View File

@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelineservice.suggests.{thriftscala => st}
object ListClientEventDetailsBuilder
case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
override def apply(
@ -20,7 +20,7 @@ object ListClientEventDetailsBuilder
conversationDetails = None,
timelinesDetails = Some(
TimelinesDetails(
injectionType = Some(st.SuggestType.OrganicListTweet.name),
injectionType = Some(suggestType.name),
controllerData = None,
sourceData = None)),
articleDetails = None,

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/AuthorChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -9,7 +9,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Singleton

7
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel
View File

@ -4,9 +4,16 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
"src/thrift/com/twitter/timelines/service:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
],
)

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BlockUserChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/DontLikeFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -17,7 +17,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tls}
import javax.inject.Inject
import javax.inject.Singleton

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EngagerSocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ExtendedReplySocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

18
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala Normal file
View File

@ -0,0 +1,18 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
class FeedbackStrings @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
private val externalStringRegistry = externalStringRegistryProvider.get()
val seeLessOftenFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOften")
val seeLessOftenConfirmationFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
}

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FeedbackUtil.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FollowedBySocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeFeedbackActionInfoBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.request.FollowingProduct
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => t}
import com.twitter.timelines.util.FeedbackMetadataSerializer
import javax.inject.Inject
import javax.inject.Singleton

10
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetSocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
@ -14,10 +14,13 @@ import javax.inject.Singleton
@Singleton
case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder: LikedBySocialContextBuilder,
listsSocialContextBuilder: ListsSocialContextBuilder,
followedBySocialContextBuilder: FollowedBySocialContextBuilder,
topicSocialContextBuilder: TopicSocialContextBuilder,
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder,
popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
def apply(
@ -31,6 +34,9 @@ case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder(query, candidate, features)
.orElse(followedBySocialContextBuilder(query, candidate, features))
.orElse(topicSocialContextBuilder(query, candidate, features))
.orElse(popularVideoSocialContextBuilder(query, candidate, features))
.orElse(listsSocialContextBuilder(query, candidate, features))
.orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
case Some(_) =>
val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
// Only hydrate the social context into the root tweet in a conversation module

6
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala
View File

@ -5,7 +5,6 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
@ -32,12 +31,13 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
@Singleton
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
externalStringRegistry = externalStringRegistryProvider.get(),
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)

52
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala Normal file
View File

@ -0,0 +1,52 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => tl}
import com.twitter.timelines.util.FeedbackRequestSerializer
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.timelineservice.thriftscala.FeedbackType
object HomeWhoToSubscribeFeedbackActionInfoBuilder {
private val FeedbackMetadata = tl.FeedbackMetadata(
injectionType = Some(SuggestType.WhoToSubscribe),
engagementType = None,
entityIds = Seq.empty,
ttlMs = None
)
private val FeedbackRequest =
tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
private val EncodedFeedbackRequest =
FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
}
@Singleton
case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest =
Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)
override def apply(
query: PipelineQuery,
candidate: UserCandidate,
candidateFeatures: FeatureMap
): Option[FeedbackActionInfo] =
whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
}

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/LikedBySocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature

50
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala Normal file
View File

@ -0,0 +1,50 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* "Your Lists" will be rendered for the context and a url link for your lists.
*/
@Singleton
case class ListsSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
candidateFeatures.get(SuggestTypeFeature) match {
case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
Some(
GeneralContext(
contextType = ListGeneralContextType,
text = stringCenter.prepare(listString),
url = userName.map(name => ""),
contextImageUrls = None,
landingUrl = None
))
case _ => None
}
}
}

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/MuteUserChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotInterestedTopicFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotRelevantChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
@ -12,7 +12,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tlst}
import javax.inject.Inject
import javax.inject.Singleton

43
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala Normal file
View File

@ -0,0 +1,43 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
case class PopularInYourAreaSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
Some(
GeneralContext(
contextType = LocationGeneralContextType,
text = stringCenter.prepare(popularInYourAreaString),
url = None,
contextImageUrls = None,
landingUrl = None
))
} else None
}
}

32
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/YouMightLikeSocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala
View File

@ -1,50 +1,48 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* Renders a fixed 'You Might Like' string above all OON Tweets.
*/
@Singleton
case class YouMightLikeSocialContextBuilder @Inject() (
case class PopularVideoSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val youMightLikeString = externalStrings.socialContextYouMightLikeString
private val popularVideoString = externalStrings.socialContextPopularVideoString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true)
val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false)
if (!isInNetwork && !isRetweet) {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) {
Some(
GeneralContext(
contextType = SparkleGeneralContextType,
text = stringCenter.prepare(youMightLikeString),
text = stringCenter.prepare(popularVideoString),
url = None,
contextImageUrls = None,
landingUrl = None
landingUrl = Some(
Url(
urlType = DeepLink,
url = ""
)
)
))
} else {
None
}
} else None
}
}

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReceivedReplySocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReportTweetChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -8,7 +8,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/RetweeterChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Singleton

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TopicSocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/UnfollowUserChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

45
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel
View File

@ -4,95 +4,56 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider",
"finatra/inject/inject-core/src/main/scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
"representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
"representation-scorer/server/src/main/thrift:thrift-scala",
"servo/repo/src/main/scala",
"snowflake/src/main/scala/com/twitter/snowflake/id",
"src/java/com/twitter/ml/api/constant",
"src/java/com/twitter/search/common/util/lang",
"src/scala/com/twitter/ml/api/util",
"src/scala/com/twitter/timelines/prediction/adapters/real_graph",
"src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/adapters/twistly",
"src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
"src/scala/com/twitter/timelines/prediction/common/util",
"src/scala/com/twitter/timelines/prediction/features/common",
"src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/features/recap",
"src/scala/com/twitter/timelines/prediction/features/time_features",
"src/scala/com/twitter/timelines/prediction/adapters/request_context",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/ml/api:data-java",
"src/thrift/com/twitter/ml/api:embedding-java",
"src/thrift/com/twitter/onboarding/relevance/features:features-java",
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"src/thrift/com/twitter/search/common:constants-java",
"src/thrift/com/twitter/socialgraph:thrift-scala",
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
"src/thrift/com/twitter/timelineranker:thrift-scala",
"src/thrift/com/twitter/timelines/author_features:thrift-java",
"src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
"src/thrift/com/twitter/timelines/impression:thrift-scala",
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
"src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala",
"src/thrift/com/twitter/user_session_store:thrift-java",
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
"src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
"stitch/stitch-core",
"stitch/stitch-gizmoduck",
"stitch/stitch-socialgraph",
"stitch/stitch-timelineservice",
"stitch/stitch-tweetypie",
"strato/config/columns/topic-signals/tsp",
"strato/config/columns/topic-signals/tsp:tsp-strato-client",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
"timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly",
"timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"topic-social-proof/server/src/main/thrift:thrift-scala",
"topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
"tweetconvosvc/thrift/src/main/thrift:thrift-scala",
"twitter-config/yaml",
"user_session_store/src/main/scala/com/twitter/user_session_store",
"util/util-core",
],

8
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala
View File

@ -1,12 +1,10 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
@ -17,16 +15,12 @@ import javax.inject.Singleton
@Singleton
case class FeedbackHistoryQueryFeatureHydrator @Inject() (
feedbackHistoryClient: FeedbackHistoryManhattanClient)
extends QueryFeatureHydrator[PipelineQuery]
with Conditionally[PipelineQuery] {
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableFeedbackFatigueParam)
override def hydrate(
query: PipelineQuery
): Stitch[FeatureMap] =

41
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowedTopicsQueryFeatureHydrator.scala
View File

@ -1,41 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class FollowedTopicsQueryFeatureHydrator @Inject() (
followedTopicsCandidateSource: FollowedTopicsCandidateSource)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
followedTopicsCandidateSource(request)
.map { topics =>
FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
}.handle {
case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
}
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
)
}

58
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorSafetyFeatureHydrator.scala
View File

@ -1,58 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.gizmoduck.{thriftscala => gt}
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.stitch.gizmoduck.Gizmoduck
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("GizmoduckAuthorSafety")
override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
val blueVerifiedStitch = authorIdOption
.map { authorId =>
gizmoduck
.getUserById(
userId = authorId,
queryFields = queryFields
)
.map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
}.getOrElse(Stitch.False)
blueVerifiedStitch.map { isBlueVerified =>
FeatureMapBuilder()
.add(AuthorIsBlueVerifiedFeature, isBlueVerified)
.build()
}
}
}

45
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala
View File

@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -11,7 +12,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
import javax.inject.Inject
import javax.inject.Singleton
@ -19,36 +21,39 @@ import javax.inject.Singleton
@Singleton
case class ImpressionBloomFilterQueryFeatureHydrator[
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
bloomFilter: ImpressionBloomFilter)
extends QueryFeatureHydrator[Query] {
bloomFilterClient: ManhattanStoreClient[
blm.ImpressionBloomFilterKey,
blm.ImpressionBloomFilterSeq
]) extends QueryFeatureHydrator[Query] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
"ImpressionBloomFilter")
private val ImpressionBloomFilterTTL = 7.day
private val ImpressionBloomFilterFalsePositiveRate = 0.002
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
private val SurfaceArea = t.SurfaceArea.HomeTimeline
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
override def hydrate(query: Query): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq =>
val updatedBloomFilterSeq =
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
else {
bloomFilter.addElements(
userId = userId,
surfaceArea = SurfaceArea,
tweetIds = query.seenTweetIds.get,
bloomFilterEntrySeq = bloomFilterSeq,
timeToLive = ImpressionBloomFilterTTL,
falsePositiveRate = ImpressionBloomFilterFalsePositiveRate
)
}
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
}
bloomFilterClient
.get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
.map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
.map { bloomFilterSeq =>
val updatedBloomFilterSeq =
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
else {
ImpressionBloomFilter.addSeenTweetIds(
surfaceArea = SurfaceArea,
tweetIds = query.seenTweetIds.get,
bloomFilterSeq = bloomFilterSeq,
timeToLive = ImpressionBloomFilterTTL,
falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam)
)
}
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
}
}
override val alerts = Seq(

41
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala Normal file
View File

@ -0,0 +1,41 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object InNetworkFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
val featureMaps = candidates.map { candidate =>
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
isSelfTweet || followedUserIds.contains(authorId)
}
FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
}
Stitch.value(featureMaps)
}
}

29
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala
View File

@ -2,8 +2,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
@ -27,17 +29,27 @@ import javax.inject.Singleton
@Singleton
case class PersistenceStoreQueryFeatureHydrator @Inject() (
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3],
statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
private val WhoToFollowExcludedUserIdsLimit = 1000
private val ServedTweetIdsDuration = 1.hour
private val ServedTweetIdsDuration = 10.minutes
private val ServedTweetIdsLimit = 100
private val ServedTweetPreviewIdsDuration = 10.hours
private val ServedTweetPreviewIdsLimit = 10
override val features: Set[Feature[_, _]] =
Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature)
Set(
ServedTweetIdsFeature,
ServedTweetPreviewIdsFeature,
PersistenceEntriesFeature,
WhoToFollowExcludedUserIdsFeature)
private val supportedClients = Seq(
ClientPlatform.IPhone,
@ -80,8 +92,19 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
.flatMap(
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
servedTweetIdsSizeStat.add(servedTweetIds.size)
val servedTweetPreviewIds = timelineResponses
.filter(_.clientPlatform == clientPlatform)
.filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
.sortBy(-_.servedTime.inMilliseconds)
.flatMap(_.entries
.filter(_.entityIdType == EntityIdType.TweetPreview)
.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
FeatureMapBuilder()
.add(ServedTweetIdsFeature, servedTweetIds)
.add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
.add(PersistenceEntriesFeature, timelineResponses)
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
.build()

7
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala
View File

@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.stitch.timelineservice.TimelineService
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
@ -37,10 +38,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
candidate.features
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.map(favoritedBy => favoritedBy -> candidate.candidate.id)
}
@ -59,7 +60,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
candidates.map { candidate =>
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
FeatureMapBuilder()

6
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala
View File

@ -32,11 +32,15 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
val realGraphScoresFeatures = realGraphFollowedUsers
.getOrElse(Seq.empty)
.sortBy(-_.score)
.map(candidate => candidate.userId -> candidate.score)
.map(candidate => candidate.userId -> scaleScore(candidate.score))
.take(RealGraphCandidateCount)
.toMap
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
}
}
// Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
private def scaleScore(score: Double): Double =
if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
}

17
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala
View File

@ -17,9 +17,13 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
import com.twitter.search.common.util.lang.ThriftLanguageUtil
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
import java.util.UUID
import javax.inject.Inject
import javax.inject.Singleton
@ -45,6 +49,9 @@ class RequestQueryFeatureHydrator[
PullToRefreshFeature,
RequestJoinIdFeature,
ServedRequestIdFeature,
TimestampFeature,
TimestampGMTDowFeature,
TimestampGMTHourFeature,
ViewerIdFeature
)
@ -67,6 +74,7 @@ class RequestQueryFeatureHydrator[
override def hydrate(query: Query): Stitch[FeatureMap] = {
val requestContext = query.deviceContext.flatMap(_.requestContextValue)
val servedRequestId = UUID.randomUUID.getMostSignificantBits
val timestamp = query.queryTime.inMilliseconds
val featureMap = FeatureMapBuilder()
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
@ -97,8 +105,15 @@ class RequestQueryFeatureHydrator[
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
.add(ServedRequestIdFeature, Some(servedRequestId))
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
.add(TimestampFeature, timestamp)
.add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
.add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
.add(HasDarkRequestFeature, hasDarkRequest)
.add(ViewerIdFeature, query.getRequiredUserId)
.add(
ViewerIdFeature,
query.getOptionalUserId
.orElse(query.getGuestId).getOrElse(
throw PipelineFailure(BadRequest, "Missing viewer id")))
.build()
Stitch.value(featureMap)

46
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSFollowedUsersQueryFeatureHydrator.scala
View File

@ -1,46 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton
case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
socialGraphStitchClient: SocialGraphStitchClient)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SGSFollowedUsers")
override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
private val SocialGraphLimit = 14999
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
val request = sg.IdsRequest(
relationships = Seq(
sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
),
pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
)
socialGraphStitchClient
.ids(request).map(_.ids)
.map { followedUsers =>
FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
}
}
}

4
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala
View File

@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph
@ -41,8 +42,7 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
val allSocialContextUserIds =
candidates.flatMap { candidate =>
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++

67
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SocialGraphServiceFeatureHydrator.scala
View File

@ -1,67 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SocialGraphService")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
val idsRequest = createIdsRequest(
userId = viewerId,
relationshipTypes = Set(sg.RelationshipType.Following),
targetIds = Some(distinctNonSelfAuthorIds)
)
socialGraphStitchClient
.ids(request = idsRequest, requestContext = None)
.map { idResult =>
authorIds.map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
val inNetworkAuthorIds = idResult.ids.toSet
val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
}
}
}
private def createIdsRequest(
userId: Long,
relationshipTypes: Set[sg.RelationshipType],
targetIds: Option[Seq[Long]] = None
): sg.IdsRequest = sg.IdsRequest(
relationshipTypes.map { relationshipType =>
sg.SrcRelationship(userId, relationshipType, targets = targetIds)
}.toSeq,
Some(sg.PageRequest(selectAll = Some(true)))
)
}

162
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala
View File

@ -1,162 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.contentrecommender.{thriftscala => cr}
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
import com.twitter.topiclisting.TopicListingViewerContext
import com.twitter.tsp.{thriftscala => tsp}
import javax.inject.Inject
import javax.inject.Singleton
import scala.collection.JavaConverters._
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
object TSPInferredTopicDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TSPInferredTopicFeatureHydrator @Inject() (
topicSocialProofClientColumn: TopicSocialProofClientColumn,
statsReceiver: StatsReceiver,
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
override val features: Set[Feature[_, _]] =
Set(
TSPInferredTopicFeature,
TSPInferredTopicDataRecordFeature,
TopicIdSocialContextFeature,
TopicContextFunctionalityTypeFeature)
private val topK = 3
private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
sid.CandidateTweetSourceId.Simcluster,
sid.CandidateTweetSourceId.CroonTweet
)
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
private val DefaultFeatureMap = FeatureMapBuilder()
.add(TSPInferredTopicFeature, Map.empty[Long, Double])
.add(TSPInferredTopicDataRecordFeature, new DataRecord())
.add(TopicIdSocialContextFeature, None)
.add(TopicContextFunctionalityTypeFeature, None)
.build()
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val tags = candidates.collect {
case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
candidate.candidate.id -> candidate.features
.getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
}.toMap
val topicSocialProofRequest =
tsp.TopicSocialProofRequest(
userId = query.getRequiredUserId,
tweetIds = candidates.map(_.candidate.id).toSet,
displayLocation = cr.DisplayLocation.HomeTimeline,
topicListingSetting = tsp.TopicListingSetting.Followable,
context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
bypassModes = None,
// Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
tags = if (tags.isEmpty) None else Some(tags)
)
topicSocialProofClientColumn.fetcher
.fetch(topicSocialProofRequest)
.map(_.v)
.map {
case Some(response) =>
candidates.map { candidate =>
val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
if (topicWithScores.nonEmpty) {
keyFoundCounter.incr()
val (socialProofId, socialProofFunctionalityType) =
if (candidate.features
.getOrElse(CandidateSourceIdFeature, None)
.exists(sourcesToSetSocialProof.contains)) {
getSocialProof(topicWithScores)
} else {
(None, None)
}
val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
val inferredTopicDataRecord =
InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
FeatureMapBuilder()
.add(TSPInferredTopicFeature, inferredTopicFeatures)
.add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
.add(TopicIdSocialContextFeature, socialProofId)
.add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
.build()
} else {
keyLossCounter.incr()
DefaultFeatureMap
}
}
case _ =>
requestFailCounter.incr()
candidates.map { _ =>
DefaultFeatureMap
}
}
}
private def getSocialProof(
topicWithScores: Seq[tsp.TopicWithScore]
): (Option[Long], Option[TopicContextFunctionalityType]) = {
val followingTopicId = topicWithScores
.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
topicId
}
if (followingTopicId.nonEmpty) {
return (followingTopicId, Some(BasicTopicContextFunctionalityType))
}
val implicitFollowingId = topicWithScores.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
topicId
}
if (implicitFollowingId.nonEmpty) {
return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
}
(None, None)
}
private def convertTopicWithScores(
topicWithScores: Seq[tsp.TopicWithScore],
): Map[Long, Double] = {
topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
}
}

251
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimeFeaturesHydrator.scala
View File

@ -1,251 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.FDsl._
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.search.common.features.{thriftscala => sc}
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
import com.twitter.timelines.prediction.features.time_features.TimeFeatures
import com.twitter.util.Duration
import scala.collection.Searching._
object TimeFeaturesDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
Stitch.value {
val richDataRecord = new RichDataRecord()
setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
FeatureMapBuilder()
.add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
.build()
}
}
private def setTimeFeatures(
richDataRecord: RichDataRecord,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
query: PipelineQuery,
): Unit = {
val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
}
private[feature_hydrator] def getTimeFeatures(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
): Option[TimeFeatures] = {
for {
requestTimestampMs <- Some(query.queryTime.inMilliseconds)
tweetId <- Some(candidate.id)
viewerId <- query.getOptionalUserId
tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
}
timeSinceSourceTweetCreation =
existingFeatures
.getOrElse(SourceTweetIdFeature, None)
.flatMap { sourceTweetId =>
timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
requestTimestampMs - sourceTweetCreationTimeMs
}
}
.getOrElse(timeSinceTweetCreation)
if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
} yield {
val timeFeatures = TimeFeatures(
timeSinceTweetCreation = timeSinceTweetCreation,
timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
)
val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
existingFeatures.getOrElse(EarlybirdFeature, None),
timeFeatures,
timeSinceSourceTweetCreation
).getOrElse(timeFeatures)
val nonPollingTimestampsMs =
query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
timeFeaturesWithLastEngagement,
requestTimestampMs,
tweetCreationTimeMs,
nonPollingTimestampsMs
)
timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
}
}
private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
if (SnowflakeId.isSnowflakeId(tweetOrUserId))
Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
else None
}
private def addLastEngagementTimeFeatures(
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
timeFeatures: TimeFeatures,
timeSinceSourceTweetCreation: Long
): Option[TimeFeatures] = {
tweetFeaturesOpt.map { tweetFeatures =>
val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
timeFeatures.copy(
lastFavSinceCreationHrs = lastFavSinceCreationHrs,
lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
lastReplySinceCreationHrs = lastReplySinceCreationHrs,
lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
lastFavSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
lastRetweetSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
lastReplySinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
lastQuoteSinceCreationHrs,
timeSinceSourceTweetCreation
)
)
}
}
private def addNonPollingTimeFeatures(
timeFeatures: TimeFeatures,
requestTimestampMs: Long,
creationTimeMs: Long,
nonPollingTimestampsMs: Option[Seq[Long]]
): Option[TimeFeatures] = {
for {
nonPollingTimestampsMs <- nonPollingTimestampsMs
lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
} yield {
val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
1.0,
timeFeatures.timeSinceTweetCreation
)
/*
* Non-polling timestamps are stored in chronological order.
* The latest timestamps occur first, therefore we need to explicitly search in reverse order.
*/
val nonPollingRequestsSinceTweetCreation =
if (nonPollingTimestampsMs.nonEmpty) {
nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
} else {
0
}
/*
* Calculate the average time between non-polling requests; include
* request time in this calculation as latest timestamp.
*/
val timeBetweenNonPollingRequestsAvg =
(requestTimestampMs - earliestNonPollingTimestampMs) / math
.max(1.0, nonPollingTimestampsMs.size)
val timeFeaturesWithNonPolling = timeFeatures.copy(
timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
tweetAgeRatio = Some(tweetAgeRatio)
)
timeFeaturesWithNonPolling
}
}
private[this] def getTimeSinceLastEngagementHrs(
lastEngagementTimeSinceCreationHrsOpt: Option[Double],
timeSinceTweetCreation: Long
): Option[Double] = {
lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
}
}
private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
val record = richDataRecord.getRecord
.setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
.setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
.setFeatureValueFromOption(
TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
features.timeSinceViewerAccountCreationSecs)
.setFeatureValue(
USER_ID_IS_SNOWFLAKE_ID,
features.timeSinceViewerAccountCreationSecs.isDefined
)
.setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
.setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
.setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
.setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
.setFeatureValueFromOption(
LAST_RETWEET_SINCE_CREATION_HRS,
features.lastRetweetSinceCreationHrs
)
.setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
.setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
/*
* set features whose values are optional as some users do not have non-polling timestamps
*/
features.timeBetweenNonPollingRequestsAvg.foreach(
record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
)
features.timeSinceLastNonPollingRequest.foreach(
record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
)
features.nonPollingRequestsSinceTweetCreation.foreach(
record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
)
features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
}
}

4
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala
View File

@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
extends QueryFeatureHydrator[Query] {
private val TweetImpressionTTL = 1.day
private val TweetImpressionCap = 3000
private val TweetImpressionTTL = 2.days
private val TweetImpressionCap = 5000
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")

47
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala
View File

@ -1,6 +1,9 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
@ -10,11 +13,13 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.util.tweetypie.RequestFields
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
@ -29,17 +34,22 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
import com.twitter.stitch.Stitch
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
import com.twitter.tweetypie.{thriftscala => tp}
import com.twitter.util.logging.Logging
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
class TweetypieFeatureHydrator @Inject() (
tweetypieStitchClient: TweetypieStitchClient,
statsReceiver: StatsReceiver)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Logging {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
override val features: Set[Feature[_, _]] = Set(
AuthorIdFeature,
ExclusiveConversationAuthorIdFeature,
InReplyToTweetIdFeature,
IsHydratedFeature,
IsNsfw,
@ -51,6 +61,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
SourceTweetIdFeature,
SourceUserIdFeature,
TweetTextFeature,
TweetLanguageFeature,
VisibilityReason
)
@ -70,9 +81,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
): Stitch[FeatureMap] = {
val safetyLevel = query.product match {
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
case ForYouProduct => rtf.SafetyLevel.TimelineHome
case ForYouProduct =>
val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
}
@ -82,9 +96,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
includeQuotedTweet = true,
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
safetyLevel = Some(safetyLevel),
forUserId = Some(query.getRequiredUserId)
forUserId = query.getOptionalUserId
)
val exclusiveAuthorIdOpt =
existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
val coreData = found.tweet.coreData
@ -106,6 +123,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
val tweetText = coreData.map(_.text)
val tweetLanguage = found.tweet.language.map(_.language)
val tweetAuthorId = coreData.map(_.userId)
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
@ -127,6 +145,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
FeatureMapBuilder()
.add(AuthorIdFeature, tweetAuthorId)
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, inReplyToTweetId)
.add(IsHydratedFeature, true)
.add(IsNsfw, Some(isNsfw))
@ -137,20 +156,24 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
.add(QuotedUserIdFeature, quotedTweetUserId)
.add(SourceTweetIdFeature, retweetedTweetId)
.add(SourceUserIdFeature, retweetedTweetUserId)
.add(TweetLanguageFeature, tweetLanguage)
.add(TweetTextFeature, tweetText)
.add(VisibilityReason, found.suppressReason)
.build()
// If no tweet result found, return default and pre-existing features
case _ =>
DefaultFeatureMap +
(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) +
(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) +
(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) +
(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) +
(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) +
(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) +
(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
DefaultFeatureMap ++ FeatureMapBuilder()
.add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
.add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
.add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
.add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
.add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
.add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
.add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
.build()
}
}
}

59
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala
View File

@ -1,59 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
import com.twitter.ml.api.DataRecordMerger
import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.CompactDataRecordConverter
import com.twitter.ml.api.util.FDsl._
import com.twitter.timelines.author_features.v1.{thriftjava => af}
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
private val originalAuthorAggregatesFeatures =
TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
.buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
private val authorFeatures = originalAuthorAggregatesFeatures ++
Seq(
UserHealthFeatures.AuthorState,
UserHealthFeatures.NumAuthorFollowers,
UserHealthFeatures.NumAuthorConnectDays,
UserHealthFeatures.NumAuthorConnect)
private val featureContext = new FeatureContext(authorFeatures: _*)
override def getFeatureContext: FeatureContext = featureContext
override val commonFeatures: Set[Feature[_]] = Set.empty
private val compactDataRecordConverter = new CompactDataRecordConverter()
private val drMerger = new DataRecordMerger()
override def setFeatures(
authorFeaturesOpt: Option[af.AuthorFeatures],
richDataRecord: RichDataRecord
): Unit = {
authorFeaturesOpt.foreach { authorFeatures =>
val dataRecord = richDataRecord.getRecord
dataRecord.setFeatureValue(
UserHealthFeatures.AuthorState,
authorFeatures.user_health.user_state.getValue.toLong)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorFollowers,
authorFeatures.user_health.num_followers.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnectDays,
authorFeatures.user_health.num_connect_days.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnect,
authorFeatures.user_health.num_connect.toDouble)
val originalAuthorAggregatesDataRecord =
compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
}
}
}

22
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala
View File

@ -1,22 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates
import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("Phase2EdgeAggregate")
override val aggregateFeatures: Set[BaseEdgeAggregateFeature] =
Set(
UserEngagerAggregateFeature,
UserEngagerGoodClickAggregateFeature,
UserInferredTopicAggregateFeature,
UserTopicAggregateFeature,
UserMediaUnderstandingAnnotationAggregateFeature
)
}

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel
View File

@ -4,6 +4,7 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
@ -16,9 +17,11 @@ scala_library(
"src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala",
"stitch/stitch-core",
"stitch/stitch-socialgraph",
"stitch/stitch-tweetypie",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"util/util-slf4j-api/src/main/scala",
],
exports = [
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala
View File

@ -72,7 +72,7 @@ object FeedbackFatigueFilter
originalAuthorId.exists(authorsToFilter.contains) ||
(likers.nonEmpty && eligibleLikers.isEmpty) ||
(followers.nonEmpty && eligibleFollowers.isEmpty) ||
(followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) ||
(candidate.features.getOrElse(IsRetweetFeature, false) &&
authorId.exists(retweetersToFilter.contains))
}

70
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala Normal file
View File

@ -0,0 +1,70 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.finagle.tracing.Trace
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph
import com.twitter.util.logging.Logging
import javax.inject.Inject
import javax.inject.Singleton
/**
* Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author
*
* If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for
* subscription tweets, so we explicitly filter those tweets out.
*/
@Singleton
case class InvalidSubscriptionTweetFilter @Inject() (
socialGraphClient: SocialGraph,
statsReceiver: StatsReceiver)
extends Filter[PipelineQuery, TweetCandidate]
with Logging {
override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet")
private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet")
private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = Stitch
.traverse(candidates) { candidate =>
val exclusiveAuthorId =
candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None)
if (exclusiveAuthorId.isDefined) {
val request = sg.ExistsRequest(
source = query.getRequiredUserId,
target = exclusiveAuthorId.get,
relationships =
Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)),
)
socialGraphClient.exists(request).map(_.exists).map { valid =>
if (!valid) invalidCounter.incr() else validCounter.incr()
valid
}
} else Stitch.value(true)
}.map { validResults =>
val (kept, removed) = candidates
.map(_.candidate)
.zip(validResults)
.partition { case (candidate, valid) => valid }
val keptCandidates = kept.map { case (candidate, _) => candidate }
val removedCandidates = removed.map { case (candidate, _) => candidate }
FilterResult(kept = keptCandidates, removed = removedCandidates)
}
}

59
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateFeatureFilter.scala
View File

@ -1,59 +0,0 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
/**
* Predicate which will be applied to each candidate. True indicates that the candidate will be
* @tparam Candidate - the type of the candidate
*/
trait ShouldKeepCandidate {
def apply(features: FeatureMap): Boolean
}
object PredicateFeatureFilter {
/**
* Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity,
* we recommend including the name of the shouldKeepCandidate parameter.
*
* @param identifier A FilterIdentifier for the new filter
* @param shouldKeepCandidate A predicate function. Candidates will be kept when
* this function returns True.
*/
def fromPredicate[Candidate <: UniversalNoun[Any]](
identifier: FilterIdentifier,
shouldKeepCandidate: ShouldKeepCandidate
): Filter[PipelineQuery, Candidate] = {
val i = identifier
new Filter[PipelineQuery, Candidate] {
override val identifier: FilterIdentifier = i
/**
* Filter the list of candidates
*
* @return a FilterResult including both the list of kept candidate and the list of removed candidates
*/
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]] = {
val allowedIds = candidates
.filter(candidate => shouldKeepCandidate(candidate.features)).map(_.candidate.id).toSet
val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition {
candidate => allowedIds.contains(candidate.id)
}
Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
}
}
}
}

20
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/KeepBestOutOfNetworkTweetPerAuthorFilter.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala
View File

@ -1,8 +1,6 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
@ -11,24 +9,20 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] {
object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("KeepBestOutOfNetworkTweetPerAuthor")
override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
// Set containing best OON tweet for each authorId
val bestCandidatesForAuthorId = candidates
.filter(!_.features.getOrElse(InNetworkFeature, true))
.groupBy(_.features.getOrElse(AuthorIdFeature, None))
.values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None)))
.toSet
val servedTweetPreviewIds =
query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet
val (removed, kept) = candidates.partition { candidate =>
!candidate.features.getOrElse(InNetworkFeature, true) &&
!bestCandidatesForAuthorId.contains(candidate)
servedTweetPreviewIds.contains(candidate.candidate.id)
}
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))

32
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala Normal file
View File

@ -0,0 +1,32 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("Reply")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
val (kept, removed) = candidates
.partition { candidate =>
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
}
val filterResult = FilterResult(
kept = kept.map(_.candidate),
removed = removed.map(_.candidate)
)
Stitch.value(filterResult)
}
}

32
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala Normal file
View File

@ -0,0 +1,32 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("Retweet")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
val (kept, removed) = candidates
.partition { candidate =>
!candidate.features.getOrElse(IsRetweetFeature, false)
}
val filterResult = FilterResult(
kept = kept.map(_.candidate),
removed = removed.map(_.candidate)
)
Stitch.value(filterResult)
}
}

1
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel
View File

@ -14,7 +14,6 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"stitch/stitch-socialgraph",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
],

8
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala
View File

@ -1,14 +1,14 @@
package com.twitter.home_mixer.functional_component.gate
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.functional_component.gate.Gate
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
import com.twitter.stitch.Stitch
import com.twitter.util.Duration
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.util.Duration
object DismissFatigueGate {
// how long a dismiss action from user needs to be respected

18
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/NonEmptySeqFeatureGate.scala
View File

@ -1,18 +0,0 @@
package com.twitter.home_mixer.functional_component.gate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.functional_component.gate.Gate
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import scala.reflect.runtime.universe._
case class NonEmptySeqFeatureGate[T: TypeTag](
feature: Feature[PipelineQuery, Seq[T]])
extends Gate[PipelineQuery] {
override val identifier: GateIdentifier = GateIdentifier(s"NonEmptySeq$feature")
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
Stitch.value(query.features.exists(_.get(feature).nonEmpty))
}

6
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala
View File

@ -19,7 +19,7 @@ object EditedTweetsCandidatePipelineQueryTransformer
override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
// The time window for which a tweet remains editable after creation.
private val EditTimeWindow = 30.minutes
private val EditTimeWindow = 60.minutes
override def transform(query: PipelineQuery): Seq[Long] = {
val applicableCandidates = getApplicableCandidates(query)
@ -29,8 +29,8 @@ object EditedTweetsCandidatePipelineQueryTransformer
// Any tweets in it could have become stale since being served.
val previousTimelineLoadTime = applicableCandidates.head.servedTime
// The time window for editing a tweet is 30 minutes,
// so we ignore responses older than (PTL Time - 30 mins).
// The time window for editing a tweet is 60 minutes,
// so we ignore responses older than (PTL Time - 60 mins).
val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
.takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)

1
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel
View File

@ -6,7 +6,6 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",

76
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala
View File

@ -35,8 +35,8 @@ object FeedbackFatigueScorer
override def onlyIf(query: PipelineQuery): Boolean =
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
private val DurationForFiltering = 14.days
private val DurationForDiscounting = 140.days
val DurationForFiltering = 14.days
val DurationForDiscounting = 140.days
private val ScoreMultiplierLowerBound = 0.2
private val ScoreMultiplierUpperBound = 1.0
private val ScoreMultiplierIncrementsCount = 4
@ -76,42 +76,56 @@ object FeedbackFatigueScorer
feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty))
val featureMaps = candidates.map { candidate =>
val multiplier = getScoreMultiplier(
candidate,
authorsToDiscount,
likersToDiscount,
followersToDiscount,
retweetersToDiscount
)
val score = candidate.features.getOrElse(ScoreFeature, None)
val originalAuthorId =
CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
val likerMultipliers = likers.flatMap(likersToDiscount.get)
val likerMultiplier =
if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size)
likerMultipliers.max
else 1.0
val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
val followerMultipliers = followers.flatMap(followersToDiscount.get)
val followerMultiplier =
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size)
followerMultipliers.max
else 1.0
val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L)
val retweeterMultiplier =
if (candidate.features.getOrElse(IsRetweetFeature, false))
retweetersToDiscount.getOrElse(authorId, 1.0)
else 1.0
val multiplier =
originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
}
Stitch.value(featureMaps)
}
private def getUserDiscounts(
def getScoreMultiplier(
candidate: CandidateWithFeatures[TweetCandidate],
authorsToDiscount: Map[Long, Double],
likersToDiscount: Map[Long, Double],
followersToDiscount: Map[Long, Double],
retweetersToDiscount: Map[Long, Double],
): Double = {
val originalAuthorId =
CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
val likerMultipliers = likers.flatMap(likersToDiscount.get)
val likerMultiplier =
if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size)
likerMultipliers.max
else 1.0
val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
val followerMultipliers = followers.flatMap(followersToDiscount.get)
val followerMultiplier =
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size &&
likers.isEmpty)
followerMultipliers.max
else 1.0
val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L)
val retweeterMultiplier =
if (candidate.features.getOrElse(IsRetweetFeature, false))
retweetersToDiscount.getOrElse(authorId, 1.0)
else 1.0
originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
}
def getUserDiscounts(
queryTime: Time,
feedbackEntries: Seq[FeedbackEntry],
): Map[Long, Double] = {

61
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/VerifiedAuthorScalingScorer.scala
View File

@ -1,61 +0,0 @@
package com.twitter.home_mixer.functional_component.scorer
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorInNetworkMultiplierParam
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorOutOfNetworkMultiplierParam
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
/**
* Scales scores of tweets whose author is Blue Verified by the provided scale factor
*/
object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate] {
override val identifier: ScorerIdentifier = ScorerIdentifier("VerifiedAuthorScaling")
override val features: Set[Feature[_, _]] = Set(ScoreFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
Stitch.value {
candidates.map { candidate =>
val score = candidate.features.getOrElse(ScoreFeature, None)
val updatedScore = getUpdatedScore(score, candidate, query)
FeatureMapBuilder().add(ScoreFeature, updatedScore).build()
}
}
}
/**
* We should only be applying this multiplier if the author of the candidate is Blue Verified.
* We also treat In-Network vs Out-of-Network differently.
*/
private def getUpdatedScore(
score: Option[Double],
candidate: CandidateWithFeatures[TweetCandidate],
query: PipelineQuery
): Option[Double] = {
val isAuthorBlueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false)
if (isAuthorBlueVerified) {
val isCandidateInNetwork = candidate.features.getOrElse(InNetworkFeature, false)
val scaleFactor =
if (isCandidateInNetwork) query.params(BlueVerifiedAuthorInNetworkMultiplierParam)
else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam)
score.map(_ * scaleFactor)
} else score
}
}

1
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel
View File

@ -5,6 +5,7 @@ scala_library(
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala
View File

@ -1,6 +1,6 @@
package com.twitter.home_mixer.functional_component.selector
import com.twitter.home_mixer.functional_component.decorator.HomeClientEventDetailsBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature

8
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel
View File

@ -7,14 +7,12 @@ scala_library(
"3rdparty/jvm/javax/inject:javax.inject",
"eventbus/client/src/main/scala/com/twitter/eventbus/client",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
@ -22,6 +20,7 @@ scala_library(
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
"src/scala/com/twitter/timelines/prediction/common/adapters",
@ -33,13 +32,12 @@ scala_library(
"src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java",
"src/thrift/com/twitter/timelines/timeline_logging:thrift-scala",
"src/thrift/com/twitter/user_session_store:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/core",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/uss",
"timelines/ml:kafka",
"timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka",
"timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding",
"timelines/src/main/scala/com/twitter/timelines/clientconfig",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",

14
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala
View File

@ -2,13 +2,14 @@ package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates
import com.twitter.home_mixer.functional_component.decorator.HomeTweetTypePredicates
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates
import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -18,15 +19,17 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.injection.scribe.InjectionScribeUtil
private[side_effect] sealed trait ClientEventsBuilder {
private val FollowingSection = Some("home_latest")
private val FollowingSection = Some("latest")
private val ForYouSection = Some("home")
private val ListTweetsSection = Some("list")
private val SubscribedSection = Some("subscribed")
protected def section(query: PipelineQuery): Option[String] = {
query.product match {
case FollowingProduct => FollowingSection
case ForYouProduct => ForYouSection
case ListTweetsProduct => ListTweetsSection
case SubscribedProduct => SubscribedSection
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
}
}
@ -52,6 +55,7 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
private val InjectedComponent = Some("injected")
private val PromotedComponent = Some("promoted")
private val WhoToFollowComponent = Some("who_to_follow")
private val WhoToSubscribeComponent = Some("who_to_subscribe")
private val WithVideoDurationComponent = Some("with_video_duration")
private val VideoDurationSumElement = Some("video_duration_sum")
private val NumVideosElement = Some("num_videos")
@ -60,7 +64,8 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
query: PipelineQuery,
injectedTweets: Seq[ItemCandidateWithDetails],
promotedTweets: Seq[ItemCandidateWithDetails],
whoToFollowUsers: Seq[ItemCandidateWithDetails]
whoToFollowUsers: Seq[ItemCandidateWithDetails],
whoToSubscribeUsers: Seq[ItemCandidateWithDetails]
): Seq[ClientEvent] = {
val baseEventNamespace = EventNamespace(
section = section(query),
@ -77,6 +82,9 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
ClientEvent(
baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction),
eventValue = count(whoToFollowUsers)),
ClientEvent(
baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction),
eventValue = count(whoToSubscribeUsers)),
)
val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map {

27
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala
View File

@ -5,6 +5,7 @@ import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.home_mixer.util.CandidatesUtil
import com.twitter.logpipeline.client.common.EventPublisher
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
@ -15,16 +16,30 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
* Side effect that logs served tweet metrics to Scribe as client events.
*/
case class HomeScribeClientEventSideEffect(
enableScribeClientEvents: Boolean,
override val logPipelinePublisher: EventPublisher[LogEvent],
injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier],
adsCandidatePipelineIdentifier: CandidatePipelineIdentifier,
adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] {
whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None)
extends ScribeClientEventSideEffect[PipelineQuery, Timeline]
with PipelineResultSideEffect.Conditionally[
PipelineQuery,
Timeline
] {
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent")
override val page = "timelinemixer"
override def onlyIf(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Boolean = enableScribeClientEvents
override def buildClientEvents(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
@ -37,13 +52,15 @@ case class HomeScribeClientEventSideEffect(
val sources = itemCandidates.groupBy(_.source)
val injectedTweets =
injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty))
val promotedTweets = sources.getOrElse(adsCandidatePipelineIdentifier, Seq.empty)
val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
// WhoToFollow module is not required for all home-mixer products, e.g. list tweets timeline.
// WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline.
val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
val whoToSubscribeUsers =
whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
val servedEvents = ServedEventsBuilder
.build(query, injectedTweets, promotedTweets, whoToFollowUsers)
.build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers)
val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets)

245
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala Normal file
View File

@ -0,0 +1,245 @@
package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.finagle.tracing.Trace
import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller
import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
import com.twitter.home_mixer.model.request.HasDeviceContext
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag
import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.inject.annotations.Flag
import com.twitter.logpipeline.client.common.EventPublisher
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
import com.twitter.util.Time
import javax.inject.Inject
import javax.inject.Singleton
/**
* Side effect that logs home timeline served candidates to Scribe.
*/
@Singleton
class HomeScribeServedCandidatesSideEffect @Inject() (
@Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean,
scribeEventPublisher: EventPublisher[thrift.ServedEntry])
extends ScribeLogEventSideEffect[
thrift.ServedEntry,
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
]
with PipelineResultSideEffect.Conditionally[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
] {
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates")
override def onlyIf(
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam)
override def buildLogEvents(
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Seq[thrift.ServedEntry] = {
val timelineType = query.product match {
case FollowingProduct => thrift.TimelineType.HomeLatest
case ForYouProduct => thrift.TimelineType.Home
case SubscribedProduct => thrift.TimelineType.HomeSubscribed
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
}
val requestProvenance = query.deviceContext.map { deviceContext =>
deviceContext.requestContextValue match {
case RequestContext.Foreground => thrift.RequestProvenance.Foreground
case RequestContext.Launch => thrift.RequestProvenance.Launch
case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
case _ => thrift.RequestProvenance.Other
}
}
val queryType = query.features.map { featureMap =>
if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
else thrift.QueryType.Other
}
val requestInfo = thrift.RequestInfo(
requestTimeMs = query.queryTime.inMilliseconds,
traceId = Trace.id.traceId.toLong,
userId = query.getOptionalUserId,
clientAppId = query.clientContext.appId,
hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
parentId = Some(Trace.id.parentId.toLong),
spanId = Some(Trace.id.spanId.toLong),
timelineType = Some(timelineType),
ipAddress = query.clientContext.ipAddress,
userAgent = query.clientContext.userAgent,
queryType = queryType,
requestProvenance = requestProvenance,
languageCode = query.clientContext.languageCode,
countryCode = query.clientContext.countryCode,
requestEndTimeMs = Some(Time.now.inMilliseconds),
servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None))
)
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
selectedCandidates.flatMap {
case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
Seq((item.candidateIdLong, item))
case module: ModuleCandidateWithDetails
if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
module.candidates.map(item => (item.candidateIdLong, item))
case _ => Seq.empty
}.toMap
val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
selectedCandidates.flatMap {
case module: ModuleCandidateWithDetails
if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
module.candidates.map { item =>
(item.candidateIdLong, item)
}
case _ => Seq.empty
}.toMap
response.instructions.zipWithIndex
.collect {
case (AddEntriesTimelineInstruction(entries), index) =>
entries.collect {
case entry: TweetItem if entry.promotedMetadata.isDefined =>
val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index)
Seq(
thrift.EntryInfo(
id = entry.id,
position = index.shortValue(),
entryId = entry.entryIdentifier,
entryType = thrift.EntryType.PromotedTweet,
sortIndex = entry.sortIndex,
verticalSize = Some(1),
displayType = Some(entry.displayType.toString),
details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails))
)
)
case entry: TweetItem =>
val candidate = tweetIdToItemCandidateMap(entry.id)
val tweetDetails = TweetDetailsMarshaller(entry, candidate)
Seq(
thrift.EntryInfo(
id = candidate.candidateIdLong,
position = index.shortValue(),
entryId = entry.entryIdentifier,
entryType = thrift.EntryType.Tweet,
sortIndex = entry.sortIndex,
verticalSize = Some(1),
score = candidate.features.getOrElse(ScoreFeature, None),
displayType = Some(entry.displayType.toString),
details = Some(thrift.ItemDetails.TweetDetails(tweetDetails))
)
)
case module: TimelineModule
if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
module.items.collect {
case ModuleItem(entry: UserItem, _, _) =>
val candidate = userIdToItemCandidateMap(entry.id)
val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate)
thrift.EntryInfo(
id = entry.id,
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.WhoToFollowModule,
sortIndex = module.sortIndex,
score = candidate.features.getOrElse(ScoreFeature, None),
displayType = Some(entry.displayType.toString),
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails))
)
}
case module: TimelineModule
if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
module.items.collect {
case ModuleItem(entry: UserItem, _, _) =>
val candidate = userIdToItemCandidateMap(entry.id)
val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate)
thrift.EntryInfo(
id = entry.id,
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.WhoToSubscribeModule,
sortIndex = module.sortIndex,
score = candidate.features.getOrElse(ScoreFeature, None),
displayType = Some(entry.displayType.toString),
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails))
)
}
case module: TimelineModule
if module.sortIndex.isDefined && module.items.headOption.exists(
_.item.isInstanceOf[TweetItem]) =>
module.items.collect {
case ModuleItem(entry: TweetItem, _, _) =>
val candidate = tweetIdToItemCandidateMap(entry.id)
thrift.EntryInfo(
id = entry.id,
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.ConversationModule,
sortIndex = module.sortIndex,
score = candidate.features.getOrElse(ScoreFeature, None),
displayType = Some(entry.displayType.toString)
)
}
case _ => Seq.empty
}.flatten
// Other instructions
case _ => Seq.empty[thrift.EntryInfo]
}.flatten.map { entryInfo =>
thrift.ServedEntry(
entry = Some(entryInfo),
request = requestInfo
)
}
}
override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] =
scribeEventPublisher
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
)
}

212
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedEntriesSideEffect.scala
View File

@ -1,212 +0,0 @@
package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.finagle.tracing.Trace
import com.twitter.home_mixer.marshaller.timeline_logging.ConversationEntryMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetEntryMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.TweetEntryMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowEntryMarshaller
import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
import com.twitter.home_mixer.model.request.HasDeviceContext
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.logpipeline.client.common.EventPublisher
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
import com.twitter.util.Time
import javax.inject.Inject
import javax.inject.Singleton
/**
* Side effect that logs home timeline served entries to Scribe.
*/
@Singleton
class HomeScribeServedEntriesSideEffect @Inject() (
scribeEventPublisher: EventPublisher[thrift.Timeline])
extends PipelineResultSideEffect[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
] {
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedEntries")
final override def apply(
inputs: PipelineResultSideEffect.Inputs[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
]
): Stitch[Unit] = {
val timelineThrift = buildTimeline(inputs)
Stitch.callFuture(scribeEventPublisher.publish(timelineThrift)).unit
}
def buildTimeline(
inputs: PipelineResultSideEffect.Inputs[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
]
): thrift.Timeline = {
val timelineType = inputs.query.product match {
case FollowingProduct => thrift.TimelineType.HomeLatest
case ForYouProduct => thrift.TimelineType.Home
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
}
val requestProvenance = inputs.query.deviceContext.map { deviceContext =>
deviceContext.requestContextValue match {
case RequestContext.Foreground => thrift.RequestProvenance.Foreground
case RequestContext.Launch => thrift.RequestProvenance.Launch
case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
case _ => thrift.RequestProvenance.Other
}
}
val queryType = inputs.query.features.map { featureMap =>
if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
else thrift.QueryType.Other
}
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
inputs.selectedCandidates.flatMap {
case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
Seq((item.candidateIdLong, item))
case module: ModuleCandidateWithDetails
if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
module.candidates.map(item => (item.candidateIdLong, item))
case _ => Seq.empty
}.toMap
val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
inputs.selectedCandidates.flatMap {
case module: ModuleCandidateWithDetails
if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
module.candidates.map { item =>
(item.candidateIdLong, item)
}
case _ => Seq.empty
}.toMap
val timelineEntries = inputs.response.instructions.zipWithIndex.collect {
case (AddEntriesTimelineInstruction(entries), index) =>
entries.collect {
case entry: TweetItem if entry.promotedMetadata.isDefined =>
val promotedTweetEntry = PromotedTweetEntryMarshaller(entry, index)
Seq(
thrift.TimelineEntry(
content = thrift.Content.PromotedTweetEntry(promotedTweetEntry),
position = index.shortValue(),
entryId = entry.entryIdentifier,
entryType = thrift.EntryType.PromotedTweet,
sortIndex = entry.sortIndex,
verticalSize = Some(1)
)
)
case entry: TweetItem =>
val candidate = tweetIdToItemCandidateMap(entry.id)
val tweetEntry = TweetEntryMarshaller(entry, candidate)
Seq(
thrift.TimelineEntry(
content = thrift.Content.TweetEntry(tweetEntry),
position = index.shortValue(),
entryId = entry.entryIdentifier,
entryType = thrift.EntryType.Tweet,
sortIndex = entry.sortIndex,
verticalSize = Some(1)
)
)
case module: TimelineModule
if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
val whoToFollowEntries = module.items.collect {
case ModuleItem(entry: UserItem, _, _) =>
val candidate = userIdToItemCandidateMap(entry.id)
val whoToFollowEntry = WhoToFollowEntryMarshaller(entry, candidate)
thrift.AtomicEntry.WtfEntry(whoToFollowEntry)
}
Seq(
thrift.TimelineEntry(
content = thrift.Content.Entries(whoToFollowEntries),
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.WhoToFollowModule,
sortIndex = module.sortIndex
)
)
case module: TimelineModule
if module.sortIndex.isDefined && module.items.headOption.exists(
_.item.isInstanceOf[TweetItem]) =>
val conversationTweetEntries = module.items.collect {
case ModuleItem(entry: TweetItem, _, _) =>
val candidate = tweetIdToItemCandidateMap(entry.id)
val conversationEntry = ConversationEntryMarshaller(entry, candidate)
thrift.AtomicEntry.ConversationEntry(conversationEntry)
}
Seq(
thrift.TimelineEntry(
content = thrift.Content.Entries(conversationTweetEntries),
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.ConversationModule,
sortIndex = module.sortIndex
)
)
case _ => Seq.empty
}.flatten
// Other instructions
case _ => Seq.empty[thrift.TimelineEntry]
}.flatten
thrift.Timeline(
timelineEntries = timelineEntries,
requestTimeMs = inputs.query.queryTime.inMilliseconds,
traceId = Trace.id.traceId.toLong,
userId = inputs.query.getOptionalUserId,
clientAppId = inputs.query.clientContext.appId,
sourceJobInstance = None,
hasDarkRequest = inputs.query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
parentId = Some(Trace.id.parentId.toLong),
spanId = Some(Trace.id.spanId.toLong),
timelineType = Some(timelineType),
ipAddress = inputs.query.clientContext.ipAddress,
userAgent = inputs.query.clientContext.userAgent,
queryType = queryType,
requestProvenance = requestProvenance,
sessionId = None,
timeZone = None,
browserNotificationPermission = None,
lastNonePollingTimeMs = None,
languageCode = inputs.query.clientContext.languageCode,
countryCode = inputs.query.clientContext.countryCode,
requestEndTimeMs = Some(Time.now.inMilliseconds),
servedRequestId = inputs.query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
requestJoinId = inputs.query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)),
requestSeenTweetIds = inputs.query.seenTweetIds
)
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
)
}

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala
View File

@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.eventbus.client.EventBusPublisher
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
@ -22,6 +23,7 @@ import javax.inject.Singleton
object PublishClientSentImpressionsEventBusSideEffect {
val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline))
val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline))
val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed))
}
/**
@ -56,6 +58,7 @@ class PublishClientSentImpressionsEventBusSideEffect @Inject() (
val surfaceArea = query.product match {
case ForYouProduct => HomeSurfaceArea
case FollowingProduct => HomeLatestSurfaceArea
case SubscribedProduct => HomeSubscribedSurfaceArea
case _ => None
}
query.seenTweetIds.map { seenTweetIds =>

47
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateImpressionBloomFilterSideEffect.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala
View File

@ -1,39 +1,46 @@
package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBloomFilter)
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, Timeline]
with PipelineResultSideEffect.Conditionally[PipelineQuery with HasSeenTweetIds, Timeline] {
private val SurfaceArea = t.SurfaceArea.HomeTimeline
class PublishImpressionBloomFilterSideEffect @Inject() (
bloomFilterClient: ManhattanStoreClient[
blm.ImpressionBloomFilterKey,
blm.ImpressionBloomFilterSeq
]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling]
with PipelineResultSideEffect.Conditionally[
PipelineQuery with HasSeenTweetIds,
HasMarshalling
] {
override val identifier: SideEffectIdentifier =
SideEffectIdentifier("UpdateImpressionBloomFilter")
SideEffectIdentifier("PublishImpressionBloomFilter")
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
override def onlyIf(
query: PipelineQuery with HasSeenTweetIds,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Boolean = query.seenTweetIds.exists(_.nonEmpty)
response: HasMarshalling
): Boolean =
query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty)
def buildEvents(query: PipelineQuery): Option[t.ImpressionBloomFilterSeq] = {
def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = {
query.features.flatMap { featureMap =>
val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature)
if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq)
@ -42,19 +49,17 @@ class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBl
}
override def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, Timeline]
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling]
): Stitch[Unit] = {
buildEvents(inputs.query)
.map { updatedBloomFilter =>
bloomFilter.writeBloomFilterSeq(
userId = inputs.query.getRequiredUserId,
surfaceArea = SurfaceArea,
impressionBloomFilterSeq = updatedBloomFilter)
.map { updatedBloomFilterSeq =>
bloomFilterClient.write(
blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea),
updatedBloomFilterSeq)
}.getOrElse(Stitch.Unit)
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8),
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(30.millis)
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8)
)
}

80
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ServedStatsSideEffect.scala
View File

@ -1,80 +0,0 @@
package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.param.HomeGlobalParams.AuthorListForStatsParam
import com.twitter.home_mixer.util.CandidatesUtil
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver)
extends PipelineResultSideEffect[PipelineQuery, Timeline] {
override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats")
private val baseStatsReceiver = statsReceiver.scope(identifier.toString)
private val authorStatsReceiver = baseStatsReceiver.scope("Author")
private val candidateSourceStatsReceiver = baseStatsReceiver.scope("CandidateSource")
private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance")
private val inNetworkStatsCounter = contentBalanceStatsReceiver.counter("InNetwork")
private val outOfNetworkStatsCounter = contentBalanceStatsReceiver.counter("OutOfNetwork")
override def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
): Stitch[Unit] = {
val tweetCandidates = CandidatesUtil
.getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]())
recordAuthorStats(tweetCandidates, inputs.query.params(AuthorListForStatsParam))
recordCandidateSourceStats(tweetCandidates)
recordContentBalanceStats(tweetCandidates)
Stitch.Unit
}
def recordAuthorStats(candidates: Seq[CandidateWithDetails], authors: Set[Long]): Unit = {
candidates
.filter { candidate =>
candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) &&
// Only include original tweets
(!candidate.features.getOrElse(IsRetweetFeature, false)) &&
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
}
.groupBy { candidate =>
(getCandidateSourceId(candidate), candidate.features.get(AuthorIdFeature).get)
}
.foreach {
case ((candidateSourceId, authorId), authorCandidates) =>
authorStatsReceiver
.scope(authorId.toString).counter(candidateSourceId).incr(authorCandidates.size)
}
}
def recordCandidateSourceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
candidates.groupBy(getCandidateSourceId).foreach {
case (candidateSourceId, candidateSourceCandidates) =>
candidateSourceStatsReceiver.counter(candidateSourceId).incr(candidateSourceCandidates.size)
}
}
def recordContentBalanceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true))
inNetworkStatsCounter.incr(in.size)
outOfNetworkStatsCounter.incr(oon.size)
}
private def getCandidateSourceId(candidate: CandidateWithDetails): String =
candidate.features.getOrElse(CandidateSourceIdFeature, None).map(_.name).getOrElse("None")
}

28
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala
View File

@ -3,8 +3,10 @@ package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
@ -97,12 +99,14 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
val entries = inputs.response.instructions.collect {
case AddEntriesTimelineInstruction(entries) =>
entries.collect {
// includes both tweets and promoted tweets
case entry: TweetItem if entry.sortIndex.isDefined =>
// includes tweets, tweet previews, and promoted tweets
case entry: TweetItem if entry.sortIndex.isDefined => {
Seq(
buildTweetEntryWithItemIds(
tweetIdToItemCandidateMap(entry.id),
entry.sortIndex.get))
entry.sortIndex.get
))
}
// tweet conversation modules are flattened to individual tweets in the persistence store
case module: TimelineModule
if module.sortIndex.isDefined && module.items.headOption.exists(
@ -125,6 +129,19 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
size = module.items.size.toShort,
itemIds = Some(userIds)
))
case module: TimelineModule
if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
val userIds = module.items
.map(item =>
UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId =
Some(item.item.id.asInstanceOf[Long])))
Seq(
EntryWithItemIds(
entityIdType = EntityIdType.WhoToSubscribe,
sortIndex = module.sortIndex.get,
size = module.items.size.toShort,
itemIds = Some(userIds)
))
}.flatten
case ShowCoverInstruction(cover) =>
Seq(
@ -216,8 +233,11 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
userId = None
)
val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false)
val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet
EntryWithItemIds(
entityIdType = EntityIdType.Tweet,
entityIdType = entityType,
sortIndex = sortIndex,
size = 1.toShort,
itemIds = Some(Seq(itemIds))

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel
View File

@ -4,14 +4,12 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
],
exports = [
"dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",

18
home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala
View File

@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProductContext
import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext
import com.twitter.home_mixer.model.request.ListTweetsProductContext
import com.twitter.home_mixer.model.request.ScoredTweetsProductContext
import com.twitter.home_mixer.model.request.SubscribedProductContext
import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.model.marshalling.request.ProductContext
import javax.inject.Inject
import javax.inject.Singleton
@ -26,15 +26,17 @@ class HomeMixerProductContextUnmarshaller @Inject() (
ForYouProductContext(
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
seenTweetIds = p.seenTweetIds,
dspClientContext = p.dspClientContext
dspClientContext = p.dspClientContext,
pushToHomeTweetId = p.pushToHomeTweetId
)
case t.ProductContext.Realtime(p) =>
case t.ProductContext.ListManagement(p) =>
throw new UnsupportedOperationException(s"This product is no longer used")
case t.ProductContext.ScoredTweets(p) =>
ScoredTweetsProductContext(
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
seenTweetIds = p.seenTweetIds,
servedTweetIds = p.servedTweetIds
servedTweetIds = p.servedTweetIds,
backfillTweetIds = p.backfillTweetIds
)
case t.ProductContext.ListTweets(p) =>
ListTweetsProductContext(
@ -46,7 +48,13 @@ class HomeMixerProductContextUnmarshaller @Inject() (
ListRecommendedUsersProductContext(
listId = p.listId,
selectedUserIds = p.selectedUserIds,
excludedUserIds = p.excludedUserIds
excludedUserIds = p.excludedUserIds,
listName = p.listName
)
case t.ProductContext.Subscribed(p) =>
SubscribedProductContext(
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
seenTweetIds = p.seenTweetIds,
)
case t.ProductContext.UnknownUnionField(field) =>
throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}")

5
home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala
View File

@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.model.marshalling.request.Product
import javax.inject.Inject
import javax.inject.Singleton
@ -17,11 +17,12 @@ class HomeMixerProductUnmarshaller @Inject() () {
def apply(product: t.Product): Product = product match {
case t.Product.Following => FollowingProduct
case t.Product.ForYou => ForYouProduct
case t.Product.Realtime =>
case t.Product.ListManagement =>
throw new UnsupportedOperationException(s"This product is no longer used")
case t.Product.ScoredTweets => ScoredTweetsProduct
case t.Product.ListTweets => ListTweetsProduct
case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct
case t.Product.Subscribed => SubscribedProduct
case t.Product.EnumUnknownProduct(value) =>
throw new UnsupportedOperationException(s"Unknown product: $value")
}

16
home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/ConversationEntryMarshaller.scala
View File

@ -1,16 +0,0 @@
package com.twitter.home_mixer.marshaller.timeline_logging
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
object ConversationEntryMarshaller {
def apply(entry: TweetItem, candidate: ItemCandidateWithDetails): thriftlog.ConversationEntry =
thriftlog.ConversationEntry(
displayedTweetId = entry.id,
displayType = Some(entry.displayType.toString),
score = candidate.features.getOrElse(ScoreFeature, None)
)
}

14
home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetEntryMarshaller.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala
View File

@ -3,15 +3,13 @@ package com.twitter.home_mixer.marshaller.timeline_logging
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
object PromotedTweetEntryMarshaller {
object PromotedTweetDetailsMarshaller {
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetEntry = {
thriftlog.PromotedTweetEntry(
id = entry.id,
advertiserId = entry.promotedMetadata.map(_.advertiserId).getOrElse(0L),
insertPosition = position,
impressionId = entry.promotedMetadata.flatMap(_.impressionString),
displayType = Some(entry.displayType.toString)
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetDetails = {
thriftlog.PromotedTweetDetails(
advertiserId = Some(entry.promotedMetadata.map(_.advertiserId).getOrElse(0L)),
insertPosition = Some(position),
impressionId = entry.promotedMetadata.flatMap(_.impressionString)
)
}
}

47
home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala Normal file
View File

@ -0,0 +1,47 @@
package com.twitter.home_mixer.marshaller.timeline_logging
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation
import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation
import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ConversationGeneralContextType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext
import com.twitter.timelines.service.{thriftscala => tst}
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
object TweetDetailsMarshaller {
private val generalContextTypeMarshaller = new GeneralContextTypeMarshaller()
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetDetails = {
val socialContext = candidate.presentation.flatMap {
case _ @UrtItemPresentation(timelineItem: TweetItem, _) => timelineItem.socialContext
case _ @UrtModulePresentation(timelineModule) =>
timelineModule.items.head.item match {
case timelineItem: TweetItem => timelineItem.socialContext
case _ => Some(ConversationGeneralContextType)
}
}
val socialContextType = socialContext match {
case Some(GeneralContext(contextType, _, _, _, _)) =>
Some(generalContextTypeMarshaller(contextType).value.toShort)
case Some(TopicContext(_, _)) => Some(tst.ContextType.Topic.value.toShort)
case _ => None
}
thriftlog.TweetDetails(
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
socialContextType = socialContextType,
suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name),
authorId = candidate.features.getOrElse(AuthorIdFeature, None),
sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None)
)
}
}

29
home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetEntryMarshaller.scala
View File

@ -1,29 +0,0 @@
package com.twitter.home_mixer.marshaller.timeline_logging
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.model.HomeFeatures.SocialContextFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.timelines.service.{thriftscala => tst}
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
object TweetEntryMarshaller {
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetEntry = {
val socialContextType = candidate.features.getOrElse(SocialContextFeature, None) match {
case Some(tst.SocialContext.GeneralContext(tst.GeneralContext(contextType, _, _, _, _))) =>
Some(contextType.value.toShort)
case Some(tst.SocialContext.TopicContext(_)) =>
Some(tst.ContextType.Topic.value.toShort)
case _ => None
}
thriftlog.TweetEntry(
id = candidate.candidateIdLong,
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
displayType = Some(entry.displayType.toString),
score = candidate.features.getOrElse(ScoreFeature, None),
socialContextType = socialContextType
)
}
}

10
home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowEntryMarshaller.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala
View File

@ -1,17 +1,13 @@
package com.twitter.home_mixer.marshaller.timeline_logging
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.ScoreFeature
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
object WhoToFollowEntryMarshaller {
object WhoToFollowDetailsMarshaller {
def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowEntry =
thriftlog.WhoToFollowEntry(
userId = entry.id,
displayType = Some(entry.displayType.toString),
score = candidate.features.getOrElse(ScoreFeature, None),
def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowDetails =
thriftlog.WhoToFollowDetails(
enableReactiveBlending = entry.enableReactiveBlending,
impressionId = entry.promotedMetadata.flatMap(_.impressionString),
advertiserId = entry.promotedMetadata.map(_.advertiserId)

5
home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel
View File

@ -4,13 +4,8 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/javax/inject:javax.inject",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"src/thrift/com/twitter/timelineservice:thrift-scala",
"timelineservice/common:model",
],
)

Some files were not shown because too many files have changed in this diff Show More