Compare commits

...

6 Commits

Author SHA1 Message Date
K.Yosuke
ef5f97b16c
Merge 94ba6c0073 into 72eda9a24f 2023-07-17 21:41:53 -05:00
twitter-team
72eda9a24f [opensource] Update home mixer with latest changes 2023-07-13 16:33:04 +05:30
Yuki
94ba6c0073
Merge branch 'twitter:main' into change-readme 2023-04-07 21:08:00 +09:00
Yuki
33507b2652
Updata:ReadMe
Added link to Contributing.md at the end of ReadMe
2023-04-01 12:48:05 +09:00
Yuki
f4d89594b0 Moved Contributing description from ReadMe.md 2023-04-01 11:00:53 +09:00
Yuki
97868d6d2d Moved Contributing description to a separate file. 2023-04-01 11:00:26 +09:00
395 changed files with 9340 additions and 3853 deletions

5
Contributing.md Normal file
View File

@ -0,0 +1,5 @@
# Contributing
We invite the community to submit GitHub issues and pull requests for suggestions on improving the recommendation algorithm. We are working on tools to manage these suggestions and sync changes to our internal repository. Any security concerns or issues should be routed to our official [bug bounty program](https://hackerone.com/twitter) through HackerOne. We hope to benefit from the collective intelligence and expertise of the global community in helping us identify issues and suggest improvements, ultimately leading to a better Twitter.
Read our blog on the open source initiative [here](https://blog.twitter.com/en_us/topics/company/2023/a-new-era-of-transparency-for-twitter).

View File

@ -64,6 +64,4 @@ We include Bazel BUILD files for most components, but not a top-level BUILD or W
## Contributing
We invite the community to submit GitHub issues and pull requests for suggestions on improving the recommendation algorithm. We are working on tools to manage these suggestions and sync changes to our internal repository. Any security concerns or issues should be routed to our official [bug bounty program](https://hackerone.com/twitter) through HackerOne. We hope to benefit from the collective intelligence and expertise of the global community in helping us identify issues and suggest improvements, ultimately leading to a better Twitter.
Read our blog on the open source initiative [here](https://blog.twitter.com/en_us/topics/company/2023/a-new-era-of-transparency-for-twitter).
Please read [this](/Contributing.md)

View File

@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
- Fetch Tweet Candidates
- ScoredTweetsInNetworkCandidatePipelineConfig
- ScoredTweetsCrMixerCandidatePipelineConfig
- ScoredTweetsTweetMixerCandidatePipelineConfig
- ScoredTweetsUtegCandidatePipelineConfig
- ScoredTweetsFrsCandidatePipelineConfig
- Feature Hydration and Scoring
@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer.
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
- ListTweetsAdsCandidatePipelineConfig (fetch ads)

View File

@ -21,6 +21,7 @@ scala_library(
"finatra/inject/inject-utils/src/main/scala",
"home-mixer/server/src/main/resources",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
@ -31,6 +32,10 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"stringcenter/client",
"stringcenter/client/src/main/java",
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",

View File

@ -2,7 +2,7 @@ package com.twitter.home_mixer
import com.twitter.finatra.http.routing.HttpWarmup
import com.twitter.finatra.httpclient.RequestBuilder._
import com.twitter.inject.Logging
import com.twitter.util.logging.Logging
import com.twitter.inject.utils.Handler
import com.twitter.util.Try
import javax.inject.Inject

View File

@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer
import com.twitter.finatra.thrift.filters._
import com.twitter.finatra.thrift.routing.ThriftRouter
import com.twitter.home_mixer.controller.HomeThriftController
import com.twitter.home_mixer.federated.HomeMixerColumn
import com.twitter.home_mixer.module._
import com.twitter.home_mixer.param.GlobalParamConfigModule
import com.twitter.home_mixer.product.HomeMixerProductModule
import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
import com.twitter.product_mixer.component_library.module.CrMixerClientModule
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
import com.twitter.product_mixer.component_library.module.EarlybirdModule
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
import com.twitter.product_mixer.core.controllers.ProductMixerController
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
import com.twitter.product_mixer.core.module.ProductMixerModule
import com.twitter.product_mixer.core.module.StratoClientModule
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.fed.server.StratoFedServer
object HomeMixerServerMain extends HomeMixerServer
class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
class HomeMixerServer
extends StratoFedServer
with ThriftServer
with Mtls
with HttpServer
with HttpMtls {
override val name = "home-mixer-server"
override val modules: Seq[Module] = Seq(
AccountRecommendationsMixerModule,
AdvertiserBrandSafetySettingsStoreModule,
BlenderClientModule,
ClientSentImpressionsPublisherModule,
ConversationServiceModule,
CrMixerClientModule,
EarlybirdModule,
ExploreRankerClientModule,
FeedbackHistoryClientModule,
GizmoduckClientModule,
GlobalParamConfigModule,
HomeAdsCandidateSourceModule,
HomeMixerFlagsModule,
HomeMixerProductModule,
HomeMixerResourcesModule,
HomeNaviModelClientModule,
ImpressionBloomFilterModule,
InjectionHistoryClientModule,
FeedbackHistoryClientModule,
ManhattanClientsModule,
ManhattanFeatureRepositoryModule,
ManhattanTweetImpressionStoreModule,
MemcachedFeatureRepositoryModule,
NaviModelClientModule,
OnboardingTaskServiceModule,
OptimizedStratoClientModule,
PeopleDiscoveryServiceModule,
@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
SimClustersRecentEngagementsClientModule,
SocialGraphServiceModule,
StaleTweetsCacheModule,
StratoClientModule,
ThriftFeatureRepositoryModule,
TimelineMixerClientModule,
TimelineRankerClientModule,
TimelineScorerClientModule,
TimelineServiceClientModule,
TimelinesPersistenceStoreClientModule,
TopicSocialProofClientModule,
TweetImpressionStoreModule,
TweetyPieClientModule,
TweetMixerClientModule,
TweetypieClientModule,
TweetypieStaticEntitiesCacheClientModule,
UserMetadataStoreModule,
UserSessionStoreModule,
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
new ProductScopeStringCenterModule()
)
def configureThrift(router: ThriftRouter): Unit = {
override def configureThrift(router: ThriftRouter): Unit = {
router
.filter[LoggingMDCFilter]
.filter[TraceIdMDCFilter]
@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
this.injector,
st.HomeMixer.ExecutePipeline))
override val dest: String = "/s/home-mixer/home-mixer:strato"
override val columns: Seq[Class[_ <: StratoFed.Column]] =
Seq(classOf[HomeMixerColumn])
override protected def warmup(): Unit = {
handle[HomeMixerThriftServerWarmupHandler]()
handle[HomeMixerHttpServerWarmupHandler]()

View File

@ -3,7 +3,7 @@ package com.twitter.home_mixer
import com.twitter.finagle.thrift.ClientId
import com.twitter.finatra.thrift.routing.ThriftWarmup
import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.inject.Logging
import com.twitter.util.logging.Logging
import com.twitter.inject.utils.Handler
import com.twitter.product_mixer.core.{thriftscala => pt}
import com.twitter.scrooge.Request

View File

@ -5,19 +5,13 @@ scala_library(
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
@ -25,10 +19,6 @@ scala_library(
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
],
exports = [
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",

View File

@ -1,18 +1,23 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
import com.twitter.product_mixer.component_library.filter.FeatureFilter
import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
override val gates: Seq[BaseGate[Query]],
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
extends DependentCandidatePipelineConfig[
@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
val tweetsWithConversationMetadata = candidates.map { candidate =>
TweetWithConversationMetadata(
tweetId = candidate.candidateIdLong,
userId = None,
sourceTweetId = None,
sourceUserId = None,
inReplyToTweetId = None,
userId = candidate.features.getOrElse(AuthorIdFeature, None),
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
conversationId = None,
ancestors = Seq.empty
)
@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
override val preFilterFeatureHydrationPhase1: Seq[
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
] = Seq(
tweetypieFeatureHydrator,
InNetworkFeatureHydrator,
)
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
RetweetDeduplicationFilter,
@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
FilterIdentifier(QuotedTweetDroppedFilterId),
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
),
invalidSubscriptionTweetFilter,
InvalidConversationModuleFilter
)

View File

@ -1,9 +1,9 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
@ -15,7 +15,7 @@ import javax.inject.Singleton
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
namesFeatureHydrator: NamesFeatureHydrator) {
def build(
@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
new ConversationServiceCandidatePipelineConfig(
conversationServiceCandidateSource,
tweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator,
namesFeatureHydrator,
invalidSubscriptionTweetFilter,
gates,
decorator
)

View File

@ -1,7 +1,7 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
import com.twitter.home_mixer.service.HomeMixerAlertConfig

View File

@ -13,8 +13,5 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
"snowflake/src/main/scala/com/twitter/snowflake/id",
"src/thrift/com/twitter/context:twitter-context-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"twitter-context/src/main/scala",
],
)

View File

@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService
import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.controllers.DebugTwitterContext
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
import com.twitter.product_mixer.core.service.urt.UrtService
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch

View File

@ -0,0 +1,24 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"stitch/stitch-repo/src/main/scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
"strato/src/main/scala/com/twitter/strato/callcontext",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
],
)

View File

@ -0,0 +1,217 @@
package com.twitter.home_mixer.federated
import com.twitter.gizmoduck.{thriftscala => gd}
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
import com.twitter.home_mixer.model.request.HomeMixerRequest
import com.twitter.home_mixer.{thriftscala => hm}
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
import com.twitter.product_mixer.core.{thriftscala => pm}
import com.twitter.stitch.Arrow
import com.twitter.stitch.Stitch
import com.twitter.strato.callcontext.CallContext
import com.twitter.strato.catalog.OpMetadata
import com.twitter.strato.config._
import com.twitter.strato.data._
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
import com.twitter.strato.graphql.timelines.{thriftscala => gql}
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.timelines.render.{thriftscala => tr}
import com.twitter.util.Try
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class HomeMixerColumn @Inject() (
homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn,
paramsBuilder: ParamsBuilder,
productPipelineRegistry: ProductPipelineRegistry)
extends StratoFed.Column(HomeMixerColumn.Path)
with StratoFed.Fetch.Arrow {
override val contactInfo: ContactInfo = ContactInfo(
contactEmail = "",
ldapGroup = "",
slackRoomId = ""
)
override val metadata: OpMetadata =
OpMetadata(
lifecycle = Some(Lifecycle.Production),
description =
Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
)
private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
ServiceIdentifierPattern(
role = "",
service = "",
env = "",
zone = Seq(""))
)
override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
override type Key = gql.TimelineKey
override type View = gql.HomeTimelineView
override type Value = tr.Timeline
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
private def createHomeMixerRequestArrow(
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn
): Arrow[(Key, View), hm.HomeMixerRequest] = {
val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
(gd.LookupContext(), Set(gd.QueryFields.Roles))
val populateUserRoles = Arrow
.flatMap[(Key, View), Option[Set[String]]] { _ =>
Stitch.collect {
CallContext.twitterUserId.map { userId =>
compositeOnUserClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch(userId, gizmoduckView).map(_.v)
.map {
_.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
}
}
}
}
val populateIpAddress = Arrow
.flatMap[(Key, View), Option[String]](_ =>
auditIpClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch((), ()).map(_.v))
Arrow.join(
populateUserRoles,
populateIpAddress
)
}
Arrow.zipWithArg(populateUserRolesAndIp).map {
case ((key, view), (roles, ipAddress)) =>
val deviceContextOpt = Some(
hm.DeviceContext(
isPolling = CallContext.isPolling,
requestContext = view.requestContext,
latestControlAvailable = view.latestControlAvailable,
autoplayEnabled = view.autoplayEnabled
))
val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
val (product, productContext) = key match {
case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
(
hm.Product.ForYou,
hm.ProductContext.ForYou(
hm.ForYou(
deviceContextOpt,
seenTweetIds,
view.dspClientContext,
view.pushToHomeTweetId
)
))
case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
(
hm.Product.Following,
hm.ProductContext.Following(
hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
(
hm.Product.Subscribed,
hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
}
val clientContext = pm.ClientContext(
userId = CallContext.twitterUserId,
guestId = CallContext.guestId,
guestIdAds = CallContext.guestIdAds,
guestIdMarketing = CallContext.guestIdMarketing,
appId = CallContext.clientApplicationId,
ipAddress = ipAddress,
userAgent = CallContext.userAgent,
countryCode = CallContext.requestCountryCode,
languageCode = CallContext.requestLanguageCode,
isTwoffice = CallContext.isInternalOrTwoffice,
userRoles = roles,
deviceId = CallContext.deviceId,
mobileDeviceId = CallContext.mobileDeviceId,
mobileDeviceAdId = CallContext.adId,
limitAdTracking = CallContext.limitAdTracking
)
hm.HomeMixerRequest(
clientContext = clientContext,
product = product,
productContext = Some(productContext),
maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
cursor = view.cursor.filter(_.nonEmpty)
)
}
}
override val fetch: Arrow[(Key, View), Result[Value]] = {
val transformThriftIntoPipelineRequest: Arrow[
(Key, View),
ProductPipelineRequest[HomeMixerRequest]
] = {
Arrow
.identity[(Key, View)]
.andThen {
createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
}
.map {
case thriftRequest =>
val request = homeMixerRequestUnmarshaller(thriftRequest)
val params = paramsBuilder.build(
clientContext = request.clientContext,
product = request.product,
featureOverrides =
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
)
ProductPipelineRequest(request, params)
}
}
val underlyingProduct: Arrow[
ProductPipelineRequest[HomeMixerRequest],
ProductPipelineResult[tr.TimelineResponse]
] = Arrow
.identity[ProductPipelineRequest[HomeMixerRequest]]
.map { pipelineRequest =>
val pipelineArrow = productPipelineRegistry
.getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
pipelineRequest.request.product)
.arrow
(pipelineArrow, pipelineRequest)
}.applyArrow
transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
_.result match {
case Some(result) => found(result.timeline)
case _ => missing
}
}
}
}
object HomeMixerColumn {
val Path = "home-mixer/homeMixer.Timeline"
private val FetchCallstack = s"$Path:fetch"
private val MaxCount: Option[Int] = Some(100)
}

View File

@ -10,11 +10,8 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"stitch/stitch-timelineservice/src/main/scala",
"strato/config/columns/recommendations/similarity:similarity-strato-client",
"strato/src/main/scala/com/twitter/strato/client",
],
exports = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",

View File

@ -6,13 +6,11 @@ scala_library(
dependencies = [
"finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
@ -25,8 +23,6 @@ scala_library(
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"stringcenter/client",
"stringcenter/client/src/main/java",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
],
)

View File

@ -1,6 +1,8 @@
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator

View File

@ -8,7 +8,9 @@ scala_library(
"finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
@ -18,6 +20,7 @@ scala_library(
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
],
)

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.finagle.tracing.Trace
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.bijection.Base64String
import com.twitter.bijection.scrooge.BinaryScalaCodec

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient

View File

@ -1,8 +1,10 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
import com.twitter.tweetypie.{thriftscala => tpt}
@ -25,74 +27,76 @@ object HomeTweetTypePredicates {
(
"has_exclusive_conversation_author_id",
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)),
("is_eligible_for_connect_boost", _ => false),
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("get_initial", _.getOrElse(GetInitialFeature, false)),
("get_newer", _.getOrElse(GetNewerFeature, false)),
("get_middle", _.getOrElse(GetMiddleFeature, false)),
("get_older", _.getOrElse(GetOlderFeature, false)),
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
("polling", _.getOrElse(PollingFeature, false)),
("tls_size_20_plus", _ => false),
("near_empty", _ => false),
("ranked_request", _ => false),
("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
(
"less_than_10_mins_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
(
"is_signup_request",
candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("empty_request", _ => false),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
"conversation_module_has_2_displayed_tweets",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)),
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
(
"served_size_between_50_and_100",
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
(
"account_age_less_than_30_minutes",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
(
"account_age_less_than_1_day",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
(
"account_age_less_than_7_days",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
("root_user_is_in_first_degree", _ => false),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
(
"part_of_utt",
_.getOrElse(EarlybirdFeature, None)
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
(
"has_home_latest_request_past_week",
_.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("viewer_is_employee", _ => false),
("viewer_is_timelines_employee", _ => false),
("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
(
"has_ancestor_authored_by_viewer",
candidate =>
@ -100,11 +104,6 @@ object HomeTweetTypePredicates {
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
(
"deep_reply",
candidate =>
@ -119,23 +118,22 @@ object HomeTweetTypePredicates {
"tweet_age_less_than_15_seconds",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 15.seconds)),
("is_followed_topic_tweet", _ => false),
("is_recommended_topic_tweet", _ => false),
("is_topic_tweet", _ => false),
("preferred_language_matches_tweet_language", _ => false),
(
"less_than_1_hour_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)),
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
(
"device_language_matches_tweet_language",
candidate =>
candidate.getOrElse(TweetLanguageFeature, None) ==
candidate.getOrElse(DeviceLanguageFeature, None)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
("in_network", _.getOrElse(FromInNetworkSourceFeature, true)),
("viewer_follows_original_author", _ => false),
("has_account_follow_prompt", _ => false),
("has_relevance_prompt", _ => false),
("has_topic_annotation_haug_prompt", _ => false),
("has_topic_annotation_random_precision_prompt", _ => false),
("has_topic_annotation_prompt", _ => false),
("in_network", _.getOrElse(InNetworkFeature, true)),
(
"has_political_annotation",
_.getOrElse(EarlybirdFeature, None).exists(
@ -153,9 +151,14 @@ object HomeTweetTypePredicates {
_.getOrElse(EarlybirdFeature, None)
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
("is_viewer_not_invited_to_reply", _ => false),
("is_viewer_invited_to_reply", _ => false),
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
(
"is_followed_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == BasicTopicContextFunctionalityType)),
(
"is_recommended_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == RecommendationTopicContextFunctionalityType)),
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
(
@ -164,8 +167,6 @@ object HomeTweetTypePredicates {
(
"has_gte_100k_favs",
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
("above_neighbor_is_topic_tweet", _ => false),
("is_topic_tweet_with_neighbor_below", _ => false),
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
(
@ -187,6 +188,7 @@ object HomeTweetTypePredicates {
(
"has_toxicity_score_above_threshold",
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
(
"text_only",
candidate =>
@ -204,23 +206,50 @@ object HomeTweetTypePredicates {
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
("3_or_more_consecutive_not_in_network", _ => false),
("2_or_more_consecutive_not_in_network", _ => false),
("5_out_of_7_not_in_network", _ => false),
("7_out_of_7_not_in_network", _ => false),
("5_out_of_5_not_in_network", _ => false),
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
("has_liked_by_social_context", _ => false),
("has_followed_by_social_context", _ => false),
("has_topic_social_context", _ => false),
("timeline_entry_has_banner", _ => false),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
(
"conversation_module_has_2_displayed_tweets",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
("served_in_recap_tweet_candidate_module_injection", _ => false),
("served_in_threaded_conversation_module", _ => false)
"has_liked_by_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
.exists(candidateFeatures
.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
(
"has_followed_by_social_context",
_.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
(
"has_topic_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(TopicIdSocialContextFeature, None)
.isDefined &&
candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
(
"video_bt_10_60_sec",
_.getOrElse(VideoDurationMsFeature, None).exists(duration =>
duration > 10000 && duration <= 60000)),
("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
(
"tweet_age_lte_30_minutes",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 30.minutes)),
(
"tweet_age_lte_1_hour",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 1.hour)),
(
"tweet_age_lte_6_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 6.hours)),
(
"tweet_age_lte_12_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 12.hours)),
(
"tweet_age_gte_24_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow >= 24.hours)),
)
val PredicateMap = CandidatePredicates.toMap

View File

@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelineservice.suggests.{thriftscala => st}
object ListClientEventDetailsBuilder
case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
override def apply(
@ -20,7 +20,7 @@ object ListClientEventDetailsBuilder
conversationDetails = None,
timelinesDetails = Some(
TimelinesDetails(
injectionType = Some(st.SuggestType.OrganicListTweet.name),
injectionType = Some(suggestType.name),
controllerData = None,
sourceData = None)),
articleDetails = None,

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -9,7 +9,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -4,9 +4,16 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
"src/thrift/com/twitter/timelines/service:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
],
)

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -17,7 +17,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tls}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

View File

@ -0,0 +1,18 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
class FeedbackStrings @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
private val externalStringRegistry = externalStringRegistryProvider.get()
val seeLessOftenFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOften")
val seeLessOftenConfirmationFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.request.FollowingProduct
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => t}
import com.twitter.timelines.util.FeedbackMetadataSerializer
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
@ -14,10 +14,13 @@ import javax.inject.Singleton
@Singleton
case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder: LikedBySocialContextBuilder,
listsSocialContextBuilder: ListsSocialContextBuilder,
followedBySocialContextBuilder: FollowedBySocialContextBuilder,
topicSocialContextBuilder: TopicSocialContextBuilder,
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder,
popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
def apply(
@ -31,6 +34,9 @@ case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder(query, candidate, features)
.orElse(followedBySocialContextBuilder(query, candidate, features))
.orElse(topicSocialContextBuilder(query, candidate, features))
.orElse(popularVideoSocialContextBuilder(query, candidate, features))
.orElse(listsSocialContextBuilder(query, candidate, features))
.orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
case Some(_) =>
val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
// Only hydrate the social context into the root tweet in a conversation module

View File

@ -5,7 +5,6 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
@ -32,12 +31,13 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
@Singleton
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
externalStringRegistry = externalStringRegistryProvider.get(),
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)

View File

@ -0,0 +1,52 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => tl}
import com.twitter.timelines.util.FeedbackRequestSerializer
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.timelineservice.thriftscala.FeedbackType
object HomeWhoToSubscribeFeedbackActionInfoBuilder {
private val FeedbackMetadata = tl.FeedbackMetadata(
injectionType = Some(SuggestType.WhoToSubscribe),
engagementType = None,
entityIds = Seq.empty,
ttlMs = None
)
private val FeedbackRequest =
tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
private val EncodedFeedbackRequest =
FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
}
@Singleton
case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest =
Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)
override def apply(
query: PipelineQuery,
candidate: UserCandidate,
candidateFeatures: FeatureMap
): Option[FeedbackActionInfo] =
whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature

View File

@ -0,0 +1,50 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* "Your Lists" will be rendered for the context and a url link for your lists.
*/
@Singleton
case class ListsSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
candidateFeatures.get(SuggestTypeFeature) match {
case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
Some(
GeneralContext(
contextType = ListGeneralContextType,
text = stringCenter.prepare(listString),
url = userName.map(name => ""),
contextImageUrls = None,
landingUrl = None
))
case _ => None
}
}
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
@ -12,7 +12,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tlst}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -0,0 +1,43 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
case class PopularInYourAreaSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
Some(
GeneralContext(
contextType = LocationGeneralContextType,
text = stringCenter.prepare(popularInYourAreaString),
url = None,
contextImageUrls = None,
landingUrl = None
))
} else None
}
}

View File

@ -1,50 +1,48 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* Renders a fixed 'You Might Like' string above all OON Tweets.
*/
@Singleton
case class YouMightLikeSocialContextBuilder @Inject() (
case class PopularVideoSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val youMightLikeString = externalStrings.socialContextYouMightLikeString
private val popularVideoString = externalStrings.socialContextPopularVideoString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true)
val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false)
if (!isInNetwork && !isRetweet) {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) {
Some(
GeneralContext(
contextType = SparkleGeneralContextType,
text = stringCenter.prepare(youMightLikeString),
text = stringCenter.prepare(popularVideoString),
url = None,
contextImageUrls = None,
landingUrl = None
landingUrl = Some(
Url(
urlType = DeepLink,
url = ""
)
)
))
} else {
None
}
} else None
}
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -8,7 +8,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -4,95 +4,56 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider",
"finatra/inject/inject-core/src/main/scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
"representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
"representation-scorer/server/src/main/thrift:thrift-scala",
"servo/repo/src/main/scala",
"snowflake/src/main/scala/com/twitter/snowflake/id",
"src/java/com/twitter/ml/api/constant",
"src/java/com/twitter/search/common/util/lang",
"src/scala/com/twitter/ml/api/util",
"src/scala/com/twitter/timelines/prediction/adapters/real_graph",
"src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/adapters/twistly",
"src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
"src/scala/com/twitter/timelines/prediction/common/util",
"src/scala/com/twitter/timelines/prediction/features/common",
"src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/features/recap",
"src/scala/com/twitter/timelines/prediction/features/time_features",
"src/scala/com/twitter/timelines/prediction/adapters/request_context",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/ml/api:data-java",
"src/thrift/com/twitter/ml/api:embedding-java",
"src/thrift/com/twitter/onboarding/relevance/features:features-java",
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"src/thrift/com/twitter/search/common:constants-java",
"src/thrift/com/twitter/socialgraph:thrift-scala",
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
"src/thrift/com/twitter/timelineranker:thrift-scala",
"src/thrift/com/twitter/timelines/author_features:thrift-java",
"src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
"src/thrift/com/twitter/timelines/impression:thrift-scala",
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
"src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala",
"src/thrift/com/twitter/user_session_store:thrift-java",
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
"src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
"stitch/stitch-core",
"stitch/stitch-gizmoduck",
"stitch/stitch-socialgraph",
"stitch/stitch-timelineservice",
"stitch/stitch-tweetypie",
"strato/config/columns/topic-signals/tsp",
"strato/config/columns/topic-signals/tsp:tsp-strato-client",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
"timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly",
"timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"topic-social-proof/server/src/main/thrift:thrift-scala",
"topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
"tweetconvosvc/thrift/src/main/thrift:thrift-scala",
"twitter-config/yaml",
"user_session_store/src/main/scala/com/twitter/user_session_store",
"util/util-core",
],

View File

@ -1,12 +1,10 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
@ -17,16 +15,12 @@ import javax.inject.Singleton
@Singleton
case class FeedbackHistoryQueryFeatureHydrator @Inject() (
feedbackHistoryClient: FeedbackHistoryManhattanClient)
extends QueryFeatureHydrator[PipelineQuery]
with Conditionally[PipelineQuery] {
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableFeedbackFatigueParam)
override def hydrate(
query: PipelineQuery
): Stitch[FeatureMap] =

View File

@ -1,41 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class FollowedTopicsQueryFeatureHydrator @Inject() (
followedTopicsCandidateSource: FollowedTopicsCandidateSource)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
followedTopicsCandidateSource(request)
.map { topics =>
FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
}.handle {
case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
}
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
)
}

View File

@ -1,58 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.gizmoduck.{thriftscala => gt}
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.stitch.gizmoduck.Gizmoduck
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("GizmoduckAuthorSafety")
override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
val blueVerifiedStitch = authorIdOption
.map { authorId =>
gizmoduck
.getUserById(
userId = authorId,
queryFields = queryFields
)
.map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
}.getOrElse(Stitch.False)
blueVerifiedStitch.map { isBlueVerified =>
FeatureMapBuilder()
.add(AuthorIsBlueVerifiedFeature, isBlueVerified)
.build()
}
}
}

View File

@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -11,7 +12,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
import javax.inject.Inject
import javax.inject.Singleton
@ -19,32 +21,35 @@ import javax.inject.Singleton
@Singleton
case class ImpressionBloomFilterQueryFeatureHydrator[
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
bloomFilter: ImpressionBloomFilter)
extends QueryFeatureHydrator[Query] {
bloomFilterClient: ManhattanStoreClient[
blm.ImpressionBloomFilterKey,
blm.ImpressionBloomFilterSeq
]) extends QueryFeatureHydrator[Query] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
"ImpressionBloomFilter")
private val ImpressionBloomFilterTTL = 7.day
private val ImpressionBloomFilterFalsePositiveRate = 0.002
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
private val SurfaceArea = t.SurfaceArea.HomeTimeline
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
override def hydrate(query: Query): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq =>
bloomFilterClient
.get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
.map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
.map { bloomFilterSeq =>
val updatedBloomFilterSeq =
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
else {
bloomFilter.addElements(
userId = userId,
ImpressionBloomFilter.addSeenTweetIds(
surfaceArea = SurfaceArea,
tweetIds = query.seenTweetIds.get,
bloomFilterEntrySeq = bloomFilterSeq,
bloomFilterSeq = bloomFilterSeq,
timeToLive = ImpressionBloomFilterTTL,
falsePositiveRate = ImpressionBloomFilterFalsePositiveRate
falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam)
)
}
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()

View File

@ -0,0 +1,41 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object InNetworkFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
val featureMaps = candidates.map { candidate =>
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
isSelfTweet || followedUserIds.contains(authorId)
}
FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
}
Stitch.value(featureMaps)
}
}

View File

@ -2,8 +2,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
@ -27,17 +29,27 @@ import javax.inject.Singleton
@Singleton
case class PersistenceStoreQueryFeatureHydrator @Inject() (
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3],
statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
private val WhoToFollowExcludedUserIdsLimit = 1000
private val ServedTweetIdsDuration = 1.hour
private val ServedTweetIdsDuration = 10.minutes
private val ServedTweetIdsLimit = 100
private val ServedTweetPreviewIdsDuration = 10.hours
private val ServedTweetPreviewIdsLimit = 10
override val features: Set[Feature[_, _]] =
Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature)
Set(
ServedTweetIdsFeature,
ServedTweetPreviewIdsFeature,
PersistenceEntriesFeature,
WhoToFollowExcludedUserIdsFeature)
private val supportedClients = Seq(
ClientPlatform.IPhone,
@ -80,8 +92,19 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
.flatMap(
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
servedTweetIdsSizeStat.add(servedTweetIds.size)
val servedTweetPreviewIds = timelineResponses
.filter(_.clientPlatform == clientPlatform)
.filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
.sortBy(-_.servedTime.inMilliseconds)
.flatMap(_.entries
.filter(_.entityIdType == EntityIdType.TweetPreview)
.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
FeatureMapBuilder()
.add(ServedTweetIdsFeature, servedTweetIds)
.add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
.add(PersistenceEntriesFeature, timelineResponses)
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
.build()

View File

@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.stitch.timelineservice.TimelineService
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
@ -37,10 +38,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
candidate.features
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.map(favoritedBy => favoritedBy -> candidate.candidate.id)
}
@ -59,7 +60,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
candidates.map { candidate =>
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
FeatureMapBuilder()

View File

@ -32,11 +32,15 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
val realGraphScoresFeatures = realGraphFollowedUsers
.getOrElse(Seq.empty)
.sortBy(-_.score)
.map(candidate => candidate.userId -> candidate.score)
.map(candidate => candidate.userId -> scaleScore(candidate.score))
.take(RealGraphCandidateCount)
.toMap
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
}
}
// Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
private def scaleScore(score: Double): Double =
if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
}

View File

@ -17,9 +17,13 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
import com.twitter.search.common.util.lang.ThriftLanguageUtil
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
import java.util.UUID
import javax.inject.Inject
import javax.inject.Singleton
@ -45,6 +49,9 @@ class RequestQueryFeatureHydrator[
PullToRefreshFeature,
RequestJoinIdFeature,
ServedRequestIdFeature,
TimestampFeature,
TimestampGMTDowFeature,
TimestampGMTHourFeature,
ViewerIdFeature
)
@ -67,6 +74,7 @@ class RequestQueryFeatureHydrator[
override def hydrate(query: Query): Stitch[FeatureMap] = {
val requestContext = query.deviceContext.flatMap(_.requestContextValue)
val servedRequestId = UUID.randomUUID.getMostSignificantBits
val timestamp = query.queryTime.inMilliseconds
val featureMap = FeatureMapBuilder()
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
@ -97,8 +105,15 @@ class RequestQueryFeatureHydrator[
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
.add(ServedRequestIdFeature, Some(servedRequestId))
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
.add(TimestampFeature, timestamp)
.add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
.add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
.add(HasDarkRequestFeature, hasDarkRequest)
.add(ViewerIdFeature, query.getRequiredUserId)
.add(
ViewerIdFeature,
query.getOptionalUserId
.orElse(query.getGuestId).getOrElse(
throw PipelineFailure(BadRequest, "Missing viewer id")))
.build()
Stitch.value(featureMap)

View File

@ -1,46 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton
case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
socialGraphStitchClient: SocialGraphStitchClient)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SGSFollowedUsers")
override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
private val SocialGraphLimit = 14999
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
val request = sg.IdsRequest(
relationships = Seq(
sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
),
pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
)
socialGraphStitchClient
.ids(request).map(_.ids)
.map { followedUsers =>
FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
}
}
}

View File

@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph
@ -41,8 +42,7 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
val allSocialContextUserIds =
candidates.flatMap { candidate =>
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++

View File

@ -1,67 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SocialGraphService")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
val idsRequest = createIdsRequest(
userId = viewerId,
relationshipTypes = Set(sg.RelationshipType.Following),
targetIds = Some(distinctNonSelfAuthorIds)
)
socialGraphStitchClient
.ids(request = idsRequest, requestContext = None)
.map { idResult =>
authorIds.map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
val inNetworkAuthorIds = idResult.ids.toSet
val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
}
}
}
private def createIdsRequest(
userId: Long,
relationshipTypes: Set[sg.RelationshipType],
targetIds: Option[Seq[Long]] = None
): sg.IdsRequest = sg.IdsRequest(
relationshipTypes.map { relationshipType =>
sg.SrcRelationship(userId, relationshipType, targets = targetIds)
}.toSeq,
Some(sg.PageRequest(selectAll = Some(true)))
)
}

View File

@ -1,162 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.contentrecommender.{thriftscala => cr}
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
import com.twitter.topiclisting.TopicListingViewerContext
import com.twitter.tsp.{thriftscala => tsp}
import javax.inject.Inject
import javax.inject.Singleton
import scala.collection.JavaConverters._
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
object TSPInferredTopicDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TSPInferredTopicFeatureHydrator @Inject() (
topicSocialProofClientColumn: TopicSocialProofClientColumn,
statsReceiver: StatsReceiver,
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
override val features: Set[Feature[_, _]] =
Set(
TSPInferredTopicFeature,
TSPInferredTopicDataRecordFeature,
TopicIdSocialContextFeature,
TopicContextFunctionalityTypeFeature)
private val topK = 3
private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
sid.CandidateTweetSourceId.Simcluster,
sid.CandidateTweetSourceId.CroonTweet
)
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
private val DefaultFeatureMap = FeatureMapBuilder()
.add(TSPInferredTopicFeature, Map.empty[Long, Double])
.add(TSPInferredTopicDataRecordFeature, new DataRecord())
.add(TopicIdSocialContextFeature, None)
.add(TopicContextFunctionalityTypeFeature, None)
.build()
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val tags = candidates.collect {
case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
candidate.candidate.id -> candidate.features
.getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
}.toMap
val topicSocialProofRequest =
tsp.TopicSocialProofRequest(
userId = query.getRequiredUserId,
tweetIds = candidates.map(_.candidate.id).toSet,
displayLocation = cr.DisplayLocation.HomeTimeline,
topicListingSetting = tsp.TopicListingSetting.Followable,
context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
bypassModes = None,
// Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
tags = if (tags.isEmpty) None else Some(tags)
)
topicSocialProofClientColumn.fetcher
.fetch(topicSocialProofRequest)
.map(_.v)
.map {
case Some(response) =>
candidates.map { candidate =>
val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
if (topicWithScores.nonEmpty) {
keyFoundCounter.incr()
val (socialProofId, socialProofFunctionalityType) =
if (candidate.features
.getOrElse(CandidateSourceIdFeature, None)
.exists(sourcesToSetSocialProof.contains)) {
getSocialProof(topicWithScores)
} else {
(None, None)
}
val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
val inferredTopicDataRecord =
InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
FeatureMapBuilder()
.add(TSPInferredTopicFeature, inferredTopicFeatures)
.add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
.add(TopicIdSocialContextFeature, socialProofId)
.add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
.build()
} else {
keyLossCounter.incr()
DefaultFeatureMap
}
}
case _ =>
requestFailCounter.incr()
candidates.map { _ =>
DefaultFeatureMap
}
}
}
private def getSocialProof(
topicWithScores: Seq[tsp.TopicWithScore]
): (Option[Long], Option[TopicContextFunctionalityType]) = {
val followingTopicId = topicWithScores
.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
topicId
}
if (followingTopicId.nonEmpty) {
return (followingTopicId, Some(BasicTopicContextFunctionalityType))
}
val implicitFollowingId = topicWithScores.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
topicId
}
if (implicitFollowingId.nonEmpty) {
return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
}
(None, None)
}
private def convertTopicWithScores(
topicWithScores: Seq[tsp.TopicWithScore],
): Map[Long, Double] = {
topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
}
}

View File

@ -1,251 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.FDsl._
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.search.common.features.{thriftscala => sc}
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
import com.twitter.timelines.prediction.features.time_features.TimeFeatures
import com.twitter.util.Duration
import scala.collection.Searching._
object TimeFeaturesDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
Stitch.value {
val richDataRecord = new RichDataRecord()
setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
FeatureMapBuilder()
.add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
.build()
}
}
private def setTimeFeatures(
richDataRecord: RichDataRecord,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
query: PipelineQuery,
): Unit = {
val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
}
private[feature_hydrator] def getTimeFeatures(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
): Option[TimeFeatures] = {
for {
requestTimestampMs <- Some(query.queryTime.inMilliseconds)
tweetId <- Some(candidate.id)
viewerId <- query.getOptionalUserId
tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
}
timeSinceSourceTweetCreation =
existingFeatures
.getOrElse(SourceTweetIdFeature, None)
.flatMap { sourceTweetId =>
timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
requestTimestampMs - sourceTweetCreationTimeMs
}
}
.getOrElse(timeSinceTweetCreation)
if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
} yield {
val timeFeatures = TimeFeatures(
timeSinceTweetCreation = timeSinceTweetCreation,
timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
)
val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
existingFeatures.getOrElse(EarlybirdFeature, None),
timeFeatures,
timeSinceSourceTweetCreation
).getOrElse(timeFeatures)
val nonPollingTimestampsMs =
query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
timeFeaturesWithLastEngagement,
requestTimestampMs,
tweetCreationTimeMs,
nonPollingTimestampsMs
)
timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
}
}
private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
if (SnowflakeId.isSnowflakeId(tweetOrUserId))
Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
else None
}
private def addLastEngagementTimeFeatures(
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
timeFeatures: TimeFeatures,
timeSinceSourceTweetCreation: Long
): Option[TimeFeatures] = {
tweetFeaturesOpt.map { tweetFeatures =>
val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
timeFeatures.copy(
lastFavSinceCreationHrs = lastFavSinceCreationHrs,
lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
lastReplySinceCreationHrs = lastReplySinceCreationHrs,
lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
lastFavSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
lastRetweetSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
lastReplySinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
lastQuoteSinceCreationHrs,
timeSinceSourceTweetCreation
)
)
}
}
private def addNonPollingTimeFeatures(
timeFeatures: TimeFeatures,
requestTimestampMs: Long,
creationTimeMs: Long,
nonPollingTimestampsMs: Option[Seq[Long]]
): Option[TimeFeatures] = {
for {
nonPollingTimestampsMs <- nonPollingTimestampsMs
lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
} yield {
val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
1.0,
timeFeatures.timeSinceTweetCreation
)
/*
* Non-polling timestamps are stored in chronological order.
* The latest timestamps occur first, therefore we need to explicitly search in reverse order.
*/
val nonPollingRequestsSinceTweetCreation =
if (nonPollingTimestampsMs.nonEmpty) {
nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
} else {
0
}
/*
* Calculate the average time between non-polling requests; include
* request time in this calculation as latest timestamp.
*/
val timeBetweenNonPollingRequestsAvg =
(requestTimestampMs - earliestNonPollingTimestampMs) / math
.max(1.0, nonPollingTimestampsMs.size)
val timeFeaturesWithNonPolling = timeFeatures.copy(
timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
tweetAgeRatio = Some(tweetAgeRatio)
)
timeFeaturesWithNonPolling
}
}
private[this] def getTimeSinceLastEngagementHrs(
lastEngagementTimeSinceCreationHrsOpt: Option[Double],
timeSinceTweetCreation: Long
): Option[Double] = {
lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
}
}
private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
val record = richDataRecord.getRecord
.setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
.setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
.setFeatureValueFromOption(
TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
features.timeSinceViewerAccountCreationSecs)
.setFeatureValue(
USER_ID_IS_SNOWFLAKE_ID,
features.timeSinceViewerAccountCreationSecs.isDefined
)
.setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
.setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
.setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
.setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
.setFeatureValueFromOption(
LAST_RETWEET_SINCE_CREATION_HRS,
features.lastRetweetSinceCreationHrs
)
.setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
.setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
/*
* set features whose values are optional as some users do not have non-polling timestamps
*/
features.timeBetweenNonPollingRequestsAvg.foreach(
record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
)
features.timeSinceLastNonPollingRequest.foreach(
record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
)
features.nonPollingRequestsSinceTweetCreation.foreach(
record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
)
features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
}
}

View File

@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
extends QueryFeatureHydrator[Query] {
private val TweetImpressionTTL = 1.day
private val TweetImpressionCap = 3000
private val TweetImpressionTTL = 2.days
private val TweetImpressionCap = 5000
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")

View File

@ -1,6 +1,9 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
@ -10,11 +13,13 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.util.tweetypie.RequestFields
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
@ -29,17 +34,22 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
import com.twitter.stitch.Stitch
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
import com.twitter.tweetypie.{thriftscala => tp}
import com.twitter.util.logging.Logging
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
class TweetypieFeatureHydrator @Inject() (
tweetypieStitchClient: TweetypieStitchClient,
statsReceiver: StatsReceiver)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Logging {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
override val features: Set[Feature[_, _]] = Set(
AuthorIdFeature,
ExclusiveConversationAuthorIdFeature,
InReplyToTweetIdFeature,
IsHydratedFeature,
IsNsfw,
@ -51,6 +61,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
SourceTweetIdFeature,
SourceUserIdFeature,
TweetTextFeature,
TweetLanguageFeature,
VisibilityReason
)
@ -70,9 +81,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
): Stitch[FeatureMap] = {
val safetyLevel = query.product match {
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
case ForYouProduct => rtf.SafetyLevel.TimelineHome
case ForYouProduct =>
val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
}
@ -82,9 +96,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
includeQuotedTweet = true,
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
safetyLevel = Some(safetyLevel),
forUserId = Some(query.getRequiredUserId)
forUserId = query.getOptionalUserId
)
val exclusiveAuthorIdOpt =
existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
val coreData = found.tweet.coreData
@ -106,6 +123,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
val tweetText = coreData.map(_.text)
val tweetLanguage = found.tweet.language.map(_.language)
val tweetAuthorId = coreData.map(_.userId)
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
@ -127,6 +145,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
FeatureMapBuilder()
.add(AuthorIdFeature, tweetAuthorId)
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, inReplyToTweetId)
.add(IsHydratedFeature, true)
.add(IsNsfw, Some(isNsfw))
@ -137,20 +156,24 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
.add(QuotedUserIdFeature, quotedTweetUserId)
.add(SourceTweetIdFeature, retweetedTweetId)
.add(SourceUserIdFeature, retweetedTweetUserId)
.add(TweetLanguageFeature, tweetLanguage)
.add(TweetTextFeature, tweetText)
.add(VisibilityReason, found.suppressReason)
.build()
// If no tweet result found, return default and pre-existing features
case _ =>
DefaultFeatureMap +
(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) +
(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) +
(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) +
(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) +
(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) +
(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) +
(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
DefaultFeatureMap ++ FeatureMapBuilder()
.add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
.add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
.add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
.add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
.add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
.add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
.add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
.build()
}
}
}

View File

@ -1,59 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
import com.twitter.ml.api.DataRecordMerger
import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.CompactDataRecordConverter
import com.twitter.ml.api.util.FDsl._
import com.twitter.timelines.author_features.v1.{thriftjava => af}
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
private val originalAuthorAggregatesFeatures =
TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
.buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
private val authorFeatures = originalAuthorAggregatesFeatures ++
Seq(
UserHealthFeatures.AuthorState,
UserHealthFeatures.NumAuthorFollowers,
UserHealthFeatures.NumAuthorConnectDays,
UserHealthFeatures.NumAuthorConnect)
private val featureContext = new FeatureContext(authorFeatures: _*)
override def getFeatureContext: FeatureContext = featureContext
override val commonFeatures: Set[Feature[_]] = Set.empty
private val compactDataRecordConverter = new CompactDataRecordConverter()
private val drMerger = new DataRecordMerger()
override def setFeatures(
authorFeaturesOpt: Option[af.AuthorFeatures],
richDataRecord: RichDataRecord
): Unit = {
authorFeaturesOpt.foreach { authorFeatures =>
val dataRecord = richDataRecord.getRecord
dataRecord.setFeatureValue(
UserHealthFeatures.AuthorState,
authorFeatures.user_health.user_state.getValue.toLong)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorFollowers,
authorFeatures.user_health.num_followers.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnectDays,
authorFeatures.user_health.num_connect_days.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnect,
authorFeatures.user_health.num_connect.toDouble)
val originalAuthorAggregatesDataRecord =
compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
}
}
}

View File

@ -1,22 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates
import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("Phase2EdgeAggregate")
override val aggregateFeatures: Set[BaseEdgeAggregateFeature] =
Set(
UserEngagerAggregateFeature,
UserEngagerGoodClickAggregateFeature,
UserInferredTopicAggregateFeature,
UserTopicAggregateFeature,
UserMediaUnderstandingAnnotationAggregateFeature
)
}

View File

@ -4,6 +4,7 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
@ -16,9 +17,11 @@ scala_library(
"src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala",
"stitch/stitch-core",
"stitch/stitch-socialgraph",
"stitch/stitch-tweetypie",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"util/util-slf4j-api/src/main/scala",
],
exports = [
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",

View File

@ -72,7 +72,7 @@ object FeedbackFatigueFilter
originalAuthorId.exists(authorsToFilter.contains) ||
(likers.nonEmpty && eligibleLikers.isEmpty) ||
(followers.nonEmpty && eligibleFollowers.isEmpty) ||
(followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) ||
(candidate.features.getOrElse(IsRetweetFeature, false) &&
authorId.exists(retweetersToFilter.contains))
}

View File

@ -0,0 +1,70 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.finagle.tracing.Trace
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph
import com.twitter.util.logging.Logging
import javax.inject.Inject
import javax.inject.Singleton
/**
* Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author
*
* If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for
* subscription tweets, so we explicitly filter those tweets out.
*/
@Singleton
case class InvalidSubscriptionTweetFilter @Inject() (
socialGraphClient: SocialGraph,
statsReceiver: StatsReceiver)
extends Filter[PipelineQuery, TweetCandidate]
with Logging {
override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet")
private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet")
private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = Stitch
.traverse(candidates) { candidate =>
val exclusiveAuthorId =
candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None)
if (exclusiveAuthorId.isDefined) {
val request = sg.ExistsRequest(
source = query.getRequiredUserId,
target = exclusiveAuthorId.get,
relationships =
Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)),
)
socialGraphClient.exists(request).map(_.exists).map { valid =>
if (!valid) invalidCounter.incr() else validCounter.incr()
valid
}
} else Stitch.value(true)
}.map { validResults =>
val (kept, removed) = candidates
.map(_.candidate)
.zip(validResults)
.partition { case (candidate, valid) => valid }
val keptCandidates = kept.map { case (candidate, _) => candidate }
val removedCandidates = removed.map { case (candidate, _) => candidate }
FilterResult(kept = keptCandidates, removed = removedCandidates)
}
}

View File

@ -1,59 +0,0 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
/**
* Predicate which will be applied to each candidate. True indicates that the candidate will be
* @tparam Candidate - the type of the candidate
*/
trait ShouldKeepCandidate {
def apply(features: FeatureMap): Boolean
}
object PredicateFeatureFilter {
/**
* Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity,
* we recommend including the name of the shouldKeepCandidate parameter.
*
* @param identifier A FilterIdentifier for the new filter
* @param shouldKeepCandidate A predicate function. Candidates will be kept when
* this function returns True.
*/
def fromPredicate[Candidate <: UniversalNoun[Any]](
identifier: FilterIdentifier,
shouldKeepCandidate: ShouldKeepCandidate
): Filter[PipelineQuery, Candidate] = {
val i = identifier
new Filter[PipelineQuery, Candidate] {
override val identifier: FilterIdentifier = i
/**
* Filter the list of candidates
*
* @return a FilterResult including both the list of kept candidate and the list of removed candidates
*/
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]] = {
val allowedIds = candidates
.filter(candidate => shouldKeepCandidate(candidate.features)).map(_.candidate.id).toSet
val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition {
candidate => allowedIds.contains(candidate.id)
}
Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
}
}
}
}

View File

@ -1,8 +1,6 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
@ -11,24 +9,20 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] {
object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("KeepBestOutOfNetworkTweetPerAuthor")
override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
// Set containing best OON tweet for each authorId
val bestCandidatesForAuthorId = candidates
.filter(!_.features.getOrElse(InNetworkFeature, true))
.groupBy(_.features.getOrElse(AuthorIdFeature, None))
.values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None)))
.toSet
val servedTweetPreviewIds =
query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet
val (removed, kept) = candidates.partition { candidate =>
!candidate.features.getOrElse(InNetworkFeature, true) &&
!bestCandidatesForAuthorId.contains(candidate)
servedTweetPreviewIds.contains(candidate.candidate.id)
}
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))

View File

@ -0,0 +1,32 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("Reply")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
val (kept, removed) = candidates
.partition { candidate =>
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
}
val filterResult = FilterResult(
kept = kept.map(_.candidate),
removed = removed.map(_.candidate)
)
Stitch.value(filterResult)
}
}

View File

@ -0,0 +1,32 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("Retweet")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
val (kept, removed) = candidates
.partition { candidate =>
!candidate.features.getOrElse(IsRetweetFeature, false)
}
val filterResult = FilterResult(
kept = kept.map(_.candidate),
removed = removed.map(_.candidate)
)
Stitch.value(filterResult)
}
}

View File

@ -14,7 +14,6 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"stitch/stitch-socialgraph",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
],

View File

@ -1,14 +1,14 @@
package com.twitter.home_mixer.functional_component.gate
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.functional_component.gate.Gate
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
import com.twitter.stitch.Stitch
import com.twitter.util.Duration
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.util.Duration
object DismissFatigueGate {
// how long a dismiss action from user needs to be respected

View File

@ -1,18 +0,0 @@
package com.twitter.home_mixer.functional_component.gate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.functional_component.gate.Gate
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import scala.reflect.runtime.universe._
case class NonEmptySeqFeatureGate[T: TypeTag](
feature: Feature[PipelineQuery, Seq[T]])
extends Gate[PipelineQuery] {
override val identifier: GateIdentifier = GateIdentifier(s"NonEmptySeq$feature")
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
Stitch.value(query.features.exists(_.get(feature).nonEmpty))
}

View File

@ -19,7 +19,7 @@ object EditedTweetsCandidatePipelineQueryTransformer
override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
// The time window for which a tweet remains editable after creation.
private val EditTimeWindow = 30.minutes
private val EditTimeWindow = 60.minutes
override def transform(query: PipelineQuery): Seq[Long] = {
val applicableCandidates = getApplicableCandidates(query)
@ -29,8 +29,8 @@ object EditedTweetsCandidatePipelineQueryTransformer
// Any tweets in it could have become stale since being served.
val previousTimelineLoadTime = applicableCandidates.head.servedTime
// The time window for editing a tweet is 30 minutes,
// so we ignore responses older than (PTL Time - 30 mins).
// The time window for editing a tweet is 60 minutes,
// so we ignore responses older than (PTL Time - 60 mins).
val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
.takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)

View File

@ -6,7 +6,6 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",

View File

@ -35,8 +35,8 @@ object FeedbackFatigueScorer
override def onlyIf(query: PipelineQuery): Boolean =
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
private val DurationForFiltering = 14.days
private val DurationForDiscounting = 140.days
val DurationForFiltering = 14.days
val DurationForDiscounting = 140.days
private val ScoreMultiplierLowerBound = 0.2
private val ScoreMultiplierUpperBound = 1.0
private val ScoreMultiplierIncrementsCount = 4
@ -76,8 +76,27 @@ object FeedbackFatigueScorer
feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty))
val featureMaps = candidates.map { candidate =>
val multiplier = getScoreMultiplier(
candidate,
authorsToDiscount,
likersToDiscount,
followersToDiscount,
retweetersToDiscount
)
val score = candidate.features.getOrElse(ScoreFeature, None)
FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
}
Stitch.value(featureMaps)
}
def getScoreMultiplier(
candidate: CandidateWithFeatures[TweetCandidate],
authorsToDiscount: Map[Long, Double],
likersToDiscount: Map[Long, Double],
followersToDiscount: Map[Long, Double],
retweetersToDiscount: Map[Long, Double],
): Double = {
val originalAuthorId =
CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
@ -92,7 +111,8 @@ object FeedbackFatigueScorer
val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
val followerMultipliers = followers.flatMap(followersToDiscount.get)
val followerMultiplier =
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size)
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size &&
likers.isEmpty)
followerMultipliers.max
else 1.0
@ -102,16 +122,10 @@ object FeedbackFatigueScorer
retweetersToDiscount.getOrElse(authorId, 1.0)
else 1.0
val multiplier =
originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
}
Stitch.value(featureMaps)
}
private def getUserDiscounts(
def getUserDiscounts(
queryTime: Time,
feedbackEntries: Seq[FeedbackEntry],
): Map[Long, Double] = {

View File

@ -1,61 +0,0 @@
package com.twitter.home_mixer.functional_component.scorer
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorInNetworkMultiplierParam
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorOutOfNetworkMultiplierParam
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
/**
* Scales scores of tweets whose author is Blue Verified by the provided scale factor
*/
object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate] {
override val identifier: ScorerIdentifier = ScorerIdentifier("VerifiedAuthorScaling")
override val features: Set[Feature[_, _]] = Set(ScoreFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
Stitch.value {
candidates.map { candidate =>
val score = candidate.features.getOrElse(ScoreFeature, None)
val updatedScore = getUpdatedScore(score, candidate, query)
FeatureMapBuilder().add(ScoreFeature, updatedScore).build()
}
}
}
/**
* We should only be applying this multiplier if the author of the candidate is Blue Verified.
* We also treat In-Network vs Out-of-Network differently.
*/
private def getUpdatedScore(
score: Option[Double],
candidate: CandidateWithFeatures[TweetCandidate],
query: PipelineQuery
): Option[Double] = {
val isAuthorBlueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false)
if (isAuthorBlueVerified) {
val isCandidateInNetwork = candidate.features.getOrElse(InNetworkFeature, false)
val scaleFactor =
if (isCandidateInNetwork) query.params(BlueVerifiedAuthorInNetworkMultiplierParam)
else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam)
score.map(_ * scaleFactor)
} else score
}
}

View File

@ -5,6 +5,7 @@ scala_library(
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",

View File

@ -1,6 +1,6 @@
package com.twitter.home_mixer.functional_component.selector
import com.twitter.home_mixer.functional_component.decorator.HomeClientEventDetailsBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature

View File

@ -7,14 +7,12 @@ scala_library(
"3rdparty/jvm/javax/inject:javax.inject",
"eventbus/client/src/main/scala/com/twitter/eventbus/client",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
@ -22,6 +20,7 @@ scala_library(
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
"src/scala/com/twitter/timelines/prediction/common/adapters",
@ -33,13 +32,12 @@ scala_library(
"src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java",
"src/thrift/com/twitter/timelines/timeline_logging:thrift-scala",
"src/thrift/com/twitter/user_session_store:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/core",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/uss",
"timelines/ml:kafka",
"timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka",
"timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding",
"timelines/src/main/scala/com/twitter/timelines/clientconfig",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",

View File

@ -2,13 +2,14 @@ package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates
import com.twitter.home_mixer.functional_component.decorator.HomeTweetTypePredicates
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates
import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -18,15 +19,17 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.injection.scribe.InjectionScribeUtil
private[side_effect] sealed trait ClientEventsBuilder {
private val FollowingSection = Some("home_latest")
private val FollowingSection = Some("latest")
private val ForYouSection = Some("home")
private val ListTweetsSection = Some("list")
private val SubscribedSection = Some("subscribed")
protected def section(query: PipelineQuery): Option[String] = {
query.product match {
case FollowingProduct => FollowingSection
case ForYouProduct => ForYouSection
case ListTweetsProduct => ListTweetsSection
case SubscribedProduct => SubscribedSection
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
}
}
@ -52,6 +55,7 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
private val InjectedComponent = Some("injected")
private val PromotedComponent = Some("promoted")
private val WhoToFollowComponent = Some("who_to_follow")
private val WhoToSubscribeComponent = Some("who_to_subscribe")
private val WithVideoDurationComponent = Some("with_video_duration")
private val VideoDurationSumElement = Some("video_duration_sum")
private val NumVideosElement = Some("num_videos")
@ -60,7 +64,8 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
query: PipelineQuery,
injectedTweets: Seq[ItemCandidateWithDetails],
promotedTweets: Seq[ItemCandidateWithDetails],
whoToFollowUsers: Seq[ItemCandidateWithDetails]
whoToFollowUsers: Seq[ItemCandidateWithDetails],
whoToSubscribeUsers: Seq[ItemCandidateWithDetails]
): Seq[ClientEvent] = {
val baseEventNamespace = EventNamespace(
section = section(query),
@ -77,6 +82,9 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
ClientEvent(
baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction),
eventValue = count(whoToFollowUsers)),
ClientEvent(
baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction),
eventValue = count(whoToSubscribeUsers)),
)
val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map {

View File

@ -5,6 +5,7 @@ import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.home_mixer.util.CandidatesUtil
import com.twitter.logpipeline.client.common.EventPublisher
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
@ -15,16 +16,30 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
* Side effect that logs served tweet metrics to Scribe as client events.
*/
case class HomeScribeClientEventSideEffect(
enableScribeClientEvents: Boolean,
override val logPipelinePublisher: EventPublisher[LogEvent],
injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier],
adsCandidatePipelineIdentifier: CandidatePipelineIdentifier,
adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] {
whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None)
extends ScribeClientEventSideEffect[PipelineQuery, Timeline]
with PipelineResultSideEffect.Conditionally[
PipelineQuery,
Timeline
] {
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent")
override val page = "timelinemixer"
override def onlyIf(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Boolean = enableScribeClientEvents
override def buildClientEvents(
query: PipelineQuery,
selectedCandidates: Seq[CandidateWithDetails],
@ -37,13 +52,15 @@ case class HomeScribeClientEventSideEffect(
val sources = itemCandidates.groupBy(_.source)
val injectedTweets =
injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty))
val promotedTweets = sources.getOrElse(adsCandidatePipelineIdentifier, Seq.empty)
val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
// WhoToFollow module is not required for all home-mixer products, e.g. list tweets timeline.
// WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline.
val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
val whoToSubscribeUsers =
whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
val servedEvents = ServedEventsBuilder
.build(query, injectedTweets, promotedTweets, whoToFollowUsers)
.build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers)
val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets)

View File

@ -0,0 +1,245 @@
package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.finagle.tracing.Trace
import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller
import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
import com.twitter.home_mixer.model.request.HasDeviceContext
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag
import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.inject.annotations.Flag
import com.twitter.logpipeline.client.common.EventPublisher
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
import com.twitter.util.Time
import javax.inject.Inject
import javax.inject.Singleton
/**
* Side effect that logs home timeline served candidates to Scribe.
*/
@Singleton
class HomeScribeServedCandidatesSideEffect @Inject() (
@Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean,
scribeEventPublisher: EventPublisher[thrift.ServedEntry])
extends ScribeLogEventSideEffect[
thrift.ServedEntry,
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
]
with PipelineResultSideEffect.Conditionally[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
] {
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates")
override def onlyIf(
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam)
override def buildLogEvents(
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Seq[thrift.ServedEntry] = {
val timelineType = query.product match {
case FollowingProduct => thrift.TimelineType.HomeLatest
case ForYouProduct => thrift.TimelineType.Home
case SubscribedProduct => thrift.TimelineType.HomeSubscribed
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
}
val requestProvenance = query.deviceContext.map { deviceContext =>
deviceContext.requestContextValue match {
case RequestContext.Foreground => thrift.RequestProvenance.Foreground
case RequestContext.Launch => thrift.RequestProvenance.Launch
case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
case _ => thrift.RequestProvenance.Other
}
}
val queryType = query.features.map { featureMap =>
if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
else thrift.QueryType.Other
}
val requestInfo = thrift.RequestInfo(
requestTimeMs = query.queryTime.inMilliseconds,
traceId = Trace.id.traceId.toLong,
userId = query.getOptionalUserId,
clientAppId = query.clientContext.appId,
hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
parentId = Some(Trace.id.parentId.toLong),
spanId = Some(Trace.id.spanId.toLong),
timelineType = Some(timelineType),
ipAddress = query.clientContext.ipAddress,
userAgent = query.clientContext.userAgent,
queryType = queryType,
requestProvenance = requestProvenance,
languageCode = query.clientContext.languageCode,
countryCode = query.clientContext.countryCode,
requestEndTimeMs = Some(Time.now.inMilliseconds),
servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None))
)
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
selectedCandidates.flatMap {
case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
Seq((item.candidateIdLong, item))
case module: ModuleCandidateWithDetails
if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
module.candidates.map(item => (item.candidateIdLong, item))
case _ => Seq.empty
}.toMap
val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
selectedCandidates.flatMap {
case module: ModuleCandidateWithDetails
if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
module.candidates.map { item =>
(item.candidateIdLong, item)
}
case _ => Seq.empty
}.toMap
response.instructions.zipWithIndex
.collect {
case (AddEntriesTimelineInstruction(entries), index) =>
entries.collect {
case entry: TweetItem if entry.promotedMetadata.isDefined =>
val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index)
Seq(
thrift.EntryInfo(
id = entry.id,
position = index.shortValue(),
entryId = entry.entryIdentifier,
entryType = thrift.EntryType.PromotedTweet,
sortIndex = entry.sortIndex,
verticalSize = Some(1),
displayType = Some(entry.displayType.toString),
details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails))
)
)
case entry: TweetItem =>
val candidate = tweetIdToItemCandidateMap(entry.id)
val tweetDetails = TweetDetailsMarshaller(entry, candidate)
Seq(
thrift.EntryInfo(
id = candidate.candidateIdLong,
position = index.shortValue(),
entryId = entry.entryIdentifier,
entryType = thrift.EntryType.Tweet,
sortIndex = entry.sortIndex,
verticalSize = Some(1),
score = candidate.features.getOrElse(ScoreFeature, None),
displayType = Some(entry.displayType.toString),
details = Some(thrift.ItemDetails.TweetDetails(tweetDetails))
)
)
case module: TimelineModule
if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
module.items.collect {
case ModuleItem(entry: UserItem, _, _) =>
val candidate = userIdToItemCandidateMap(entry.id)
val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate)
thrift.EntryInfo(
id = entry.id,
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.WhoToFollowModule,
sortIndex = module.sortIndex,
score = candidate.features.getOrElse(ScoreFeature, None),
displayType = Some(entry.displayType.toString),
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails))
)
}
case module: TimelineModule
if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
module.items.collect {
case ModuleItem(entry: UserItem, _, _) =>
val candidate = userIdToItemCandidateMap(entry.id)
val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate)
thrift.EntryInfo(
id = entry.id,
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.WhoToSubscribeModule,
sortIndex = module.sortIndex,
score = candidate.features.getOrElse(ScoreFeature, None),
displayType = Some(entry.displayType.toString),
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails))
)
}
case module: TimelineModule
if module.sortIndex.isDefined && module.items.headOption.exists(
_.item.isInstanceOf[TweetItem]) =>
module.items.collect {
case ModuleItem(entry: TweetItem, _, _) =>
val candidate = tweetIdToItemCandidateMap(entry.id)
thrift.EntryInfo(
id = entry.id,
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.ConversationModule,
sortIndex = module.sortIndex,
score = candidate.features.getOrElse(ScoreFeature, None),
displayType = Some(entry.displayType.toString)
)
}
case _ => Seq.empty
}.flatten
// Other instructions
case _ => Seq.empty[thrift.EntryInfo]
}.flatten.map { entryInfo =>
thrift.ServedEntry(
entry = Some(entryInfo),
request = requestInfo
)
}
}
override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] =
scribeEventPublisher
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
)
}

View File

@ -1,212 +0,0 @@
package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.finagle.tracing.Trace
import com.twitter.home_mixer.marshaller.timeline_logging.ConversationEntryMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetEntryMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.TweetEntryMarshaller
import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowEntryMarshaller
import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
import com.twitter.home_mixer.model.request.HasDeviceContext
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.logpipeline.client.common.EventPublisher
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
import com.twitter.util.Time
import javax.inject.Inject
import javax.inject.Singleton
/**
* Side effect that logs home timeline served entries to Scribe.
*/
@Singleton
class HomeScribeServedEntriesSideEffect @Inject() (
scribeEventPublisher: EventPublisher[thrift.Timeline])
extends PipelineResultSideEffect[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
] {
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedEntries")
final override def apply(
inputs: PipelineResultSideEffect.Inputs[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
]
): Stitch[Unit] = {
val timelineThrift = buildTimeline(inputs)
Stitch.callFuture(scribeEventPublisher.publish(timelineThrift)).unit
}
def buildTimeline(
inputs: PipelineResultSideEffect.Inputs[
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
Timeline
]
): thrift.Timeline = {
val timelineType = inputs.query.product match {
case FollowingProduct => thrift.TimelineType.HomeLatest
case ForYouProduct => thrift.TimelineType.Home
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
}
val requestProvenance = inputs.query.deviceContext.map { deviceContext =>
deviceContext.requestContextValue match {
case RequestContext.Foreground => thrift.RequestProvenance.Foreground
case RequestContext.Launch => thrift.RequestProvenance.Launch
case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
case _ => thrift.RequestProvenance.Other
}
}
val queryType = inputs.query.features.map { featureMap =>
if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
else thrift.QueryType.Other
}
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
inputs.selectedCandidates.flatMap {
case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
Seq((item.candidateIdLong, item))
case module: ModuleCandidateWithDetails
if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
module.candidates.map(item => (item.candidateIdLong, item))
case _ => Seq.empty
}.toMap
val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
inputs.selectedCandidates.flatMap {
case module: ModuleCandidateWithDetails
if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
module.candidates.map { item =>
(item.candidateIdLong, item)
}
case _ => Seq.empty
}.toMap
val timelineEntries = inputs.response.instructions.zipWithIndex.collect {
case (AddEntriesTimelineInstruction(entries), index) =>
entries.collect {
case entry: TweetItem if entry.promotedMetadata.isDefined =>
val promotedTweetEntry = PromotedTweetEntryMarshaller(entry, index)
Seq(
thrift.TimelineEntry(
content = thrift.Content.PromotedTweetEntry(promotedTweetEntry),
position = index.shortValue(),
entryId = entry.entryIdentifier,
entryType = thrift.EntryType.PromotedTweet,
sortIndex = entry.sortIndex,
verticalSize = Some(1)
)
)
case entry: TweetItem =>
val candidate = tweetIdToItemCandidateMap(entry.id)
val tweetEntry = TweetEntryMarshaller(entry, candidate)
Seq(
thrift.TimelineEntry(
content = thrift.Content.TweetEntry(tweetEntry),
position = index.shortValue(),
entryId = entry.entryIdentifier,
entryType = thrift.EntryType.Tweet,
sortIndex = entry.sortIndex,
verticalSize = Some(1)
)
)
case module: TimelineModule
if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
val whoToFollowEntries = module.items.collect {
case ModuleItem(entry: UserItem, _, _) =>
val candidate = userIdToItemCandidateMap(entry.id)
val whoToFollowEntry = WhoToFollowEntryMarshaller(entry, candidate)
thrift.AtomicEntry.WtfEntry(whoToFollowEntry)
}
Seq(
thrift.TimelineEntry(
content = thrift.Content.Entries(whoToFollowEntries),
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.WhoToFollowModule,
sortIndex = module.sortIndex
)
)
case module: TimelineModule
if module.sortIndex.isDefined && module.items.headOption.exists(
_.item.isInstanceOf[TweetItem]) =>
val conversationTweetEntries = module.items.collect {
case ModuleItem(entry: TweetItem, _, _) =>
val candidate = tweetIdToItemCandidateMap(entry.id)
val conversationEntry = ConversationEntryMarshaller(entry, candidate)
thrift.AtomicEntry.ConversationEntry(conversationEntry)
}
Seq(
thrift.TimelineEntry(
content = thrift.Content.Entries(conversationTweetEntries),
position = index.shortValue(),
entryId = module.entryIdentifier,
entryType = thrift.EntryType.ConversationModule,
sortIndex = module.sortIndex
)
)
case _ => Seq.empty
}.flatten
// Other instructions
case _ => Seq.empty[thrift.TimelineEntry]
}.flatten
thrift.Timeline(
timelineEntries = timelineEntries,
requestTimeMs = inputs.query.queryTime.inMilliseconds,
traceId = Trace.id.traceId.toLong,
userId = inputs.query.getOptionalUserId,
clientAppId = inputs.query.clientContext.appId,
sourceJobInstance = None,
hasDarkRequest = inputs.query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
parentId = Some(Trace.id.parentId.toLong),
spanId = Some(Trace.id.spanId.toLong),
timelineType = Some(timelineType),
ipAddress = inputs.query.clientContext.ipAddress,
userAgent = inputs.query.clientContext.userAgent,
queryType = queryType,
requestProvenance = requestProvenance,
sessionId = None,
timeZone = None,
browserNotificationPermission = None,
lastNonePollingTimeMs = None,
languageCode = inputs.query.clientContext.languageCode,
countryCode = inputs.query.clientContext.countryCode,
requestEndTimeMs = Some(Time.now.inMilliseconds),
servedRequestId = inputs.query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
requestJoinId = inputs.query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)),
requestSeenTweetIds = inputs.query.seenTweetIds
)
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
)
}

View File

@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.eventbus.client.EventBusPublisher
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
@ -22,6 +23,7 @@ import javax.inject.Singleton
object PublishClientSentImpressionsEventBusSideEffect {
val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline))
val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline))
val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed))
}
/**
@ -56,6 +58,7 @@ class PublishClientSentImpressionsEventBusSideEffect @Inject() (
val surfaceArea = query.product match {
case ForYouProduct => HomeSurfaceArea
case FollowingProduct => HomeLatestSurfaceArea
case SubscribedProduct => HomeSubscribedSurfaceArea
case _ => None
}
query.seenTweetIds.map { seenTweetIds =>

View File

@ -1,39 +1,46 @@
package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBloomFilter)
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, Timeline]
with PipelineResultSideEffect.Conditionally[PipelineQuery with HasSeenTweetIds, Timeline] {
private val SurfaceArea = t.SurfaceArea.HomeTimeline
class PublishImpressionBloomFilterSideEffect @Inject() (
bloomFilterClient: ManhattanStoreClient[
blm.ImpressionBloomFilterKey,
blm.ImpressionBloomFilterSeq
]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling]
with PipelineResultSideEffect.Conditionally[
PipelineQuery with HasSeenTweetIds,
HasMarshalling
] {
override val identifier: SideEffectIdentifier =
SideEffectIdentifier("UpdateImpressionBloomFilter")
SideEffectIdentifier("PublishImpressionBloomFilter")
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
override def onlyIf(
query: PipelineQuery with HasSeenTweetIds,
selectedCandidates: Seq[CandidateWithDetails],
remainingCandidates: Seq[CandidateWithDetails],
droppedCandidates: Seq[CandidateWithDetails],
response: Timeline
): Boolean = query.seenTweetIds.exists(_.nonEmpty)
response: HasMarshalling
): Boolean =
query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty)
def buildEvents(query: PipelineQuery): Option[t.ImpressionBloomFilterSeq] = {
def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = {
query.features.flatMap { featureMap =>
val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature)
if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq)
@ -42,19 +49,17 @@ class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBl
}
override def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, Timeline]
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling]
): Stitch[Unit] = {
buildEvents(inputs.query)
.map { updatedBloomFilter =>
bloomFilter.writeBloomFilterSeq(
userId = inputs.query.getRequiredUserId,
surfaceArea = SurfaceArea,
impressionBloomFilterSeq = updatedBloomFilter)
.map { updatedBloomFilterSeq =>
bloomFilterClient.write(
blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea),
updatedBloomFilterSeq)
}.getOrElse(Stitch.Unit)
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8),
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(30.millis)
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8)
)
}

View File

@ -1,80 +0,0 @@
package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.param.HomeGlobalParams.AuthorListForStatsParam
import com.twitter.home_mixer.util.CandidatesUtil
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver)
extends PipelineResultSideEffect[PipelineQuery, Timeline] {
override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats")
private val baseStatsReceiver = statsReceiver.scope(identifier.toString)
private val authorStatsReceiver = baseStatsReceiver.scope("Author")
private val candidateSourceStatsReceiver = baseStatsReceiver.scope("CandidateSource")
private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance")
private val inNetworkStatsCounter = contentBalanceStatsReceiver.counter("InNetwork")
private val outOfNetworkStatsCounter = contentBalanceStatsReceiver.counter("OutOfNetwork")
override def apply(
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
): Stitch[Unit] = {
val tweetCandidates = CandidatesUtil
.getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]())
recordAuthorStats(tweetCandidates, inputs.query.params(AuthorListForStatsParam))
recordCandidateSourceStats(tweetCandidates)
recordContentBalanceStats(tweetCandidates)
Stitch.Unit
}
def recordAuthorStats(candidates: Seq[CandidateWithDetails], authors: Set[Long]): Unit = {
candidates
.filter { candidate =>
candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) &&
// Only include original tweets
(!candidate.features.getOrElse(IsRetweetFeature, false)) &&
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
}
.groupBy { candidate =>
(getCandidateSourceId(candidate), candidate.features.get(AuthorIdFeature).get)
}
.foreach {
case ((candidateSourceId, authorId), authorCandidates) =>
authorStatsReceiver
.scope(authorId.toString).counter(candidateSourceId).incr(authorCandidates.size)
}
}
def recordCandidateSourceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
candidates.groupBy(getCandidateSourceId).foreach {
case (candidateSourceId, candidateSourceCandidates) =>
candidateSourceStatsReceiver.counter(candidateSourceId).incr(candidateSourceCandidates.size)
}
}
def recordContentBalanceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true))
inNetworkStatsCounter.incr(in.size)
outOfNetworkStatsCounter.incr(oon.size)
}
private def getCandidateSourceId(candidate: CandidateWithDetails): String =
candidate.features.getOrElse(CandidateSourceIdFeature, None).map(_.name).getOrElse("None")
}

View File

@ -3,8 +3,10 @@ package com.twitter.home_mixer.functional_component.side_effect
import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
@ -97,12 +99,14 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
val entries = inputs.response.instructions.collect {
case AddEntriesTimelineInstruction(entries) =>
entries.collect {
// includes both tweets and promoted tweets
case entry: TweetItem if entry.sortIndex.isDefined =>
// includes tweets, tweet previews, and promoted tweets
case entry: TweetItem if entry.sortIndex.isDefined => {
Seq(
buildTweetEntryWithItemIds(
tweetIdToItemCandidateMap(entry.id),
entry.sortIndex.get))
entry.sortIndex.get
))
}
// tweet conversation modules are flattened to individual tweets in the persistence store
case module: TimelineModule
if module.sortIndex.isDefined && module.items.headOption.exists(
@ -125,6 +129,19 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
size = module.items.size.toShort,
itemIds = Some(userIds)
))
case module: TimelineModule
if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
val userIds = module.items
.map(item =>
UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId =
Some(item.item.id.asInstanceOf[Long])))
Seq(
EntryWithItemIds(
entityIdType = EntityIdType.WhoToSubscribe,
sortIndex = module.sortIndex.get,
size = module.items.size.toShort,
itemIds = Some(userIds)
))
}.flatten
case ShowCoverInstruction(cover) =>
Seq(
@ -216,8 +233,11 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
userId = None
)
val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false)
val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet
EntryWithItemIds(
entityIdType = EntityIdType.Tweet,
entityIdType = entityType,
sortIndex = sortIndex,
size = 1.toShort,
itemIds = Some(Seq(itemIds))

View File

@ -4,14 +4,12 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
],
exports = [
"dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",

View File

@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProductContext
import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext
import com.twitter.home_mixer.model.request.ListTweetsProductContext
import com.twitter.home_mixer.model.request.ScoredTweetsProductContext
import com.twitter.home_mixer.model.request.SubscribedProductContext
import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.model.marshalling.request.ProductContext
import javax.inject.Inject
import javax.inject.Singleton
@ -26,15 +26,17 @@ class HomeMixerProductContextUnmarshaller @Inject() (
ForYouProductContext(
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
seenTweetIds = p.seenTweetIds,
dspClientContext = p.dspClientContext
dspClientContext = p.dspClientContext,
pushToHomeTweetId = p.pushToHomeTweetId
)
case t.ProductContext.Realtime(p) =>
case t.ProductContext.ListManagement(p) =>
throw new UnsupportedOperationException(s"This product is no longer used")
case t.ProductContext.ScoredTweets(p) =>
ScoredTweetsProductContext(
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
seenTweetIds = p.seenTweetIds,
servedTweetIds = p.servedTweetIds
servedTweetIds = p.servedTweetIds,
backfillTweetIds = p.backfillTweetIds
)
case t.ProductContext.ListTweets(p) =>
ListTweetsProductContext(
@ -46,7 +48,13 @@ class HomeMixerProductContextUnmarshaller @Inject() (
ListRecommendedUsersProductContext(
listId = p.listId,
selectedUserIds = p.selectedUserIds,
excludedUserIds = p.excludedUserIds
excludedUserIds = p.excludedUserIds,
listName = p.listName
)
case t.ProductContext.Subscribed(p) =>
SubscribedProductContext(
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
seenTweetIds = p.seenTweetIds,
)
case t.ProductContext.UnknownUnionField(field) =>
throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}")

View File

@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.model.marshalling.request.Product
import javax.inject.Inject
import javax.inject.Singleton
@ -17,11 +17,12 @@ class HomeMixerProductUnmarshaller @Inject() () {
def apply(product: t.Product): Product = product match {
case t.Product.Following => FollowingProduct
case t.Product.ForYou => ForYouProduct
case t.Product.Realtime =>
case t.Product.ListManagement =>
throw new UnsupportedOperationException(s"This product is no longer used")
case t.Product.ScoredTweets => ScoredTweetsProduct
case t.Product.ListTweets => ListTweetsProduct
case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct
case t.Product.Subscribed => SubscribedProduct
case t.Product.EnumUnknownProduct(value) =>
throw new UnsupportedOperationException(s"Unknown product: $value")
}

View File

@ -1,16 +0,0 @@
package com.twitter.home_mixer.marshaller.timeline_logging
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
object ConversationEntryMarshaller {
def apply(entry: TweetItem, candidate: ItemCandidateWithDetails): thriftlog.ConversationEntry =
thriftlog.ConversationEntry(
displayedTweetId = entry.id,
displayType = Some(entry.displayType.toString),
score = candidate.features.getOrElse(ScoreFeature, None)
)
}

View File

@ -3,15 +3,13 @@ package com.twitter.home_mixer.marshaller.timeline_logging
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
object PromotedTweetEntryMarshaller {
object PromotedTweetDetailsMarshaller {
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetEntry = {
thriftlog.PromotedTweetEntry(
id = entry.id,
advertiserId = entry.promotedMetadata.map(_.advertiserId).getOrElse(0L),
insertPosition = position,
impressionId = entry.promotedMetadata.flatMap(_.impressionString),
displayType = Some(entry.displayType.toString)
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetDetails = {
thriftlog.PromotedTweetDetails(
advertiserId = Some(entry.promotedMetadata.map(_.advertiserId).getOrElse(0L)),
insertPosition = Some(position),
impressionId = entry.promotedMetadata.flatMap(_.impressionString)
)
}
}

View File

@ -0,0 +1,47 @@
package com.twitter.home_mixer.marshaller.timeline_logging
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation
import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation
import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ConversationGeneralContextType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext
import com.twitter.timelines.service.{thriftscala => tst}
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
object TweetDetailsMarshaller {
private val generalContextTypeMarshaller = new GeneralContextTypeMarshaller()
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetDetails = {
val socialContext = candidate.presentation.flatMap {
case _ @UrtItemPresentation(timelineItem: TweetItem, _) => timelineItem.socialContext
case _ @UrtModulePresentation(timelineModule) =>
timelineModule.items.head.item match {
case timelineItem: TweetItem => timelineItem.socialContext
case _ => Some(ConversationGeneralContextType)
}
}
val socialContextType = socialContext match {
case Some(GeneralContext(contextType, _, _, _, _)) =>
Some(generalContextTypeMarshaller(contextType).value.toShort)
case Some(TopicContext(_, _)) => Some(tst.ContextType.Topic.value.toShort)
case _ => None
}
thriftlog.TweetDetails(
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
socialContextType = socialContextType,
suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name),
authorId = candidate.features.getOrElse(AuthorIdFeature, None),
sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None)
)
}
}

View File

@ -1,29 +0,0 @@
package com.twitter.home_mixer.marshaller.timeline_logging
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.model.HomeFeatures.SocialContextFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
import com.twitter.timelines.service.{thriftscala => tst}
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
object TweetEntryMarshaller {
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetEntry = {
val socialContextType = candidate.features.getOrElse(SocialContextFeature, None) match {
case Some(tst.SocialContext.GeneralContext(tst.GeneralContext(contextType, _, _, _, _))) =>
Some(contextType.value.toShort)
case Some(tst.SocialContext.TopicContext(_)) =>
Some(tst.ContextType.Topic.value.toShort)
case _ => None
}
thriftlog.TweetEntry(
id = candidate.candidateIdLong,
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
displayType = Some(entry.displayType.toString),
score = candidate.features.getOrElse(ScoreFeature, None),
socialContextType = socialContextType
)
}
}

Some files were not shown because too many files have changed in this diff Show More