mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-12-23 02:31:52 +01:00
Compare commits
2 Commits
c7183cba15
...
5aa58bca37
Author | SHA1 | Date | |
---|---|---|---|
|
5aa58bca37 | ||
|
72eda9a24f |
@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
|
|||||||
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
|
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
|
||||||
- Fetch Tweet Candidates
|
- Fetch Tweet Candidates
|
||||||
- ScoredTweetsInNetworkCandidatePipelineConfig
|
- ScoredTweetsInNetworkCandidatePipelineConfig
|
||||||
- ScoredTweetsCrMixerCandidatePipelineConfig
|
- ScoredTweetsTweetMixerCandidatePipelineConfig
|
||||||
- ScoredTweetsUtegCandidatePipelineConfig
|
- ScoredTweetsUtegCandidatePipelineConfig
|
||||||
- ScoredTweetsFrsCandidatePipelineConfig
|
- ScoredTweetsFrsCandidatePipelineConfig
|
||||||
- Feature Hydration and Scoring
|
- Feature Hydration and Scoring
|
||||||
@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer.
|
|||||||
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
|
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
|
||||||
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
|
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
|
||||||
- ListTweetsAdsCandidatePipelineConfig (fetch ads)
|
- ListTweetsAdsCandidatePipelineConfig (fetch ads)
|
||||||
|
|
||||||
|
@ -21,6 +21,7 @@ scala_library(
|
|||||||
"finatra/inject/inject-utils/src/main/scala",
|
"finatra/inject/inject-utils/src/main/scala",
|
||||||
"home-mixer/server/src/main/resources",
|
"home-mixer/server/src/main/resources",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
|
||||||
@ -31,6 +32,10 @@ scala_library(
|
|||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
|
||||||
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
|
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
||||||
|
"strato/config/columns/auth-context:auth-context-strato-client",
|
||||||
|
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/fed",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||||
"stringcenter/client",
|
"stringcenter/client",
|
||||||
"stringcenter/client/src/main/java",
|
"stringcenter/client/src/main/java",
|
||||||
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",
|
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",
|
||||||
|
@ -2,7 +2,7 @@ package com.twitter.home_mixer
|
|||||||
|
|
||||||
import com.twitter.finatra.http.routing.HttpWarmup
|
import com.twitter.finatra.http.routing.HttpWarmup
|
||||||
import com.twitter.finatra.httpclient.RequestBuilder._
|
import com.twitter.finatra.httpclient.RequestBuilder._
|
||||||
import com.twitter.inject.Logging
|
import com.twitter.util.logging.Logging
|
||||||
import com.twitter.inject.utils.Handler
|
import com.twitter.inject.utils.Handler
|
||||||
import com.twitter.util.Try
|
import com.twitter.util.Try
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
|
@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer
|
|||||||
import com.twitter.finatra.thrift.filters._
|
import com.twitter.finatra.thrift.filters._
|
||||||
import com.twitter.finatra.thrift.routing.ThriftRouter
|
import com.twitter.finatra.thrift.routing.ThriftRouter
|
||||||
import com.twitter.home_mixer.controller.HomeThriftController
|
import com.twitter.home_mixer.controller.HomeThriftController
|
||||||
|
import com.twitter.home_mixer.federated.HomeMixerColumn
|
||||||
import com.twitter.home_mixer.module._
|
import com.twitter.home_mixer.module._
|
||||||
import com.twitter.home_mixer.param.GlobalParamConfigModule
|
import com.twitter.home_mixer.param.GlobalParamConfigModule
|
||||||
import com.twitter.home_mixer.product.HomeMixerProductModule
|
import com.twitter.home_mixer.product.HomeMixerProductModule
|
||||||
import com.twitter.home_mixer.{thriftscala => st}
|
import com.twitter.home_mixer.{thriftscala => st}
|
||||||
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
|
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
|
||||||
import com.twitter.product_mixer.component_library.module.CrMixerClientModule
|
|
||||||
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
|
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
|
||||||
import com.twitter.product_mixer.component_library.module.EarlybirdModule
|
import com.twitter.product_mixer.component_library.module.EarlybirdModule
|
||||||
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
|
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
|
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
|
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
|
||||||
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
|
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
|
||||||
import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
|
|
||||||
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
|
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
|
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
|
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
|
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
|
||||||
|
import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
|
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
|
||||||
import com.twitter.product_mixer.core.controllers.ProductMixerController
|
import com.twitter.product_mixer.core.controllers.ProductMixerController
|
||||||
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
|
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
|
||||||
import com.twitter.product_mixer.core.module.ProductMixerModule
|
import com.twitter.product_mixer.core.module.ProductMixerModule
|
||||||
import com.twitter.product_mixer.core.module.StratoClientModule
|
|
||||||
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
|
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
|
||||||
|
import com.twitter.strato.fed.StratoFed
|
||||||
|
import com.twitter.strato.fed.server.StratoFedServer
|
||||||
|
|
||||||
object HomeMixerServerMain extends HomeMixerServer
|
object HomeMixerServerMain extends HomeMixerServer
|
||||||
|
|
||||||
class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
|
class HomeMixerServer
|
||||||
|
extends StratoFedServer
|
||||||
|
with ThriftServer
|
||||||
|
with Mtls
|
||||||
|
with HttpServer
|
||||||
|
with HttpMtls {
|
||||||
override val name = "home-mixer-server"
|
override val name = "home-mixer-server"
|
||||||
|
|
||||||
override val modules: Seq[Module] = Seq(
|
override val modules: Seq[Module] = Seq(
|
||||||
AccountRecommendationsMixerModule,
|
AccountRecommendationsMixerModule,
|
||||||
AdvertiserBrandSafetySettingsStoreModule,
|
AdvertiserBrandSafetySettingsStoreModule,
|
||||||
|
BlenderClientModule,
|
||||||
ClientSentImpressionsPublisherModule,
|
ClientSentImpressionsPublisherModule,
|
||||||
ConversationServiceModule,
|
ConversationServiceModule,
|
||||||
CrMixerClientModule,
|
|
||||||
EarlybirdModule,
|
EarlybirdModule,
|
||||||
ExploreRankerClientModule,
|
ExploreRankerClientModule,
|
||||||
|
FeedbackHistoryClientModule,
|
||||||
GizmoduckClientModule,
|
GizmoduckClientModule,
|
||||||
GlobalParamConfigModule,
|
GlobalParamConfigModule,
|
||||||
HomeAdsCandidateSourceModule,
|
HomeAdsCandidateSourceModule,
|
||||||
HomeMixerFlagsModule,
|
HomeMixerFlagsModule,
|
||||||
HomeMixerProductModule,
|
HomeMixerProductModule,
|
||||||
HomeMixerResourcesModule,
|
HomeMixerResourcesModule,
|
||||||
HomeNaviModelClientModule,
|
|
||||||
ImpressionBloomFilterModule,
|
ImpressionBloomFilterModule,
|
||||||
InjectionHistoryClientModule,
|
InjectionHistoryClientModule,
|
||||||
FeedbackHistoryClientModule,
|
|
||||||
ManhattanClientsModule,
|
ManhattanClientsModule,
|
||||||
ManhattanFeatureRepositoryModule,
|
ManhattanFeatureRepositoryModule,
|
||||||
ManhattanTweetImpressionStoreModule,
|
ManhattanTweetImpressionStoreModule,
|
||||||
MemcachedFeatureRepositoryModule,
|
MemcachedFeatureRepositoryModule,
|
||||||
|
NaviModelClientModule,
|
||||||
OnboardingTaskServiceModule,
|
OnboardingTaskServiceModule,
|
||||||
OptimizedStratoClientModule,
|
OptimizedStratoClientModule,
|
||||||
PeopleDiscoveryServiceModule,
|
PeopleDiscoveryServiceModule,
|
||||||
@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
|
|||||||
SimClustersRecentEngagementsClientModule,
|
SimClustersRecentEngagementsClientModule,
|
||||||
SocialGraphServiceModule,
|
SocialGraphServiceModule,
|
||||||
StaleTweetsCacheModule,
|
StaleTweetsCacheModule,
|
||||||
StratoClientModule,
|
|
||||||
ThriftFeatureRepositoryModule,
|
ThriftFeatureRepositoryModule,
|
||||||
TimelineMixerClientModule,
|
|
||||||
TimelineRankerClientModule,
|
TimelineRankerClientModule,
|
||||||
TimelineScorerClientModule,
|
TimelineScorerClientModule,
|
||||||
TimelineServiceClientModule,
|
TimelineServiceClientModule,
|
||||||
TimelinesPersistenceStoreClientModule,
|
TimelinesPersistenceStoreClientModule,
|
||||||
|
TopicSocialProofClientModule,
|
||||||
TweetImpressionStoreModule,
|
TweetImpressionStoreModule,
|
||||||
TweetyPieClientModule,
|
TweetMixerClientModule,
|
||||||
|
TweetypieClientModule,
|
||||||
TweetypieStaticEntitiesCacheClientModule,
|
TweetypieStaticEntitiesCacheClientModule,
|
||||||
UserMetadataStoreModule,
|
|
||||||
UserSessionStoreModule,
|
UserSessionStoreModule,
|
||||||
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
|
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
|
||||||
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
|
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
|
||||||
new ProductScopeStringCenterModule()
|
new ProductScopeStringCenterModule()
|
||||||
)
|
)
|
||||||
|
|
||||||
def configureThrift(router: ThriftRouter): Unit = {
|
override def configureThrift(router: ThriftRouter): Unit = {
|
||||||
router
|
router
|
||||||
.filter[LoggingMDCFilter]
|
.filter[LoggingMDCFilter]
|
||||||
.filter[TraceIdMDCFilter]
|
.filter[TraceIdMDCFilter]
|
||||||
@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
|
|||||||
this.injector,
|
this.injector,
|
||||||
st.HomeMixer.ExecutePipeline))
|
st.HomeMixer.ExecutePipeline))
|
||||||
|
|
||||||
|
override val dest: String = "/s/home-mixer/home-mixer:strato"
|
||||||
|
|
||||||
|
override val columns: Seq[Class[_ <: StratoFed.Column]] =
|
||||||
|
Seq(classOf[HomeMixerColumn])
|
||||||
|
|
||||||
override protected def warmup(): Unit = {
|
override protected def warmup(): Unit = {
|
||||||
handle[HomeMixerThriftServerWarmupHandler]()
|
handle[HomeMixerThriftServerWarmupHandler]()
|
||||||
handle[HomeMixerHttpServerWarmupHandler]()
|
handle[HomeMixerHttpServerWarmupHandler]()
|
||||||
|
@ -3,7 +3,7 @@ package com.twitter.home_mixer
|
|||||||
import com.twitter.finagle.thrift.ClientId
|
import com.twitter.finagle.thrift.ClientId
|
||||||
import com.twitter.finatra.thrift.routing.ThriftWarmup
|
import com.twitter.finatra.thrift.routing.ThriftWarmup
|
||||||
import com.twitter.home_mixer.{thriftscala => st}
|
import com.twitter.home_mixer.{thriftscala => st}
|
||||||
import com.twitter.inject.Logging
|
import com.twitter.util.logging.Logging
|
||||||
import com.twitter.inject.utils.Handler
|
import com.twitter.inject.utils.Handler
|
||||||
import com.twitter.product_mixer.core.{thriftscala => pt}
|
import com.twitter.product_mixer.core.{thriftscala => pt}
|
||||||
import com.twitter.scrooge.Request
|
import com.twitter.scrooge.Request
|
||||||
|
@ -5,19 +5,13 @@ scala_library(
|
|||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
||||||
@ -25,10 +19,6 @@ scala_library(
|
|||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
|
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
|
||||||
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
|
||||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
|
||||||
],
|
],
|
||||||
exports = [
|
exports = [
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
||||||
|
@ -1,18 +1,23 @@
|
|||||||
package com.twitter.home_mixer.candidate_pipeline
|
package com.twitter.home_mixer.candidate_pipeline
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
|
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
|
||||||
import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
|
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
|
||||||
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
|
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
|
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
|
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
|
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
|
||||||
import com.twitter.product_mixer.component_library.filter.FeatureFilter
|
import com.twitter.product_mixer.component_library.filter.FeatureFilter
|
||||||
|
import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
|
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
|
||||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||||
@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
|
|||||||
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
||||||
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
||||||
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
||||||
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
|
|
||||||
namesFeatureHydrator: NamesFeatureHydrator,
|
namesFeatureHydrator: NamesFeatureHydrator,
|
||||||
|
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
|
||||||
override val gates: Seq[BaseGate[Query]],
|
override val gates: Seq[BaseGate[Query]],
|
||||||
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
|
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
|
||||||
extends DependentCandidatePipelineConfig[
|
extends DependentCandidatePipelineConfig[
|
||||||
@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
|||||||
val tweetsWithConversationMetadata = candidates.map { candidate =>
|
val tweetsWithConversationMetadata = candidates.map { candidate =>
|
||||||
TweetWithConversationMetadata(
|
TweetWithConversationMetadata(
|
||||||
tweetId = candidate.candidateIdLong,
|
tweetId = candidate.candidateIdLong,
|
||||||
userId = None,
|
userId = candidate.features.getOrElse(AuthorIdFeature, None),
|
||||||
sourceTweetId = None,
|
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
|
||||||
sourceUserId = None,
|
sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
|
||||||
inReplyToTweetId = None,
|
inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
|
||||||
conversationId = None,
|
conversationId = None,
|
||||||
ancestors = Seq.empty
|
ancestors = Seq.empty
|
||||||
)
|
)
|
||||||
@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
|||||||
|
|
||||||
override val preFilterFeatureHydrationPhase1: Seq[
|
override val preFilterFeatureHydrationPhase1: Seq[
|
||||||
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
|
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
|
||||||
] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
|
] = Seq(
|
||||||
|
tweetypieFeatureHydrator,
|
||||||
|
InNetworkFeatureHydrator,
|
||||||
|
)
|
||||||
|
|
||||||
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
|
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
|
||||||
RetweetDeduplicationFilter,
|
RetweetDeduplicationFilter,
|
||||||
@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
|||||||
FilterIdentifier(QuotedTweetDroppedFilterId),
|
FilterIdentifier(QuotedTweetDroppedFilterId),
|
||||||
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
|
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
|
||||||
),
|
),
|
||||||
|
invalidSubscriptionTweetFilter,
|
||||||
InvalidConversationModuleFilter
|
InvalidConversationModuleFilter
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -1,9 +1,9 @@
|
|||||||
package com.twitter.home_mixer.candidate_pipeline
|
package com.twitter.home_mixer.candidate_pipeline
|
||||||
|
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
||||||
|
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
|
||||||
|
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||||
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
|
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
|
||||||
@ -15,7 +15,7 @@ import javax.inject.Singleton
|
|||||||
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
|
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
|
||||||
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
||||||
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
||||||
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
|
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
|
||||||
namesFeatureHydrator: NamesFeatureHydrator) {
|
namesFeatureHydrator: NamesFeatureHydrator) {
|
||||||
|
|
||||||
def build(
|
def build(
|
||||||
@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
|
|||||||
new ConversationServiceCandidatePipelineConfig(
|
new ConversationServiceCandidatePipelineConfig(
|
||||||
conversationServiceCandidateSource,
|
conversationServiceCandidateSource,
|
||||||
tweetypieFeatureHydrator,
|
tweetypieFeatureHydrator,
|
||||||
socialGraphServiceFeatureHydrator,
|
|
||||||
namesFeatureHydrator,
|
namesFeatureHydrator,
|
||||||
|
invalidSubscriptionTweetFilter,
|
||||||
gates,
|
gates,
|
||||||
decorator
|
decorator
|
||||||
)
|
)
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
package com.twitter.home_mixer.candidate_pipeline
|
package com.twitter.home_mixer.candidate_pipeline
|
||||||
|
|
||||||
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
|
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
|
||||||
import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
|
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
|
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
|
@ -13,8 +13,5 @@ scala_library(
|
|||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
|
||||||
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
||||||
"src/thrift/com/twitter/context:twitter-context-scala",
|
|
||||||
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
|
||||||
"twitter-context/src/main/scala",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService
|
|||||||
import com.twitter.home_mixer.{thriftscala => t}
|
import com.twitter.home_mixer.{thriftscala => t}
|
||||||
import com.twitter.product_mixer.core.controllers.DebugTwitterContext
|
import com.twitter.product_mixer.core.controllers.DebugTwitterContext
|
||||||
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
|
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
|
||||||
|
import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
|
||||||
import com.twitter.product_mixer.core.service.urt.UrtService
|
import com.twitter.product_mixer.core.service.urt.UrtService
|
||||||
import com.twitter.snowflake.id.SnowflakeId
|
import com.twitter.snowflake.id.SnowflakeId
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
|
@ -0,0 +1,24 @@
|
|||||||
|
scala_library(
|
||||||
|
sources = ["*.scala"],
|
||||||
|
compiler_option_sets = ["fatal_warnings"],
|
||||||
|
strict_deps = True,
|
||||||
|
tags = ["bazel-compatible"],
|
||||||
|
dependencies = [
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
|
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||||
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||||
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
|
||||||
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
|
||||||
|
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
|
||||||
|
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||||
|
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
||||||
|
"stitch/stitch-repo/src/main/scala",
|
||||||
|
"strato/config/columns/auth-context:auth-context-strato-client",
|
||||||
|
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
|
||||||
|
"strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/callcontext",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/fed",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||||
|
],
|
||||||
|
)
|
@ -0,0 +1,217 @@
|
|||||||
|
package com.twitter.home_mixer.federated
|
||||||
|
|
||||||
|
import com.twitter.gizmoduck.{thriftscala => gd}
|
||||||
|
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
|
||||||
|
import com.twitter.home_mixer.model.request.HomeMixerRequest
|
||||||
|
import com.twitter.home_mixer.{thriftscala => hm}
|
||||||
|
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
|
||||||
|
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
|
||||||
|
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
|
||||||
|
import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
|
||||||
|
import com.twitter.product_mixer.core.{thriftscala => pm}
|
||||||
|
import com.twitter.stitch.Arrow
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
import com.twitter.strato.callcontext.CallContext
|
||||||
|
import com.twitter.strato.catalog.OpMetadata
|
||||||
|
import com.twitter.strato.config._
|
||||||
|
import com.twitter.strato.data._
|
||||||
|
import com.twitter.strato.fed.StratoFed
|
||||||
|
import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
|
||||||
|
import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
|
||||||
|
import com.twitter.strato.graphql.timelines.{thriftscala => gql}
|
||||||
|
import com.twitter.strato.thrift.ScroogeConv
|
||||||
|
import com.twitter.timelines.render.{thriftscala => tr}
|
||||||
|
import com.twitter.util.Try
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
class HomeMixerColumn @Inject() (
|
||||||
|
homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
|
||||||
|
compositeOnUserClientColumn: CompositeOnUserClientColumn,
|
||||||
|
auditIpClientColumn: AuditIpClientColumn,
|
||||||
|
paramsBuilder: ParamsBuilder,
|
||||||
|
productPipelineRegistry: ProductPipelineRegistry)
|
||||||
|
extends StratoFed.Column(HomeMixerColumn.Path)
|
||||||
|
with StratoFed.Fetch.Arrow {
|
||||||
|
|
||||||
|
override val contactInfo: ContactInfo = ContactInfo(
|
||||||
|
contactEmail = "",
|
||||||
|
ldapGroup = "",
|
||||||
|
slackRoomId = ""
|
||||||
|
)
|
||||||
|
|
||||||
|
override val metadata: OpMetadata =
|
||||||
|
OpMetadata(
|
||||||
|
lifecycle = Some(Lifecycle.Production),
|
||||||
|
description =
|
||||||
|
Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
|
||||||
|
)
|
||||||
|
|
||||||
|
private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
|
||||||
|
private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
|
||||||
|
ServiceIdentifierPattern(
|
||||||
|
role = "",
|
||||||
|
service = "",
|
||||||
|
env = "",
|
||||||
|
zone = Seq(""))
|
||||||
|
)
|
||||||
|
|
||||||
|
override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
|
||||||
|
|
||||||
|
override type Key = gql.TimelineKey
|
||||||
|
override type View = gql.HomeTimelineView
|
||||||
|
override type Value = tr.Timeline
|
||||||
|
|
||||||
|
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
|
||||||
|
override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
|
||||||
|
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
|
||||||
|
|
||||||
|
private def createHomeMixerRequestArrow(
|
||||||
|
compositeOnUserClientColumn: CompositeOnUserClientColumn,
|
||||||
|
auditIpClientColumn: AuditIpClientColumn
|
||||||
|
): Arrow[(Key, View), hm.HomeMixerRequest] = {
|
||||||
|
|
||||||
|
val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
|
||||||
|
val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
|
||||||
|
(gd.LookupContext(), Set(gd.QueryFields.Roles))
|
||||||
|
|
||||||
|
val populateUserRoles = Arrow
|
||||||
|
.flatMap[(Key, View), Option[Set[String]]] { _ =>
|
||||||
|
Stitch.collect {
|
||||||
|
CallContext.twitterUserId.map { userId =>
|
||||||
|
compositeOnUserClientColumn.fetcher
|
||||||
|
.callStack(HomeMixerColumn.FetchCallstack)
|
||||||
|
.fetch(userId, gizmoduckView).map(_.v)
|
||||||
|
.map {
|
||||||
|
_.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val populateIpAddress = Arrow
|
||||||
|
.flatMap[(Key, View), Option[String]](_ =>
|
||||||
|
auditIpClientColumn.fetcher
|
||||||
|
.callStack(HomeMixerColumn.FetchCallstack)
|
||||||
|
.fetch((), ()).map(_.v))
|
||||||
|
|
||||||
|
Arrow.join(
|
||||||
|
populateUserRoles,
|
||||||
|
populateIpAddress
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
Arrow.zipWithArg(populateUserRolesAndIp).map {
|
||||||
|
case ((key, view), (roles, ipAddress)) =>
|
||||||
|
val deviceContextOpt = Some(
|
||||||
|
hm.DeviceContext(
|
||||||
|
isPolling = CallContext.isPolling,
|
||||||
|
requestContext = view.requestContext,
|
||||||
|
latestControlAvailable = view.latestControlAvailable,
|
||||||
|
autoplayEnabled = view.autoplayEnabled
|
||||||
|
))
|
||||||
|
val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
|
||||||
|
|
||||||
|
val (product, productContext) = key match {
|
||||||
|
case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
|
||||||
|
(
|
||||||
|
hm.Product.ForYou,
|
||||||
|
hm.ProductContext.ForYou(
|
||||||
|
hm.ForYou(
|
||||||
|
deviceContextOpt,
|
||||||
|
seenTweetIds,
|
||||||
|
view.dspClientContext,
|
||||||
|
view.pushToHomeTweetId
|
||||||
|
)
|
||||||
|
))
|
||||||
|
case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
|
||||||
|
(
|
||||||
|
hm.Product.Following,
|
||||||
|
hm.ProductContext.Following(
|
||||||
|
hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
|
||||||
|
case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
|
||||||
|
(
|
||||||
|
hm.Product.Subscribed,
|
||||||
|
hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
|
||||||
|
case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
|
||||||
|
}
|
||||||
|
|
||||||
|
val clientContext = pm.ClientContext(
|
||||||
|
userId = CallContext.twitterUserId,
|
||||||
|
guestId = CallContext.guestId,
|
||||||
|
guestIdAds = CallContext.guestIdAds,
|
||||||
|
guestIdMarketing = CallContext.guestIdMarketing,
|
||||||
|
appId = CallContext.clientApplicationId,
|
||||||
|
ipAddress = ipAddress,
|
||||||
|
userAgent = CallContext.userAgent,
|
||||||
|
countryCode = CallContext.requestCountryCode,
|
||||||
|
languageCode = CallContext.requestLanguageCode,
|
||||||
|
isTwoffice = CallContext.isInternalOrTwoffice,
|
||||||
|
userRoles = roles,
|
||||||
|
deviceId = CallContext.deviceId,
|
||||||
|
mobileDeviceId = CallContext.mobileDeviceId,
|
||||||
|
mobileDeviceAdId = CallContext.adId,
|
||||||
|
limitAdTracking = CallContext.limitAdTracking
|
||||||
|
)
|
||||||
|
|
||||||
|
hm.HomeMixerRequest(
|
||||||
|
clientContext = clientContext,
|
||||||
|
product = product,
|
||||||
|
productContext = Some(productContext),
|
||||||
|
maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
|
||||||
|
cursor = view.cursor.filter(_.nonEmpty)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override val fetch: Arrow[(Key, View), Result[Value]] = {
|
||||||
|
val transformThriftIntoPipelineRequest: Arrow[
|
||||||
|
(Key, View),
|
||||||
|
ProductPipelineRequest[HomeMixerRequest]
|
||||||
|
] = {
|
||||||
|
Arrow
|
||||||
|
.identity[(Key, View)]
|
||||||
|
.andThen {
|
||||||
|
createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
|
||||||
|
}
|
||||||
|
.map {
|
||||||
|
case thriftRequest =>
|
||||||
|
val request = homeMixerRequestUnmarshaller(thriftRequest)
|
||||||
|
val params = paramsBuilder.build(
|
||||||
|
clientContext = request.clientContext,
|
||||||
|
product = request.product,
|
||||||
|
featureOverrides =
|
||||||
|
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
|
||||||
|
)
|
||||||
|
ProductPipelineRequest(request, params)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val underlyingProduct: Arrow[
|
||||||
|
ProductPipelineRequest[HomeMixerRequest],
|
||||||
|
ProductPipelineResult[tr.TimelineResponse]
|
||||||
|
] = Arrow
|
||||||
|
.identity[ProductPipelineRequest[HomeMixerRequest]]
|
||||||
|
.map { pipelineRequest =>
|
||||||
|
val pipelineArrow = productPipelineRegistry
|
||||||
|
.getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
|
||||||
|
pipelineRequest.request.product)
|
||||||
|
.arrow
|
||||||
|
(pipelineArrow, pipelineRequest)
|
||||||
|
}.applyArrow
|
||||||
|
|
||||||
|
transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
|
||||||
|
_.result match {
|
||||||
|
case Some(result) => found(result.timeline)
|
||||||
|
case _ => missing
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
object HomeMixerColumn {
|
||||||
|
val Path = "home-mixer/homeMixer.Timeline"
|
||||||
|
private val FetchCallstack = s"$Path:fetch"
|
||||||
|
private val MaxCount: Option[Int] = Some(100)
|
||||||
|
}
|
@ -10,11 +10,8 @@ scala_library(
|
|||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||||
"src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
|
|
||||||
"src/thrift/com/twitter/search:earlybird-scala",
|
"src/thrift/com/twitter/search:earlybird-scala",
|
||||||
"stitch/stitch-timelineservice/src/main/scala",
|
"stitch/stitch-timelineservice/src/main/scala",
|
||||||
"strato/config/columns/recommendations/similarity:similarity-strato-client",
|
|
||||||
"strato/src/main/scala/com/twitter/strato/client",
|
|
||||||
],
|
],
|
||||||
exports = [
|
exports = [
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||||
|
@ -6,13 +6,11 @@ scala_library(
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"finagle/finagle-core/src/main",
|
"finagle/finagle-core/src/main",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"joinkey/src/main/scala/com/twitter/joinkey/context",
|
|
||||||
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
|
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
|
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
|
||||||
@ -25,8 +23,6 @@ scala_library(
|
|||||||
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
|
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
|
||||||
"stringcenter/client",
|
"stringcenter/client",
|
||||||
"stringcenter/client/src/main/java",
|
"stringcenter/client/src/main/java",
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
|
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
|
|
||||||
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -1,6 +1,8 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator
|
||||||
|
|
||||||
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
|
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
|
||||||
|
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
|
||||||
|
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
|
||||||
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
|
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
|
||||||
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator
|
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator
|
||||||
|
@ -8,7 +8,9 @@ scala_library(
|
|||||||
"finagle/finagle-core/src/main",
|
"finagle/finagle-core/src/main",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
|
"joinkey/src/main/scala/com/twitter/joinkey/context",
|
||||||
|
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
|
||||||
@ -18,6 +20,7 @@ scala_library(
|
|||||||
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
|
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
|
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
|
||||||
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.builder
|
||||||
|
|
||||||
import com.twitter.finagle.tracing.Trace
|
import com.twitter.finagle.tracing.Trace
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.builder
|
||||||
|
|
||||||
import com.twitter.bijection.Base64String
|
import com.twitter.bijection.Base64String
|
||||||
import com.twitter.bijection.scrooge.BinaryScalaCodec
|
import com.twitter.bijection.scrooge.BinaryScalaCodec
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient
|
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient
|
@ -1,8 +1,10 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.builder
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.home_mixer.model.HomeFeatures._
|
import com.twitter.home_mixer.model.HomeFeatures._
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
|
||||||
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
|
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
|
||||||
import com.twitter.tweetypie.{thriftscala => tpt}
|
import com.twitter.tweetypie.{thriftscala => tpt}
|
||||||
|
|
||||||
@ -25,74 +27,76 @@ object HomeTweetTypePredicates {
|
|||||||
(
|
(
|
||||||
"has_exclusive_conversation_author_id",
|
"has_exclusive_conversation_author_id",
|
||||||
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
|
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
|
||||||
("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)),
|
("is_eligible_for_connect_boost", _ => false),
|
||||||
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
|
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
|
||||||
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
|
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
|
||||||
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
|
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
|
||||||
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
|
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
|
||||||
(
|
|
||||||
"is_self_thread_tweet",
|
|
||||||
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
|
|
||||||
("get_initial", _.getOrElse(GetInitialFeature, false)),
|
("get_initial", _.getOrElse(GetInitialFeature, false)),
|
||||||
("get_newer", _.getOrElse(GetNewerFeature, false)),
|
("get_newer", _.getOrElse(GetNewerFeature, false)),
|
||||||
("get_middle", _.getOrElse(GetMiddleFeature, false)),
|
("get_middle", _.getOrElse(GetMiddleFeature, false)),
|
||||||
("get_older", _.getOrElse(GetOlderFeature, false)),
|
("get_older", _.getOrElse(GetOlderFeature, false)),
|
||||||
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
|
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
|
||||||
("polling", _.getOrElse(PollingFeature, false)),
|
("polling", _.getOrElse(PollingFeature, false)),
|
||||||
("tls_size_20_plus", _ => false),
|
("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)),
|
||||||
("near_empty", _ => false),
|
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
|
||||||
("ranked_request", _ => false),
|
|
||||||
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
|
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
|
||||||
|
(
|
||||||
|
"less_than_10_mins_since_lnpt",
|
||||||
|
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
|
||||||
|
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
|
||||||
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
|
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
|
||||||
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
|
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
|
||||||
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
|
|
||||||
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
|
|
||||||
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
|
|
||||||
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
|
|
||||||
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
|
|
||||||
(
|
(
|
||||||
"is_signup_request",
|
"conversation_module_has_2_displayed_tweets",
|
||||||
candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
|
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
|
||||||
("empty_request", _ => false),
|
("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)),
|
||||||
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
|
|
||||||
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
|
|
||||||
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
|
|
||||||
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
|
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
|
||||||
(
|
(
|
||||||
"served_size_between_50_and_100",
|
"served_size_between_50_and_100",
|
||||||
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
|
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
|
||||||
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
|
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
|
||||||
|
(
|
||||||
|
"is_self_thread_tweet",
|
||||||
|
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
|
||||||
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
|
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
|
||||||
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
|
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
|
||||||
|
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
|
||||||
|
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
|
||||||
|
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
|
||||||
(
|
(
|
||||||
"account_age_less_than_30_minutes",
|
"account_age_less_than_30_minutes",
|
||||||
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
|
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
|
||||||
|
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
|
||||||
|
(
|
||||||
|
"directed_at_user_is_in_first_degree",
|
||||||
|
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
|
||||||
|
(
|
||||||
|
"has_semantic_core_annotation",
|
||||||
|
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
|
||||||
|
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
|
||||||
(
|
(
|
||||||
"account_age_less_than_1_day",
|
"account_age_less_than_1_day",
|
||||||
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
|
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
|
||||||
(
|
(
|
||||||
"account_age_less_than_7_days",
|
"account_age_less_than_7_days",
|
||||||
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
|
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
|
||||||
(
|
|
||||||
"directed_at_user_is_in_first_degree",
|
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
|
|
||||||
("root_user_is_in_first_degree", _ => false),
|
|
||||||
(
|
|
||||||
"has_semantic_core_annotation",
|
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
|
|
||||||
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
|
|
||||||
(
|
(
|
||||||
"part_of_utt",
|
"part_of_utt",
|
||||||
_.getOrElse(EarlybirdFeature, None)
|
_.getOrElse(EarlybirdFeature, None)
|
||||||
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
|
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
|
||||||
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
|
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
|
||||||
|
(
|
||||||
|
"has_home_latest_request_past_week",
|
||||||
|
_.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
|
||||||
|
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
|
||||||
|
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
|
||||||
|
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
|
||||||
|
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
|
||||||
|
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
|
||||||
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
|
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
|
||||||
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
|
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
|
||||||
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
|
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
|
||||||
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
|
|
||||||
("viewer_is_employee", _ => false),
|
|
||||||
("viewer_is_timelines_employee", _ => false),
|
|
||||||
("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
|
|
||||||
(
|
(
|
||||||
"has_ancestor_authored_by_viewer",
|
"has_ancestor_authored_by_viewer",
|
||||||
candidate =>
|
candidate =>
|
||||||
@ -100,11 +104,6 @@ object HomeTweetTypePredicates {
|
|||||||
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
|
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
|
||||||
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
|
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
|
||||||
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
|
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
|
||||||
(
|
|
||||||
"root_ancestor",
|
|
||||||
candidate =>
|
|
||||||
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
|
|
||||||
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
|
|
||||||
(
|
(
|
||||||
"deep_reply",
|
"deep_reply",
|
||||||
candidate =>
|
candidate =>
|
||||||
@ -119,23 +118,22 @@ object HomeTweetTypePredicates {
|
|||||||
"tweet_age_less_than_15_seconds",
|
"tweet_age_less_than_15_seconds",
|
||||||
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
.exists(_.untilNow <= 15.seconds)),
|
.exists(_.untilNow <= 15.seconds)),
|
||||||
("is_followed_topic_tweet", _ => false),
|
(
|
||||||
("is_recommended_topic_tweet", _ => false),
|
"less_than_1_hour_since_lnpt",
|
||||||
("is_topic_tweet", _ => false),
|
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)),
|
||||||
("preferred_language_matches_tweet_language", _ => false),
|
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
|
||||||
(
|
(
|
||||||
"device_language_matches_tweet_language",
|
"device_language_matches_tweet_language",
|
||||||
candidate =>
|
candidate =>
|
||||||
candidate.getOrElse(TweetLanguageFeature, None) ==
|
candidate.getOrElse(TweetLanguageFeature, None) ==
|
||||||
candidate.getOrElse(DeviceLanguageFeature, None)),
|
candidate.getOrElse(DeviceLanguageFeature, None)),
|
||||||
|
(
|
||||||
|
"root_ancestor",
|
||||||
|
candidate =>
|
||||||
|
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
|
||||||
|
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
|
||||||
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
|
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
|
||||||
("in_network", _.getOrElse(FromInNetworkSourceFeature, true)),
|
("in_network", _.getOrElse(InNetworkFeature, true)),
|
||||||
("viewer_follows_original_author", _ => false),
|
|
||||||
("has_account_follow_prompt", _ => false),
|
|
||||||
("has_relevance_prompt", _ => false),
|
|
||||||
("has_topic_annotation_haug_prompt", _ => false),
|
|
||||||
("has_topic_annotation_random_precision_prompt", _ => false),
|
|
||||||
("has_topic_annotation_prompt", _ => false),
|
|
||||||
(
|
(
|
||||||
"has_political_annotation",
|
"has_political_annotation",
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(
|
_.getOrElse(EarlybirdFeature, None).exists(
|
||||||
@ -153,9 +151,14 @@ object HomeTweetTypePredicates {
|
|||||||
_.getOrElse(EarlybirdFeature, None)
|
_.getOrElse(EarlybirdFeature, None)
|
||||||
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
|
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
|
||||||
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
|
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
|
||||||
("is_viewer_not_invited_to_reply", _ => false),
|
(
|
||||||
("is_viewer_invited_to_reply", _ => false),
|
"is_followed_topic_tweet",
|
||||||
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
|
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
|
||||||
|
.exists(_ == BasicTopicContextFunctionalityType)),
|
||||||
|
(
|
||||||
|
"is_recommended_topic_tweet",
|
||||||
|
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
|
||||||
|
.exists(_ == RecommendationTopicContextFunctionalityType)),
|
||||||
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
|
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
|
||||||
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
|
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
|
||||||
(
|
(
|
||||||
@ -164,8 +167,6 @@ object HomeTweetTypePredicates {
|
|||||||
(
|
(
|
||||||
"has_gte_100k_favs",
|
"has_gte_100k_favs",
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
|
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
|
||||||
("above_neighbor_is_topic_tweet", _ => false),
|
|
||||||
("is_topic_tweet_with_neighbor_below", _ => false),
|
|
||||||
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
|
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
|
||||||
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
|
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
|
||||||
(
|
(
|
||||||
@ -187,6 +188,7 @@ object HomeTweetTypePredicates {
|
|||||||
(
|
(
|
||||||
"has_toxicity_score_above_threshold",
|
"has_toxicity_score_above_threshold",
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
|
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
|
||||||
|
("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
|
||||||
(
|
(
|
||||||
"text_only",
|
"text_only",
|
||||||
candidate =>
|
candidate =>
|
||||||
@ -204,23 +206,50 @@ object HomeTweetTypePredicates {
|
|||||||
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
|
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
|
||||||
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
|
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
|
||||||
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
|
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
|
||||||
("3_or_more_consecutive_not_in_network", _ => false),
|
|
||||||
("2_or_more_consecutive_not_in_network", _ => false),
|
|
||||||
("5_out_of_7_not_in_network", _ => false),
|
|
||||||
("7_out_of_7_not_in_network", _ => false),
|
|
||||||
("5_out_of_5_not_in_network", _ => false),
|
|
||||||
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
|
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
|
||||||
("has_liked_by_social_context", _ => false),
|
|
||||||
("has_followed_by_social_context", _ => false),
|
|
||||||
("has_topic_social_context", _ => false),
|
|
||||||
("timeline_entry_has_banner", _ => false),
|
|
||||||
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
|
|
||||||
(
|
(
|
||||||
"conversation_module_has_2_displayed_tweets",
|
"has_liked_by_social_context",
|
||||||
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
|
candidateFeatures =>
|
||||||
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
|
candidateFeatures
|
||||||
("served_in_recap_tweet_candidate_module_injection", _ => false),
|
.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
|
||||||
("served_in_threaded_conversation_module", _ => false)
|
.exists(candidateFeatures
|
||||||
|
.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
|
||||||
|
(
|
||||||
|
"has_followed_by_social_context",
|
||||||
|
_.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
|
||||||
|
(
|
||||||
|
"has_topic_social_context",
|
||||||
|
candidateFeatures =>
|
||||||
|
candidateFeatures
|
||||||
|
.getOrElse(TopicIdSocialContextFeature, None)
|
||||||
|
.isDefined &&
|
||||||
|
candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
|
||||||
|
("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
|
||||||
|
(
|
||||||
|
"video_bt_10_60_sec",
|
||||||
|
_.getOrElse(VideoDurationMsFeature, None).exists(duration =>
|
||||||
|
duration > 10000 && duration <= 60000)),
|
||||||
|
("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
|
||||||
|
(
|
||||||
|
"tweet_age_lte_30_minutes",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow <= 30.minutes)),
|
||||||
|
(
|
||||||
|
"tweet_age_lte_1_hour",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow <= 1.hour)),
|
||||||
|
(
|
||||||
|
"tweet_age_lte_6_hours",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow <= 6.hours)),
|
||||||
|
(
|
||||||
|
"tweet_age_lte_12_hours",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow <= 12.hours)),
|
||||||
|
(
|
||||||
|
"tweet_age_gte_24_hours",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow >= 24.hours)),
|
||||||
)
|
)
|
||||||
|
|
||||||
val PredicateMap = CandidatePredicates.toMap
|
val PredicateMap = CandidatePredicates.toMap
|
@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
|
|||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||||
|
|
||||||
object ListClientEventDetailsBuilder
|
case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
|
||||||
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
|
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
|
||||||
|
|
||||||
override def apply(
|
override def apply(
|
||||||
@ -20,7 +20,7 @@ object ListClientEventDetailsBuilder
|
|||||||
conversationDetails = None,
|
conversationDetails = None,
|
||||||
timelinesDetails = Some(
|
timelinesDetails = Some(
|
||||||
TimelinesDetails(
|
TimelinesDetails(
|
||||||
injectionType = Some(st.SuggestType.OrganicListTweet.name),
|
injectionType = Some(suggestType.name),
|
||||||
controllerData = None,
|
controllerData = None,
|
||||||
sourceData = None)),
|
sourceData = None)),
|
||||||
articleDetails = None,
|
articleDetails = None,
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
@ -9,7 +9,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
|
|||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
import com.twitter.timelines.service.{thriftscala => t}
|
import com.twitter.timelines.service.{thriftscala => t}
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -4,9 +4,16 @@ scala_library(
|
|||||||
strict_deps = True,
|
strict_deps = True,
|
||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
|
||||||
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
|
||||||
"src/thrift/com/twitter/timelines/service:thrift-scala",
|
"src/thrift/com/twitter/timelines/service:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||||
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||||
@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
@ -17,7 +17,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
|
|||||||
import com.twitter.timelineservice.model.FeedbackInfo
|
import com.twitter.timelineservice.model.FeedbackInfo
|
||||||
import com.twitter.timelineservice.model.FeedbackMetadata
|
import com.twitter.timelineservice.model.FeedbackMetadata
|
||||||
import com.twitter.timelineservice.{thriftscala => tls}
|
import com.twitter.timelineservice.{thriftscala => tls}
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
|
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature
|
@ -0,0 +1,18 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
|
import com.twitter.stringcenter.client.ExternalStringRegistry
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Provider
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
class FeedbackStrings @Inject() (
|
||||||
|
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
|
||||||
|
private val externalStringRegistry = externalStringRegistryProvider.get()
|
||||||
|
|
||||||
|
val seeLessOftenFeedbackString =
|
||||||
|
externalStringRegistry.createProdString("Feedback.seeLessOften")
|
||||||
|
val seeLessOftenConfirmationFeedbackString =
|
||||||
|
externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
|
||||||
|
}
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
|
|||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.timelines.service.{thriftscala => t}
|
import com.twitter.timelines.service.{thriftscala => t}
|
||||||
import com.twitter.timelines.util.FeedbackMetadataSerializer
|
import com.twitter.timelines.util.FeedbackMetadataSerializer
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
|
||||||
@ -14,10 +14,13 @@ import javax.inject.Singleton
|
|||||||
@Singleton
|
@Singleton
|
||||||
case class HomeTweetSocialContextBuilder @Inject() (
|
case class HomeTweetSocialContextBuilder @Inject() (
|
||||||
likedBySocialContextBuilder: LikedBySocialContextBuilder,
|
likedBySocialContextBuilder: LikedBySocialContextBuilder,
|
||||||
|
listsSocialContextBuilder: ListsSocialContextBuilder,
|
||||||
followedBySocialContextBuilder: FollowedBySocialContextBuilder,
|
followedBySocialContextBuilder: FollowedBySocialContextBuilder,
|
||||||
topicSocialContextBuilder: TopicSocialContextBuilder,
|
topicSocialContextBuilder: TopicSocialContextBuilder,
|
||||||
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
|
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
|
||||||
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
|
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder,
|
||||||
|
popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
|
||||||
|
popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
|
||||||
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
def apply(
|
def apply(
|
||||||
@ -31,6 +34,9 @@ case class HomeTweetSocialContextBuilder @Inject() (
|
|||||||
likedBySocialContextBuilder(query, candidate, features)
|
likedBySocialContextBuilder(query, candidate, features)
|
||||||
.orElse(followedBySocialContextBuilder(query, candidate, features))
|
.orElse(followedBySocialContextBuilder(query, candidate, features))
|
||||||
.orElse(topicSocialContextBuilder(query, candidate, features))
|
.orElse(topicSocialContextBuilder(query, candidate, features))
|
||||||
|
.orElse(popularVideoSocialContextBuilder(query, candidate, features))
|
||||||
|
.orElse(listsSocialContextBuilder(query, candidate, features))
|
||||||
|
.orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
|
||||||
case Some(_) =>
|
case Some(_) =>
|
||||||
val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
|
val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
|
||||||
// Only hydrate the social context into the root tweet in a conversation module
|
// Only hydrate the social context into the root tweet in a conversation module
|
@ -5,7 +5,6 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
|
|||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.ExternalStringRegistry
|
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Provider
|
import javax.inject.Provider
|
||||||
@ -32,12 +31,13 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
|
|||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
|
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
|
||||||
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
|
feedbackStrings: FeedbackStrings,
|
||||||
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
|
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
|
||||||
|
|
||||||
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
|
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
|
||||||
externalStringRegistry = externalStringRegistryProvider.get(),
|
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
|
||||||
|
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
|
||||||
stringCenter = stringCenterProvider.get(),
|
stringCenter = stringCenterProvider.get(),
|
||||||
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
|
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
|
||||||
)
|
)
|
||||||
|
@ -0,0 +1,52 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
|
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
|
||||||
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Provider
|
||||||
|
import javax.inject.Singleton
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.timelines.service.{thriftscala => tl}
|
||||||
|
import com.twitter.timelines.util.FeedbackRequestSerializer
|
||||||
|
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
|
||||||
|
import com.twitter.timelineservice.thriftscala.FeedbackType
|
||||||
|
|
||||||
|
object HomeWhoToSubscribeFeedbackActionInfoBuilder {
|
||||||
|
private val FeedbackMetadata = tl.FeedbackMetadata(
|
||||||
|
injectionType = Some(SuggestType.WhoToSubscribe),
|
||||||
|
engagementType = None,
|
||||||
|
entityIds = Seq.empty,
|
||||||
|
ttlMs = None
|
||||||
|
)
|
||||||
|
private val FeedbackRequest =
|
||||||
|
tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
|
||||||
|
private val EncodedFeedbackRequest =
|
||||||
|
FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
|
||||||
|
feedbackStrings: FeedbackStrings,
|
||||||
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
|
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
|
||||||
|
|
||||||
|
private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
|
||||||
|
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
|
||||||
|
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
|
||||||
|
stringCenter = stringCenterProvider.get(),
|
||||||
|
encodedFeedbackRequest =
|
||||||
|
Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
|
||||||
|
)
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidate: UserCandidate,
|
||||||
|
candidateFeatures: FeatureMap
|
||||||
|
): Option[FeedbackActionInfo] =
|
||||||
|
whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
|
||||||
|
}
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature
|
@ -0,0 +1,50 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
|
||||||
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
import com.twitter.timelineservice.suggests.{thriftscala => t}
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Provider
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
/**
|
||||||
|
* "Your Lists" will be rendered for the context and a url link for your lists.
|
||||||
|
*/
|
||||||
|
@Singleton
|
||||||
|
case class ListsSocialContextBuilder @Inject() (
|
||||||
|
externalStrings: HomeMixerExternalStrings,
|
||||||
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
|
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
|
private val stringCenter = stringCenterProvider.get()
|
||||||
|
private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
|
||||||
|
|
||||||
|
def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidate: TweetCandidate,
|
||||||
|
candidateFeatures: FeatureMap
|
||||||
|
): Option[SocialContext] = {
|
||||||
|
candidateFeatures.get(SuggestTypeFeature) match {
|
||||||
|
case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
|
||||||
|
val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
|
||||||
|
Some(
|
||||||
|
GeneralContext(
|
||||||
|
contextType = ListGeneralContextType,
|
||||||
|
text = stringCenter.prepare(listString),
|
||||||
|
url = userName.map(name => ""),
|
||||||
|
contextImageUrls = None,
|
||||||
|
landingUrl = None
|
||||||
|
))
|
||||||
|
case _ => None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||||
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
|
||||||
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
@ -12,7 +12,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
|
|||||||
import com.twitter.timelineservice.model.FeedbackInfo
|
import com.twitter.timelineservice.model.FeedbackInfo
|
||||||
import com.twitter.timelineservice.model.FeedbackMetadata
|
import com.twitter.timelineservice.model.FeedbackMetadata
|
||||||
import com.twitter.timelineservice.{thriftscala => tlst}
|
import com.twitter.timelineservice.{thriftscala => tlst}
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -0,0 +1,43 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Provider
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
case class PopularInYourAreaSocialContextBuilder @Inject() (
|
||||||
|
externalStrings: HomeMixerExternalStrings,
|
||||||
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
|
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
|
private val stringCenter = stringCenterProvider.get()
|
||||||
|
private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
|
||||||
|
|
||||||
|
def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidate: TweetCandidate,
|
||||||
|
candidateFeatures: FeatureMap
|
||||||
|
): Option[SocialContext] = {
|
||||||
|
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
|
||||||
|
if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
|
||||||
|
Some(
|
||||||
|
GeneralContext(
|
||||||
|
contextType = LocationGeneralContextType,
|
||||||
|
text = stringCenter.prepare(popularInYourAreaString),
|
||||||
|
url = None,
|
||||||
|
contextImageUrls = None,
|
||||||
|
landingUrl = None
|
||||||
|
))
|
||||||
|
} else None
|
||||||
|
}
|
||||||
|
}
|
@ -1,50 +1,48 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
|
||||||
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Provider
|
import javax.inject.Provider
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
/**
|
|
||||||
* Renders a fixed 'You Might Like' string above all OON Tweets.
|
|
||||||
*/
|
|
||||||
@Singleton
|
@Singleton
|
||||||
case class YouMightLikeSocialContextBuilder @Inject() (
|
case class PopularVideoSocialContextBuilder @Inject() (
|
||||||
externalStrings: HomeMixerExternalStrings,
|
externalStrings: HomeMixerExternalStrings,
|
||||||
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
private val stringCenter = stringCenterProvider.get()
|
private val stringCenter = stringCenterProvider.get()
|
||||||
private val youMightLikeString = externalStrings.socialContextYouMightLikeString
|
private val popularVideoString = externalStrings.socialContextPopularVideoString
|
||||||
|
|
||||||
def apply(
|
def apply(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
candidate: TweetCandidate,
|
candidate: TweetCandidate,
|
||||||
candidateFeatures: FeatureMap
|
candidateFeatures: FeatureMap
|
||||||
): Option[SocialContext] = {
|
): Option[SocialContext] = {
|
||||||
val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true)
|
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
|
||||||
val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false)
|
if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) {
|
||||||
if (!isInNetwork && !isRetweet) {
|
|
||||||
Some(
|
Some(
|
||||||
GeneralContext(
|
GeneralContext(
|
||||||
contextType = SparkleGeneralContextType,
|
contextType = SparkleGeneralContextType,
|
||||||
text = stringCenter.prepare(youMightLikeString),
|
text = stringCenter.prepare(popularVideoString),
|
||||||
url = None,
|
url = None,
|
||||||
contextImageUrls = None,
|
contextImageUrls = None,
|
||||||
landingUrl = None
|
landingUrl = Some(
|
||||||
|
Url(
|
||||||
|
urlType = DeepLink,
|
||||||
|
url = ""
|
||||||
|
)
|
||||||
|
)
|
||||||
))
|
))
|
||||||
} else {
|
} else None
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
@ -8,7 +8,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||||
@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
|
|||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
import com.twitter.timelines.service.{thriftscala => t}
|
import com.twitter.timelines.service.{thriftscala => t}
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -4,95 +4,56 @@ scala_library(
|
|||||||
strict_deps = True,
|
strict_deps = True,
|
||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
|
||||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||||
"configapi/configapi-decider",
|
"configapi/configapi-decider",
|
||||||
"finatra/inject/inject-core/src/main/scala",
|
"finatra/inject/inject-core/src/main/scala",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
|
|
||||||
"joinkey/src/main/scala/com/twitter/joinkey/context",
|
"joinkey/src/main/scala/com/twitter/joinkey/context",
|
||||||
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
|
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
|
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
|
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
|
||||||
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
|
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
|
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
||||||
"representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
|
|
||||||
"representation-scorer/server/src/main/thrift:thrift-scala",
|
|
||||||
"servo/repo/src/main/scala",
|
|
||||||
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
||||||
"src/java/com/twitter/ml/api/constant",
|
|
||||||
"src/java/com/twitter/search/common/util/lang",
|
"src/java/com/twitter/search/common/util/lang",
|
||||||
"src/scala/com/twitter/ml/api/util",
|
"src/scala/com/twitter/timelines/prediction/adapters/request_context",
|
||||||
"src/scala/com/twitter/timelines/prediction/adapters/real_graph",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/adapters/twistly",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/common/util",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/features/common",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/features/recap",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/features/time_features",
|
|
||||||
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||||
"src/thrift/com/twitter/ml/api:data-java",
|
|
||||||
"src/thrift/com/twitter/ml/api:embedding-java",
|
|
||||||
"src/thrift/com/twitter/onboarding/relevance/features:features-java",
|
|
||||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||||
"src/thrift/com/twitter/search:earlybird-scala",
|
"src/thrift/com/twitter/search:earlybird-scala",
|
||||||
"src/thrift/com/twitter/search/common:constants-java",
|
"src/thrift/com/twitter/search/common:constants-java",
|
||||||
"src/thrift/com/twitter/socialgraph:thrift-scala",
|
"src/thrift/com/twitter/socialgraph:thrift-scala",
|
||||||
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
|
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
|
||||||
"src/thrift/com/twitter/timelineranker:thrift-scala",
|
"src/thrift/com/twitter/timelineranker:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelines/author_features:thrift-java",
|
|
||||||
"src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
|
|
||||||
"src/thrift/com/twitter/timelines/impression:thrift-scala",
|
"src/thrift/com/twitter/timelines/impression:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
|
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
|
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
|
||||||
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
|
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
|
||||||
"src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
|
|
||||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||||
"src/thrift/com/twitter/user_session_store:thrift-java",
|
"src/thrift/com/twitter/user_session_store:thrift-java",
|
||||||
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
|
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
|
||||||
"src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
|
|
||||||
"stitch/stitch-core",
|
"stitch/stitch-core",
|
||||||
"stitch/stitch-gizmoduck",
|
"stitch/stitch-gizmoduck",
|
||||||
"stitch/stitch-socialgraph",
|
"stitch/stitch-socialgraph",
|
||||||
"stitch/stitch-timelineservice",
|
"stitch/stitch-timelineservice",
|
||||||
"stitch/stitch-tweetypie",
|
"stitch/stitch-tweetypie",
|
||||||
"strato/config/columns/topic-signals/tsp",
|
|
||||||
"strato/config/columns/topic-signals/tsp:tsp-strato-client",
|
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
|
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
|
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
|
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly",
|
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
|
|
||||||
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
|
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
|
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
|
||||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
||||||
"topic-social-proof/server/src/main/thrift:thrift-scala",
|
|
||||||
"topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
|
|
||||||
"tweetconvosvc/thrift/src/main/thrift:thrift-scala",
|
|
||||||
"twitter-config/yaml",
|
|
||||||
"user_session_store/src/main/scala/com/twitter/user_session_store",
|
"user_session_store/src/main/scala/com/twitter/user_session_store",
|
||||||
"util/util-core",
|
"util/util-core",
|
||||||
],
|
],
|
||||||
|
@ -1,12 +1,10 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
package com.twitter.home_mixer.functional_component.feature_hydrator
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
|
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
import com.twitter.product_mixer.core.feature.Feature
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
@ -17,16 +15,12 @@ import javax.inject.Singleton
|
|||||||
@Singleton
|
@Singleton
|
||||||
case class FeedbackHistoryQueryFeatureHydrator @Inject() (
|
case class FeedbackHistoryQueryFeatureHydrator @Inject() (
|
||||||
feedbackHistoryClient: FeedbackHistoryManhattanClient)
|
feedbackHistoryClient: FeedbackHistoryManhattanClient)
|
||||||
extends QueryFeatureHydrator[PipelineQuery]
|
extends QueryFeatureHydrator[PipelineQuery] {
|
||||||
with Conditionally[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
|
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
|
||||||
|
|
||||||
override def onlyIf(query: PipelineQuery): Boolean =
|
|
||||||
query.params(EnableFeedbackFatigueParam)
|
|
||||||
|
|
||||||
override def hydrate(
|
override def hydrate(
|
||||||
query: PipelineQuery
|
query: PipelineQuery
|
||||||
): Stitch[FeatureMap] =
|
): Stitch[FeatureMap] =
|
||||||
|
@ -1,41 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
|
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
case class FollowedTopicsQueryFeatureHydrator @Inject() (
|
|
||||||
followedTopicsCandidateSource: FollowedTopicsCandidateSource)
|
|
||||||
extends QueryFeatureHydrator[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
|
|
||||||
|
|
||||||
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
|
|
||||||
val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
|
|
||||||
followedTopicsCandidateSource(request)
|
|
||||||
.map { topics =>
|
|
||||||
FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
|
|
||||||
}.handle {
|
|
||||||
case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override val alerts = Seq(
|
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
|
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
|
|
||||||
)
|
|
||||||
}
|
|
@ -1,58 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.gizmoduck.{thriftscala => gt}
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
|
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.stitch.gizmoduck.Gizmoduck
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
|
|
||||||
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
|
|
||||||
with Conditionally[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier =
|
|
||||||
FeatureHydratorIdentifier("GizmoduckAuthorSafety")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
|
|
||||||
|
|
||||||
override def onlyIf(query: PipelineQuery): Boolean =
|
|
||||||
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
|
|
||||||
|
|
||||||
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidate: TweetCandidate,
|
|
||||||
existingFeatures: FeatureMap
|
|
||||||
): Stitch[FeatureMap] = {
|
|
||||||
val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
|
|
||||||
|
|
||||||
val blueVerifiedStitch = authorIdOption
|
|
||||||
.map { authorId =>
|
|
||||||
gizmoduck
|
|
||||||
.getUserById(
|
|
||||||
userId = authorId,
|
|
||||||
queryFields = queryFields
|
|
||||||
)
|
|
||||||
.map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
|
|
||||||
}.getOrElse(Stitch.False)
|
|
||||||
|
|
||||||
blueVerifiedStitch.map { isBlueVerified =>
|
|
||||||
FeatureMapBuilder()
|
|
||||||
.add(AuthorIsBlueVerifiedFeature, isBlueVerified)
|
|
||||||
.build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
|
|||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
|
||||||
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
||||||
|
import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
import com.twitter.product_mixer.core.feature.Feature
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
@ -11,7 +12,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
|
|||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
|
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
|
||||||
|
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
|
||||||
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
|
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
@ -19,32 +21,35 @@ import javax.inject.Singleton
|
|||||||
@Singleton
|
@Singleton
|
||||||
case class ImpressionBloomFilterQueryFeatureHydrator[
|
case class ImpressionBloomFilterQueryFeatureHydrator[
|
||||||
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
|
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
|
||||||
bloomFilter: ImpressionBloomFilter)
|
bloomFilterClient: ManhattanStoreClient[
|
||||||
extends QueryFeatureHydrator[Query] {
|
blm.ImpressionBloomFilterKey,
|
||||||
|
blm.ImpressionBloomFilterSeq
|
||||||
|
]) extends QueryFeatureHydrator[Query] {
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||||
"ImpressionBloomFilter")
|
"ImpressionBloomFilter")
|
||||||
|
|
||||||
private val ImpressionBloomFilterTTL = 7.day
|
private val ImpressionBloomFilterTTL = 7.day
|
||||||
private val ImpressionBloomFilterFalsePositiveRate = 0.002
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
|
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
|
||||||
|
|
||||||
private val SurfaceArea = t.SurfaceArea.HomeTimeline
|
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
|
||||||
|
|
||||||
override def hydrate(query: Query): Stitch[FeatureMap] = {
|
override def hydrate(query: Query): Stitch[FeatureMap] = {
|
||||||
val userId = query.getRequiredUserId
|
val userId = query.getRequiredUserId
|
||||||
bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq =>
|
bloomFilterClient
|
||||||
|
.get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
|
||||||
|
.map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
|
||||||
|
.map { bloomFilterSeq =>
|
||||||
val updatedBloomFilterSeq =
|
val updatedBloomFilterSeq =
|
||||||
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
|
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
|
||||||
else {
|
else {
|
||||||
bloomFilter.addElements(
|
ImpressionBloomFilter.addSeenTweetIds(
|
||||||
userId = userId,
|
|
||||||
surfaceArea = SurfaceArea,
|
surfaceArea = SurfaceArea,
|
||||||
tweetIds = query.seenTweetIds.get,
|
tweetIds = query.seenTweetIds.get,
|
||||||
bloomFilterEntrySeq = bloomFilterSeq,
|
bloomFilterSeq = bloomFilterSeq,
|
||||||
timeToLive = ImpressionBloomFilterTTL,
|
timeToLive = ImpressionBloomFilterTTL,
|
||||||
falsePositiveRate = ImpressionBloomFilterFalsePositiveRate
|
falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
|
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
|
||||||
|
@ -0,0 +1,41 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.feature_hydrator
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
|
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.feature.Feature
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||||
|
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
|
||||||
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
|
||||||
|
object InNetworkFeatureHydrator
|
||||||
|
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
|
||||||
|
|
||||||
|
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
|
): Stitch[Seq[FeatureMap]] = {
|
||||||
|
val viewerId = query.getRequiredUserId
|
||||||
|
val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
|
||||||
|
|
||||||
|
val featureMaps = candidates.map { candidate =>
|
||||||
|
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
|
||||||
|
val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
|
||||||
|
// Users cannot follow themselves but this is in network by definition
|
||||||
|
val isSelfTweet = authorId == viewerId
|
||||||
|
isSelfTweet || followedUserIds.contains(authorId)
|
||||||
|
}
|
||||||
|
FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
|
||||||
|
}
|
||||||
|
Stitch.value(featureMaps)
|
||||||
|
}
|
||||||
|
}
|
@ -2,8 +2,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
|
|||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
|
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
|
||||||
|
import com.twitter.finagle.stats.StatsReceiver
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
|
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
@ -27,17 +29,27 @@ import javax.inject.Singleton
|
|||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
case class PersistenceStoreQueryFeatureHydrator @Inject() (
|
case class PersistenceStoreQueryFeatureHydrator @Inject() (
|
||||||
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
|
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3],
|
||||||
|
statsReceiver: StatsReceiver)
|
||||||
extends QueryFeatureHydrator[PipelineQuery] {
|
extends QueryFeatureHydrator[PipelineQuery] {
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
|
||||||
|
|
||||||
|
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
|
||||||
|
private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
|
||||||
|
|
||||||
private val WhoToFollowExcludedUserIdsLimit = 1000
|
private val WhoToFollowExcludedUserIdsLimit = 1000
|
||||||
private val ServedTweetIdsDuration = 1.hour
|
private val ServedTweetIdsDuration = 10.minutes
|
||||||
private val ServedTweetIdsLimit = 100
|
private val ServedTweetIdsLimit = 100
|
||||||
|
private val ServedTweetPreviewIdsDuration = 10.hours
|
||||||
|
private val ServedTweetPreviewIdsLimit = 10
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] =
|
override val features: Set[Feature[_, _]] =
|
||||||
Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature)
|
Set(
|
||||||
|
ServedTweetIdsFeature,
|
||||||
|
ServedTweetPreviewIdsFeature,
|
||||||
|
PersistenceEntriesFeature,
|
||||||
|
WhoToFollowExcludedUserIdsFeature)
|
||||||
|
|
||||||
private val supportedClients = Seq(
|
private val supportedClients = Seq(
|
||||||
ClientPlatform.IPhone,
|
ClientPlatform.IPhone,
|
||||||
@ -80,8 +92,19 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
|
|||||||
.flatMap(
|
.flatMap(
|
||||||
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
|
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
|
||||||
|
|
||||||
|
servedTweetIdsSizeStat.add(servedTweetIds.size)
|
||||||
|
|
||||||
|
val servedTweetPreviewIds = timelineResponses
|
||||||
|
.filter(_.clientPlatform == clientPlatform)
|
||||||
|
.filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
|
||||||
|
.sortBy(-_.servedTime.inMilliseconds)
|
||||||
|
.flatMap(_.entries
|
||||||
|
.filter(_.entityIdType == EntityIdType.TweetPreview)
|
||||||
|
.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
|
||||||
|
|
||||||
FeatureMapBuilder()
|
FeatureMapBuilder()
|
||||||
.add(ServedTweetIdsFeature, servedTweetIds)
|
.add(ServedTweetIdsFeature, servedTweetIds)
|
||||||
|
.add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
|
||||||
.add(PersistenceEntriesFeature, timelineResponses)
|
.add(PersistenceEntriesFeature, timelineResponses)
|
||||||
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
|
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
|
||||||
.build()
|
.build()
|
||||||
|
@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
|
|||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.util.OffloadFuturePools
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.stitch.timelineservice.TimelineService
|
import com.twitter.stitch.timelineservice.TimelineService
|
||||||
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
|
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
|
||||||
@ -37,10 +38,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
|
|||||||
override def apply(
|
override def apply(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
|
||||||
val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
|
val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
|
||||||
candidate.features
|
candidate.features
|
||||||
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
|
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
|
||||||
.map(favoritedBy => favoritedBy -> candidate.candidate.id)
|
.map(favoritedBy => favoritedBy -> candidate.candidate.id)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -59,7 +60,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
|
|||||||
|
|
||||||
candidates.map { candidate =>
|
candidates.map { candidate =>
|
||||||
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
|
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
|
||||||
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
|
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
|
||||||
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
|
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
|
||||||
|
|
||||||
FeatureMapBuilder()
|
FeatureMapBuilder()
|
||||||
|
@ -32,11 +32,15 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
|
|||||||
val realGraphScoresFeatures = realGraphFollowedUsers
|
val realGraphScoresFeatures = realGraphFollowedUsers
|
||||||
.getOrElse(Seq.empty)
|
.getOrElse(Seq.empty)
|
||||||
.sortBy(-_.score)
|
.sortBy(-_.score)
|
||||||
.map(candidate => candidate.userId -> candidate.score)
|
.map(candidate => candidate.userId -> scaleScore(candidate.score))
|
||||||
.take(RealGraphCandidateCount)
|
.take(RealGraphCandidateCount)
|
||||||
.toMap
|
.toMap
|
||||||
|
|
||||||
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
|
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
|
||||||
|
private def scaleScore(score: Double): Double =
|
||||||
|
if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
|
||||||
}
|
}
|
||||||
|
@ -17,9 +17,13 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
|
||||||
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
|
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
|
||||||
|
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||||
import com.twitter.search.common.util.lang.ThriftLanguageUtil
|
import com.twitter.search.common.util.lang.ThriftLanguageUtil
|
||||||
import com.twitter.snowflake.id.SnowflakeId
|
import com.twitter.snowflake.id.SnowflakeId
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
|
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
|
||||||
|
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
|
||||||
import java.util.UUID
|
import java.util.UUID
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
@ -45,6 +49,9 @@ class RequestQueryFeatureHydrator[
|
|||||||
PullToRefreshFeature,
|
PullToRefreshFeature,
|
||||||
RequestJoinIdFeature,
|
RequestJoinIdFeature,
|
||||||
ServedRequestIdFeature,
|
ServedRequestIdFeature,
|
||||||
|
TimestampFeature,
|
||||||
|
TimestampGMTDowFeature,
|
||||||
|
TimestampGMTHourFeature,
|
||||||
ViewerIdFeature
|
ViewerIdFeature
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -67,6 +74,7 @@ class RequestQueryFeatureHydrator[
|
|||||||
override def hydrate(query: Query): Stitch[FeatureMap] = {
|
override def hydrate(query: Query): Stitch[FeatureMap] = {
|
||||||
val requestContext = query.deviceContext.flatMap(_.requestContextValue)
|
val requestContext = query.deviceContext.flatMap(_.requestContextValue)
|
||||||
val servedRequestId = UUID.randomUUID.getMostSignificantBits
|
val servedRequestId = UUID.randomUUID.getMostSignificantBits
|
||||||
|
val timestamp = query.queryTime.inMilliseconds
|
||||||
|
|
||||||
val featureMap = FeatureMapBuilder()
|
val featureMap = FeatureMapBuilder()
|
||||||
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
|
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
|
||||||
@ -97,8 +105,15 @@ class RequestQueryFeatureHydrator[
|
|||||||
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
|
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
|
||||||
.add(ServedRequestIdFeature, Some(servedRequestId))
|
.add(ServedRequestIdFeature, Some(servedRequestId))
|
||||||
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
|
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
|
||||||
|
.add(TimestampFeature, timestamp)
|
||||||
|
.add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
|
||||||
|
.add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
|
||||||
.add(HasDarkRequestFeature, hasDarkRequest)
|
.add(HasDarkRequestFeature, hasDarkRequest)
|
||||||
.add(ViewerIdFeature, query.getRequiredUserId)
|
.add(
|
||||||
|
ViewerIdFeature,
|
||||||
|
query.getOptionalUserId
|
||||||
|
.orElse(query.getGuestId).getOrElse(
|
||||||
|
throw PipelineFailure(BadRequest, "Missing viewer id")))
|
||||||
.build()
|
.build()
|
||||||
|
|
||||||
Stitch.value(featureMap)
|
Stitch.value(featureMap)
|
||||||
|
@ -1,46 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.socialgraph.{thriftscala => sg}
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
|
|
||||||
socialGraphStitchClient: SocialGraphStitchClient)
|
|
||||||
extends QueryFeatureHydrator[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier =
|
|
||||||
FeatureHydratorIdentifier("SGSFollowedUsers")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
|
|
||||||
|
|
||||||
private val SocialGraphLimit = 14999
|
|
||||||
|
|
||||||
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
|
|
||||||
val userId = query.getRequiredUserId
|
|
||||||
|
|
||||||
val request = sg.IdsRequest(
|
|
||||||
relationships = Seq(
|
|
||||||
sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
|
|
||||||
sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
|
|
||||||
),
|
|
||||||
pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
|
|
||||||
)
|
|
||||||
|
|
||||||
socialGraphStitchClient
|
|
||||||
.ids(request).map(_.ids)
|
|
||||||
.map { followedUsers =>
|
|
||||||
FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
|
|||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.util.OffloadFuturePools
|
||||||
import com.twitter.socialgraph.{thriftscala => sg}
|
import com.twitter.socialgraph.{thriftscala => sg}
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.stitch.socialgraph.SocialGraph
|
import com.twitter.stitch.socialgraph.SocialGraph
|
||||||
@ -41,8 +42,7 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
|
|||||||
override def apply(
|
override def apply(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
|
||||||
|
|
||||||
val allSocialContextUserIds =
|
val allSocialContextUserIds =
|
||||||
candidates.flatMap { candidate =>
|
candidates.flatMap { candidate =>
|
||||||
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++
|
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++
|
||||||
|
@ -1,67 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.socialgraph.{thriftscala => sg}
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
|
|
||||||
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier =
|
|
||||||
FeatureHydratorIdentifier("SocialGraphService")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
|
||||||
val viewerId = query.getRequiredUserId
|
|
||||||
|
|
||||||
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
|
|
||||||
val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
|
|
||||||
val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
|
|
||||||
|
|
||||||
val idsRequest = createIdsRequest(
|
|
||||||
userId = viewerId,
|
|
||||||
relationshipTypes = Set(sg.RelationshipType.Following),
|
|
||||||
targetIds = Some(distinctNonSelfAuthorIds)
|
|
||||||
)
|
|
||||||
|
|
||||||
socialGraphStitchClient
|
|
||||||
.ids(request = idsRequest, requestContext = None)
|
|
||||||
.map { idResult =>
|
|
||||||
authorIds.map { authorId =>
|
|
||||||
// Users cannot follow themselves but this is in network by definition
|
|
||||||
val isSelfTweet = authorId == viewerId
|
|
||||||
val inNetworkAuthorIds = idResult.ids.toSet
|
|
||||||
val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
|
|
||||||
FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def createIdsRequest(
|
|
||||||
userId: Long,
|
|
||||||
relationshipTypes: Set[sg.RelationshipType],
|
|
||||||
targetIds: Option[Seq[Long]] = None
|
|
||||||
): sg.IdsRequest = sg.IdsRequest(
|
|
||||||
relationshipTypes.map { relationshipType =>
|
|
||||||
sg.SrcRelationship(userId, relationshipType, targets = targetIds)
|
|
||||||
}.toSeq,
|
|
||||||
Some(sg.PageRequest(selectAll = Some(true)))
|
|
||||||
)
|
|
||||||
}
|
|
@ -1,162 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.contentrecommender.{thriftscala => cr}
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
|
|
||||||
import com.twitter.ml.api.DataRecord
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
|
||||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
|
|
||||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
|
|
||||||
import com.twitter.topiclisting.TopicListingViewerContext
|
|
||||||
import com.twitter.tsp.{thriftscala => tsp}
|
|
||||||
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
import scala.collection.JavaConverters._
|
|
||||||
|
|
||||||
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
|
|
||||||
object TSPInferredTopicDataRecordFeature
|
|
||||||
extends DataRecordInAFeature[TweetCandidate]
|
|
||||||
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
|
|
||||||
override def defaultValue: DataRecord = new DataRecord()
|
|
||||||
}
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class TSPInferredTopicFeatureHydrator @Inject() (
|
|
||||||
topicSocialProofClientColumn: TopicSocialProofClientColumn,
|
|
||||||
statsReceiver: StatsReceiver,
|
|
||||||
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] =
|
|
||||||
Set(
|
|
||||||
TSPInferredTopicFeature,
|
|
||||||
TSPInferredTopicDataRecordFeature,
|
|
||||||
TopicIdSocialContextFeature,
|
|
||||||
TopicContextFunctionalityTypeFeature)
|
|
||||||
|
|
||||||
private val topK = 3
|
|
||||||
|
|
||||||
private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
|
|
||||||
sid.CandidateTweetSourceId.Simcluster,
|
|
||||||
sid.CandidateTweetSourceId.CroonTweet
|
|
||||||
)
|
|
||||||
|
|
||||||
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
|
|
||||||
private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
|
|
||||||
private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
|
|
||||||
private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
|
|
||||||
|
|
||||||
private val DefaultFeatureMap = FeatureMapBuilder()
|
|
||||||
.add(TSPInferredTopicFeature, Map.empty[Long, Double])
|
|
||||||
.add(TSPInferredTopicDataRecordFeature, new DataRecord())
|
|
||||||
.add(TopicIdSocialContextFeature, None)
|
|
||||||
.add(TopicContextFunctionalityTypeFeature, None)
|
|
||||||
.build()
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
|
||||||
val tags = candidates.collect {
|
|
||||||
case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
|
|
||||||
candidate.candidate.id -> candidate.features
|
|
||||||
.getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
|
|
||||||
}.toMap
|
|
||||||
|
|
||||||
val topicSocialProofRequest =
|
|
||||||
tsp.TopicSocialProofRequest(
|
|
||||||
userId = query.getRequiredUserId,
|
|
||||||
tweetIds = candidates.map(_.candidate.id).toSet,
|
|
||||||
displayLocation = cr.DisplayLocation.HomeTimeline,
|
|
||||||
topicListingSetting = tsp.TopicListingSetting.Followable,
|
|
||||||
context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
|
|
||||||
bypassModes = None,
|
|
||||||
// Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
|
|
||||||
tags = if (tags.isEmpty) None else Some(tags)
|
|
||||||
)
|
|
||||||
|
|
||||||
topicSocialProofClientColumn.fetcher
|
|
||||||
.fetch(topicSocialProofRequest)
|
|
||||||
.map(_.v)
|
|
||||||
.map {
|
|
||||||
case Some(response) =>
|
|
||||||
candidates.map { candidate =>
|
|
||||||
val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
|
|
||||||
if (topicWithScores.nonEmpty) {
|
|
||||||
keyFoundCounter.incr()
|
|
||||||
val (socialProofId, socialProofFunctionalityType) =
|
|
||||||
if (candidate.features
|
|
||||||
.getOrElse(CandidateSourceIdFeature, None)
|
|
||||||
.exists(sourcesToSetSocialProof.contains)) {
|
|
||||||
getSocialProof(topicWithScores)
|
|
||||||
} else {
|
|
||||||
(None, None)
|
|
||||||
}
|
|
||||||
val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
|
|
||||||
val inferredTopicDataRecord =
|
|
||||||
InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
|
|
||||||
FeatureMapBuilder()
|
|
||||||
.add(TSPInferredTopicFeature, inferredTopicFeatures)
|
|
||||||
.add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
|
|
||||||
.add(TopicIdSocialContextFeature, socialProofId)
|
|
||||||
.add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
|
|
||||||
.build()
|
|
||||||
} else {
|
|
||||||
keyLossCounter.incr()
|
|
||||||
DefaultFeatureMap
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case _ =>
|
|
||||||
requestFailCounter.incr()
|
|
||||||
candidates.map { _ =>
|
|
||||||
DefaultFeatureMap
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def getSocialProof(
|
|
||||||
topicWithScores: Seq[tsp.TopicWithScore]
|
|
||||||
): (Option[Long], Option[TopicContextFunctionalityType]) = {
|
|
||||||
val followingTopicId = topicWithScores
|
|
||||||
.collectFirst {
|
|
||||||
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
|
|
||||||
topicId
|
|
||||||
}
|
|
||||||
if (followingTopicId.nonEmpty) {
|
|
||||||
return (followingTopicId, Some(BasicTopicContextFunctionalityType))
|
|
||||||
}
|
|
||||||
val implicitFollowingId = topicWithScores.collectFirst {
|
|
||||||
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
|
|
||||||
topicId
|
|
||||||
}
|
|
||||||
if (implicitFollowingId.nonEmpty) {
|
|
||||||
return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
|
|
||||||
}
|
|
||||||
(None, None)
|
|
||||||
}
|
|
||||||
|
|
||||||
private def convertTopicWithScores(
|
|
||||||
topicWithScores: Seq[tsp.TopicWithScore],
|
|
||||||
): Map[Long, Double] = {
|
|
||||||
topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,251 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
|
||||||
import com.twitter.ml.api.DataRecord
|
|
||||||
import com.twitter.ml.api.RichDataRecord
|
|
||||||
import com.twitter.ml.api.util.FDsl._
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
|
||||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.search.common.features.{thriftscala => sc}
|
|
||||||
import com.twitter.snowflake.id.SnowflakeId
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
|
|
||||||
import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
|
|
||||||
import com.twitter.timelines.prediction.features.time_features.TimeFeatures
|
|
||||||
import com.twitter.util.Duration
|
|
||||||
import scala.collection.Searching._
|
|
||||||
|
|
||||||
object TimeFeaturesDataRecordFeature
|
|
||||||
extends DataRecordInAFeature[TweetCandidate]
|
|
||||||
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
|
|
||||||
override def defaultValue: DataRecord = new DataRecord()
|
|
||||||
}
|
|
||||||
|
|
||||||
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidate: TweetCandidate,
|
|
||||||
existingFeatures: FeatureMap
|
|
||||||
): Stitch[FeatureMap] = {
|
|
||||||
Stitch.value {
|
|
||||||
val richDataRecord = new RichDataRecord()
|
|
||||||
setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
|
|
||||||
FeatureMapBuilder()
|
|
||||||
.add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
|
|
||||||
.build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def setTimeFeatures(
|
|
||||||
richDataRecord: RichDataRecord,
|
|
||||||
candidate: TweetCandidate,
|
|
||||||
existingFeatures: FeatureMap,
|
|
||||||
query: PipelineQuery,
|
|
||||||
): Unit = {
|
|
||||||
val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
|
|
||||||
timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
|
|
||||||
}
|
|
||||||
|
|
||||||
private[feature_hydrator] def getTimeFeatures(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidate: TweetCandidate,
|
|
||||||
existingFeatures: FeatureMap,
|
|
||||||
): Option[TimeFeatures] = {
|
|
||||||
for {
|
|
||||||
requestTimestampMs <- Some(query.queryTime.inMilliseconds)
|
|
||||||
tweetId <- Some(candidate.id)
|
|
||||||
viewerId <- query.getOptionalUserId
|
|
||||||
tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
|
|
||||||
timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
|
|
||||||
accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
|
|
||||||
Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
|
|
||||||
}
|
|
||||||
timeSinceSourceTweetCreation =
|
|
||||||
existingFeatures
|
|
||||||
.getOrElse(SourceTweetIdFeature, None)
|
|
||||||
.flatMap { sourceTweetId =>
|
|
||||||
timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
|
|
||||||
requestTimestampMs - sourceTweetCreationTimeMs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.getOrElse(timeSinceTweetCreation)
|
|
||||||
if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
|
|
||||||
} yield {
|
|
||||||
val timeFeatures = TimeFeatures(
|
|
||||||
timeSinceTweetCreation = timeSinceTweetCreation,
|
|
||||||
timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
|
|
||||||
timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
|
|
||||||
isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
|
|
||||||
isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
|
|
||||||
accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
|
|
||||||
isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
|
|
||||||
)
|
|
||||||
|
|
||||||
val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
|
|
||||||
existingFeatures.getOrElse(EarlybirdFeature, None),
|
|
||||||
timeFeatures,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
).getOrElse(timeFeatures)
|
|
||||||
|
|
||||||
val nonPollingTimestampsMs =
|
|
||||||
query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
|
|
||||||
val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
|
|
||||||
timeFeaturesWithLastEngagement,
|
|
||||||
requestTimestampMs,
|
|
||||||
tweetCreationTimeMs,
|
|
||||||
nonPollingTimestampsMs
|
|
||||||
)
|
|
||||||
timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
|
|
||||||
if (SnowflakeId.isSnowflakeId(tweetOrUserId))
|
|
||||||
Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
|
|
||||||
else None
|
|
||||||
}
|
|
||||||
|
|
||||||
private def addLastEngagementTimeFeatures(
|
|
||||||
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
|
|
||||||
timeFeatures: TimeFeatures,
|
|
||||||
timeSinceSourceTweetCreation: Long
|
|
||||||
): Option[TimeFeatures] = {
|
|
||||||
tweetFeaturesOpt.map { tweetFeatures =>
|
|
||||||
val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
|
|
||||||
val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
|
|
||||||
val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
|
|
||||||
val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
|
|
||||||
|
|
||||||
timeFeatures.copy(
|
|
||||||
lastFavSinceCreationHrs = lastFavSinceCreationHrs,
|
|
||||||
lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
|
|
||||||
lastReplySinceCreationHrs = lastReplySinceCreationHrs,
|
|
||||||
lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
|
|
||||||
timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
|
|
||||||
lastFavSinceCreationHrs,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
),
|
|
||||||
timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
|
|
||||||
lastRetweetSinceCreationHrs,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
),
|
|
||||||
timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
|
|
||||||
lastReplySinceCreationHrs,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
),
|
|
||||||
timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
|
|
||||||
lastQuoteSinceCreationHrs,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def addNonPollingTimeFeatures(
|
|
||||||
timeFeatures: TimeFeatures,
|
|
||||||
requestTimestampMs: Long,
|
|
||||||
creationTimeMs: Long,
|
|
||||||
nonPollingTimestampsMs: Option[Seq[Long]]
|
|
||||||
): Option[TimeFeatures] = {
|
|
||||||
for {
|
|
||||||
nonPollingTimestampsMs <- nonPollingTimestampsMs
|
|
||||||
lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
|
|
||||||
earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
|
|
||||||
} yield {
|
|
||||||
val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
|
|
||||||
val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
|
|
||||||
1.0,
|
|
||||||
timeFeatures.timeSinceTweetCreation
|
|
||||||
)
|
|
||||||
/*
|
|
||||||
* Non-polling timestamps are stored in chronological order.
|
|
||||||
* The latest timestamps occur first, therefore we need to explicitly search in reverse order.
|
|
||||||
*/
|
|
||||||
val nonPollingRequestsSinceTweetCreation =
|
|
||||||
if (nonPollingTimestampsMs.nonEmpty) {
|
|
||||||
nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
* Calculate the average time between non-polling requests; include
|
|
||||||
* request time in this calculation as latest timestamp.
|
|
||||||
*/
|
|
||||||
val timeBetweenNonPollingRequestsAvg =
|
|
||||||
(requestTimestampMs - earliestNonPollingTimestampMs) / math
|
|
||||||
.max(1.0, nonPollingTimestampsMs.size)
|
|
||||||
val timeFeaturesWithNonPolling = timeFeatures.copy(
|
|
||||||
timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
|
|
||||||
timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
|
|
||||||
nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
|
|
||||||
tweetAgeRatio = Some(tweetAgeRatio)
|
|
||||||
)
|
|
||||||
timeFeaturesWithNonPolling
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private[this] def getTimeSinceLastEngagementHrs(
|
|
||||||
lastEngagementTimeSinceCreationHrsOpt: Option[Double],
|
|
||||||
timeSinceTweetCreation: Long
|
|
||||||
): Option[Double] = {
|
|
||||||
lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
|
|
||||||
val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
|
|
||||||
timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
|
|
||||||
val record = richDataRecord.getRecord
|
|
||||||
.setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
|
|
||||||
.setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
|
|
||||||
.setFeatureValueFromOption(
|
|
||||||
TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
|
|
||||||
features.timeSinceViewerAccountCreationSecs)
|
|
||||||
.setFeatureValue(
|
|
||||||
USER_ID_IS_SNOWFLAKE_ID,
|
|
||||||
features.timeSinceViewerAccountCreationSecs.isDefined
|
|
||||||
)
|
|
||||||
.setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
|
|
||||||
.setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
|
|
||||||
.setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
|
|
||||||
.setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
|
|
||||||
.setFeatureValueFromOption(
|
|
||||||
LAST_RETWEET_SINCE_CREATION_HRS,
|
|
||||||
features.lastRetweetSinceCreationHrs
|
|
||||||
)
|
|
||||||
.setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
|
|
||||||
.setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
|
|
||||||
.setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
|
|
||||||
.setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
|
|
||||||
.setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
|
|
||||||
.setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
|
|
||||||
/*
|
|
||||||
* set features whose values are optional as some users do not have non-polling timestamps
|
|
||||||
*/
|
|
||||||
features.timeBetweenNonPollingRequestsAvg.foreach(
|
|
||||||
record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
|
|
||||||
)
|
|
||||||
features.timeSinceLastNonPollingRequest.foreach(
|
|
||||||
record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
|
|
||||||
)
|
|
||||||
features.nonPollingRequestsSinceTweetCreation.foreach(
|
|
||||||
record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
|
|
||||||
)
|
|
||||||
features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
|
|
||||||
}
|
|
||||||
}
|
|
@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
|
|||||||
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
|
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
|
||||||
extends QueryFeatureHydrator[Query] {
|
extends QueryFeatureHydrator[Query] {
|
||||||
|
|
||||||
private val TweetImpressionTTL = 1.day
|
private val TweetImpressionTTL = 2.days
|
||||||
private val TweetImpressionCap = 3000
|
private val TweetImpressionCap = 5000
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")
|
||||||
|
|
||||||
|
@ -1,6 +1,9 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
package com.twitter.home_mixer.functional_component.feature_hydrator
|
||||||
|
|
||||||
|
import com.twitter.finagle.stats.StatsReceiver
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
|
||||||
@ -10,11 +13,13 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
|
|||||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
|
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
||||||
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
import com.twitter.home_mixer.util.tweetypie.RequestFields
|
import com.twitter.home_mixer.util.tweetypie.RequestFields
|
||||||
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
|
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
|
||||||
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
|
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
|
||||||
@ -29,17 +34,22 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
|
|||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
|
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
|
||||||
import com.twitter.tweetypie.{thriftscala => tp}
|
import com.twitter.tweetypie.{thriftscala => tp}
|
||||||
|
import com.twitter.util.logging.Logging
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
|
class TweetypieFeatureHydrator @Inject() (
|
||||||
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
tweetypieStitchClient: TweetypieStitchClient,
|
||||||
|
statsReceiver: StatsReceiver)
|
||||||
|
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
|
||||||
|
with Logging {
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(
|
override val features: Set[Feature[_, _]] = Set(
|
||||||
AuthorIdFeature,
|
AuthorIdFeature,
|
||||||
|
ExclusiveConversationAuthorIdFeature,
|
||||||
InReplyToTweetIdFeature,
|
InReplyToTweetIdFeature,
|
||||||
IsHydratedFeature,
|
IsHydratedFeature,
|
||||||
IsNsfw,
|
IsNsfw,
|
||||||
@ -51,6 +61,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
SourceTweetIdFeature,
|
SourceTweetIdFeature,
|
||||||
SourceUserIdFeature,
|
SourceUserIdFeature,
|
||||||
TweetTextFeature,
|
TweetTextFeature,
|
||||||
|
TweetLanguageFeature,
|
||||||
VisibilityReason
|
VisibilityReason
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -70,9 +81,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
): Stitch[FeatureMap] = {
|
): Stitch[FeatureMap] = {
|
||||||
val safetyLevel = query.product match {
|
val safetyLevel = query.product match {
|
||||||
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
|
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
|
||||||
case ForYouProduct => rtf.SafetyLevel.TimelineHome
|
case ForYouProduct =>
|
||||||
|
val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
|
||||||
|
if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
|
||||||
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
|
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
|
||||||
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
|
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
|
||||||
|
case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
|
||||||
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
|
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -82,9 +96,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
includeQuotedTweet = true,
|
includeQuotedTweet = true,
|
||||||
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
|
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
|
||||||
safetyLevel = Some(safetyLevel),
|
safetyLevel = Some(safetyLevel),
|
||||||
forUserId = Some(query.getRequiredUserId)
|
forUserId = query.getOptionalUserId
|
||||||
)
|
)
|
||||||
|
|
||||||
|
val exclusiveAuthorIdOpt =
|
||||||
|
existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
|
||||||
|
|
||||||
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
|
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
|
||||||
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
|
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
|
||||||
val coreData = found.tweet.coreData
|
val coreData = found.tweet.coreData
|
||||||
@ -106,6 +123,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
|
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
|
||||||
|
|
||||||
val tweetText = coreData.map(_.text)
|
val tweetText = coreData.map(_.text)
|
||||||
|
val tweetLanguage = found.tweet.language.map(_.language)
|
||||||
|
|
||||||
val tweetAuthorId = coreData.map(_.userId)
|
val tweetAuthorId = coreData.map(_.userId)
|
||||||
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
|
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
|
||||||
@ -127,6 +145,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
|
|
||||||
FeatureMapBuilder()
|
FeatureMapBuilder()
|
||||||
.add(AuthorIdFeature, tweetAuthorId)
|
.add(AuthorIdFeature, tweetAuthorId)
|
||||||
|
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
|
||||||
.add(InReplyToTweetIdFeature, inReplyToTweetId)
|
.add(InReplyToTweetIdFeature, inReplyToTweetId)
|
||||||
.add(IsHydratedFeature, true)
|
.add(IsHydratedFeature, true)
|
||||||
.add(IsNsfw, Some(isNsfw))
|
.add(IsNsfw, Some(isNsfw))
|
||||||
@ -137,20 +156,24 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
.add(QuotedUserIdFeature, quotedTweetUserId)
|
.add(QuotedUserIdFeature, quotedTweetUserId)
|
||||||
.add(SourceTweetIdFeature, retweetedTweetId)
|
.add(SourceTweetIdFeature, retweetedTweetId)
|
||||||
.add(SourceUserIdFeature, retweetedTweetUserId)
|
.add(SourceUserIdFeature, retweetedTweetUserId)
|
||||||
|
.add(TweetLanguageFeature, tweetLanguage)
|
||||||
.add(TweetTextFeature, tweetText)
|
.add(TweetTextFeature, tweetText)
|
||||||
.add(VisibilityReason, found.suppressReason)
|
.add(VisibilityReason, found.suppressReason)
|
||||||
.build()
|
.build()
|
||||||
|
|
||||||
// If no tweet result found, return default and pre-existing features
|
// If no tweet result found, return default and pre-existing features
|
||||||
case _ =>
|
case _ =>
|
||||||
DefaultFeatureMap +
|
DefaultFeatureMap ++ FeatureMapBuilder()
|
||||||
(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) +
|
.add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
|
||||||
(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) +
|
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
|
||||||
(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) +
|
.add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
|
||||||
(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) +
|
.add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
|
||||||
(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) +
|
.add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
|
||||||
(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) +
|
.add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
|
||||||
(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
|
.add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
|
||||||
|
.add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
|
||||||
|
.add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
|
||||||
|
.build()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,59 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
|
|
||||||
|
|
||||||
import com.twitter.ml.api.DataRecordMerger
|
|
||||||
import com.twitter.ml.api.Feature
|
|
||||||
import com.twitter.ml.api.FeatureContext
|
|
||||||
import com.twitter.ml.api.RichDataRecord
|
|
||||||
import com.twitter.ml.api.util.CompactDataRecordConverter
|
|
||||||
import com.twitter.ml.api.util.FDsl._
|
|
||||||
import com.twitter.timelines.author_features.v1.{thriftjava => af}
|
|
||||||
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
|
|
||||||
import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
|
|
||||||
import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
|
|
||||||
|
|
||||||
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
|
|
||||||
|
|
||||||
private val originalAuthorAggregatesFeatures =
|
|
||||||
TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
|
|
||||||
.buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
|
|
||||||
private val authorFeatures = originalAuthorAggregatesFeatures ++
|
|
||||||
Seq(
|
|
||||||
UserHealthFeatures.AuthorState,
|
|
||||||
UserHealthFeatures.NumAuthorFollowers,
|
|
||||||
UserHealthFeatures.NumAuthorConnectDays,
|
|
||||||
UserHealthFeatures.NumAuthorConnect)
|
|
||||||
private val featureContext = new FeatureContext(authorFeatures: _*)
|
|
||||||
|
|
||||||
override def getFeatureContext: FeatureContext = featureContext
|
|
||||||
|
|
||||||
override val commonFeatures: Set[Feature[_]] = Set.empty
|
|
||||||
|
|
||||||
private val compactDataRecordConverter = new CompactDataRecordConverter()
|
|
||||||
private val drMerger = new DataRecordMerger()
|
|
||||||
|
|
||||||
override def setFeatures(
|
|
||||||
authorFeaturesOpt: Option[af.AuthorFeatures],
|
|
||||||
richDataRecord: RichDataRecord
|
|
||||||
): Unit = {
|
|
||||||
authorFeaturesOpt.foreach { authorFeatures =>
|
|
||||||
val dataRecord = richDataRecord.getRecord
|
|
||||||
|
|
||||||
dataRecord.setFeatureValue(
|
|
||||||
UserHealthFeatures.AuthorState,
|
|
||||||
authorFeatures.user_health.user_state.getValue.toLong)
|
|
||||||
dataRecord.setFeatureValue(
|
|
||||||
UserHealthFeatures.NumAuthorFollowers,
|
|
||||||
authorFeatures.user_health.num_followers.toDouble)
|
|
||||||
dataRecord.setFeatureValue(
|
|
||||||
UserHealthFeatures.NumAuthorConnectDays,
|
|
||||||
authorFeatures.user_health.num_connect_days.toDouble)
|
|
||||||
dataRecord.setFeatureValue(
|
|
||||||
UserHealthFeatures.NumAuthorConnect,
|
|
||||||
authorFeatures.user_health.num_connect.toDouble)
|
|
||||||
|
|
||||||
val originalAuthorAggregatesDataRecord =
|
|
||||||
compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
|
|
||||||
drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,22 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier =
|
|
||||||
FeatureHydratorIdentifier("Phase2EdgeAggregate")
|
|
||||||
|
|
||||||
override val aggregateFeatures: Set[BaseEdgeAggregateFeature] =
|
|
||||||
Set(
|
|
||||||
UserEngagerAggregateFeature,
|
|
||||||
UserEngagerGoodClickAggregateFeature,
|
|
||||||
UserInferredTopicAggregateFeature,
|
|
||||||
UserTopicAggregateFeature,
|
|
||||||
UserMediaUnderstandingAnnotationAggregateFeature
|
|
||||||
)
|
|
||||||
}
|
|
@ -4,6 +4,7 @@ scala_library(
|
|||||||
strict_deps = True,
|
strict_deps = True,
|
||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
|
||||||
@ -16,9 +17,11 @@ scala_library(
|
|||||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||||
"stitch/stitch-core",
|
"stitch/stitch-core",
|
||||||
|
"stitch/stitch-socialgraph",
|
||||||
"stitch/stitch-tweetypie",
|
"stitch/stitch-tweetypie",
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
||||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
||||||
|
"util/util-slf4j-api/src/main/scala",
|
||||||
],
|
],
|
||||||
exports = [
|
exports = [
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
||||||
|
@ -72,7 +72,7 @@ object FeedbackFatigueFilter
|
|||||||
|
|
||||||
originalAuthorId.exists(authorsToFilter.contains) ||
|
originalAuthorId.exists(authorsToFilter.contains) ||
|
||||||
(likers.nonEmpty && eligibleLikers.isEmpty) ||
|
(likers.nonEmpty && eligibleLikers.isEmpty) ||
|
||||||
(followers.nonEmpty && eligibleFollowers.isEmpty) ||
|
(followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) ||
|
||||||
(candidate.features.getOrElse(IsRetweetFeature, false) &&
|
(candidate.features.getOrElse(IsRetweetFeature, false) &&
|
||||||
authorId.exists(retweetersToFilter.contains))
|
authorId.exists(retweetersToFilter.contains))
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,70 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.filter
|
||||||
|
|
||||||
|
import com.twitter.finagle.stats.StatsReceiver
|
||||||
|
import com.twitter.finagle.tracing.Trace
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||||
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.socialgraph.{thriftscala => sg}
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
import com.twitter.stitch.socialgraph.SocialGraph
|
||||||
|
import com.twitter.util.logging.Logging
|
||||||
|
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author
|
||||||
|
*
|
||||||
|
* If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for
|
||||||
|
* subscription tweets, so we explicitly filter those tweets out.
|
||||||
|
*/
|
||||||
|
@Singleton
|
||||||
|
case class InvalidSubscriptionTweetFilter @Inject() (
|
||||||
|
socialGraphClient: SocialGraph,
|
||||||
|
statsReceiver: StatsReceiver)
|
||||||
|
extends Filter[PipelineQuery, TweetCandidate]
|
||||||
|
with Logging {
|
||||||
|
|
||||||
|
override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet")
|
||||||
|
|
||||||
|
private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
|
||||||
|
private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet")
|
||||||
|
private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet")
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
|
): Stitch[FilterResult[TweetCandidate]] = Stitch
|
||||||
|
.traverse(candidates) { candidate =>
|
||||||
|
val exclusiveAuthorId =
|
||||||
|
candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None)
|
||||||
|
|
||||||
|
if (exclusiveAuthorId.isDefined) {
|
||||||
|
val request = sg.ExistsRequest(
|
||||||
|
source = query.getRequiredUserId,
|
||||||
|
target = exclusiveAuthorId.get,
|
||||||
|
relationships =
|
||||||
|
Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)),
|
||||||
|
)
|
||||||
|
socialGraphClient.exists(request).map(_.exists).map { valid =>
|
||||||
|
if (!valid) invalidCounter.incr() else validCounter.incr()
|
||||||
|
valid
|
||||||
|
}
|
||||||
|
} else Stitch.value(true)
|
||||||
|
}.map { validResults =>
|
||||||
|
val (kept, removed) = candidates
|
||||||
|
.map(_.candidate)
|
||||||
|
.zip(validResults)
|
||||||
|
.partition { case (candidate, valid) => valid }
|
||||||
|
|
||||||
|
val keptCandidates = kept.map { case (candidate, _) => candidate }
|
||||||
|
val removedCandidates = removed.map { case (candidate, _) => candidate }
|
||||||
|
|
||||||
|
FilterResult(kept = keptCandidates, removed = removedCandidates)
|
||||||
|
}
|
||||||
|
}
|
@ -1,59 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.filter
|
|
||||||
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
|
||||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
|
||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
|
||||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Predicate which will be applied to each candidate. True indicates that the candidate will be
|
|
||||||
* @tparam Candidate - the type of the candidate
|
|
||||||
*/
|
|
||||||
trait ShouldKeepCandidate {
|
|
||||||
def apply(features: FeatureMap): Boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
object PredicateFeatureFilter {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity,
|
|
||||||
* we recommend including the name of the shouldKeepCandidate parameter.
|
|
||||||
*
|
|
||||||
* @param identifier A FilterIdentifier for the new filter
|
|
||||||
* @param shouldKeepCandidate A predicate function. Candidates will be kept when
|
|
||||||
* this function returns True.
|
|
||||||
*/
|
|
||||||
def fromPredicate[Candidate <: UniversalNoun[Any]](
|
|
||||||
identifier: FilterIdentifier,
|
|
||||||
shouldKeepCandidate: ShouldKeepCandidate
|
|
||||||
): Filter[PipelineQuery, Candidate] = {
|
|
||||||
val i = identifier
|
|
||||||
|
|
||||||
new Filter[PipelineQuery, Candidate] {
|
|
||||||
override val identifier: FilterIdentifier = i
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Filter the list of candidates
|
|
||||||
*
|
|
||||||
* @return a FilterResult including both the list of kept candidate and the list of removed candidates
|
|
||||||
*/
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
|
||||||
): Stitch[FilterResult[Candidate]] = {
|
|
||||||
val allowedIds = candidates
|
|
||||||
.filter(candidate => shouldKeepCandidate(candidate.features)).map(_.candidate.id).toSet
|
|
||||||
|
|
||||||
val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition {
|
|
||||||
candidate => allowedIds.contains(candidate.id)
|
|
||||||
}
|
|
||||||
|
|
||||||
Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,8 +1,6 @@
|
|||||||
package com.twitter.home_mixer.functional_component.filter
|
package com.twitter.home_mixer.functional_component.filter
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||||
@ -11,24 +9,20 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
|||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
|
|
||||||
object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] {
|
object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
override val identifier: FilterIdentifier = FilterIdentifier("KeepBestOutOfNetworkTweetPerAuthor")
|
override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews")
|
||||||
|
|
||||||
override def apply(
|
override def apply(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
): Stitch[FilterResult[TweetCandidate]] = {
|
): Stitch[FilterResult[TweetCandidate]] = {
|
||||||
// Set containing best OON tweet for each authorId
|
|
||||||
val bestCandidatesForAuthorId = candidates
|
val servedTweetPreviewIds =
|
||||||
.filter(!_.features.getOrElse(InNetworkFeature, true))
|
query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet
|
||||||
.groupBy(_.features.getOrElse(AuthorIdFeature, None))
|
|
||||||
.values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None)))
|
|
||||||
.toSet
|
|
||||||
|
|
||||||
val (removed, kept) = candidates.partition { candidate =>
|
val (removed, kept) = candidates.partition { candidate =>
|
||||||
!candidate.features.getOrElse(InNetworkFeature, true) &&
|
servedTweetPreviewIds.contains(candidate.candidate.id)
|
||||||
!bestCandidatesForAuthorId.contains(candidate)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))
|
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))
|
@ -0,0 +1,32 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.filter
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||||
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
|
||||||
|
object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||||
|
override val identifier: FilterIdentifier = FilterIdentifier("Reply")
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
|
): Stitch[FilterResult[TweetCandidate]] = {
|
||||||
|
|
||||||
|
val (kept, removed) = candidates
|
||||||
|
.partition { candidate =>
|
||||||
|
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
|
||||||
|
}
|
||||||
|
|
||||||
|
val filterResult = FilterResult(
|
||||||
|
kept = kept.map(_.candidate),
|
||||||
|
removed = removed.map(_.candidate)
|
||||||
|
)
|
||||||
|
|
||||||
|
Stitch.value(filterResult)
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,32 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.filter
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||||
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
|
||||||
|
object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||||
|
override val identifier: FilterIdentifier = FilterIdentifier("Retweet")
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
|
): Stitch[FilterResult[TweetCandidate]] = {
|
||||||
|
|
||||||
|
val (kept, removed) = candidates
|
||||||
|
.partition { candidate =>
|
||||||
|
!candidate.features.getOrElse(IsRetweetFeature, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
val filterResult = FilterResult(
|
||||||
|
kept = kept.map(_.candidate),
|
||||||
|
removed = removed.map(_.candidate)
|
||||||
|
)
|
||||||
|
|
||||||
|
Stitch.value(filterResult)
|
||||||
|
}
|
||||||
|
}
|
@ -14,7 +14,6 @@ scala_library(
|
|||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
||||||
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||||
"stitch/stitch-socialgraph",
|
"stitch/stitch-socialgraph",
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
|
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
||||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
||||||
],
|
],
|
||||||
|
@ -1,14 +1,14 @@
|
|||||||
package com.twitter.home_mixer.functional_component.gate
|
package com.twitter.home_mixer.functional_component.gate
|
||||||
|
|
||||||
|
import com.twitter.conversions.DurationOps._
|
||||||
|
import com.twitter.product_mixer.core.feature.Feature
|
||||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
|
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.util.Duration
|
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
|
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
|
||||||
|
import com.twitter.util.Duration
|
||||||
|
|
||||||
object DismissFatigueGate {
|
object DismissFatigueGate {
|
||||||
// how long a dismiss action from user needs to be respected
|
// how long a dismiss action from user needs to be respected
|
||||||
|
@ -1,18 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.gate
|
|
||||||
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import scala.reflect.runtime.universe._
|
|
||||||
|
|
||||||
case class NonEmptySeqFeatureGate[T: TypeTag](
|
|
||||||
feature: Feature[PipelineQuery, Seq[T]])
|
|
||||||
extends Gate[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: GateIdentifier = GateIdentifier(s"NonEmptySeq$feature")
|
|
||||||
|
|
||||||
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
|
|
||||||
Stitch.value(query.features.exists(_.get(feature).nonEmpty))
|
|
||||||
}
|
|
@ -19,7 +19,7 @@ object EditedTweetsCandidatePipelineQueryTransformer
|
|||||||
override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
|
override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
|
||||||
|
|
||||||
// The time window for which a tweet remains editable after creation.
|
// The time window for which a tweet remains editable after creation.
|
||||||
private val EditTimeWindow = 30.minutes
|
private val EditTimeWindow = 60.minutes
|
||||||
|
|
||||||
override def transform(query: PipelineQuery): Seq[Long] = {
|
override def transform(query: PipelineQuery): Seq[Long] = {
|
||||||
val applicableCandidates = getApplicableCandidates(query)
|
val applicableCandidates = getApplicableCandidates(query)
|
||||||
@ -29,8 +29,8 @@ object EditedTweetsCandidatePipelineQueryTransformer
|
|||||||
// Any tweets in it could have become stale since being served.
|
// Any tweets in it could have become stale since being served.
|
||||||
val previousTimelineLoadTime = applicableCandidates.head.servedTime
|
val previousTimelineLoadTime = applicableCandidates.head.servedTime
|
||||||
|
|
||||||
// The time window for editing a tweet is 30 minutes,
|
// The time window for editing a tweet is 60 minutes,
|
||||||
// so we ignore responses older than (PTL Time - 30 mins).
|
// so we ignore responses older than (PTL Time - 60 mins).
|
||||||
val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
|
val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
|
||||||
.takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)
|
.takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)
|
||||||
|
|
||||||
|
@ -6,7 +6,6 @@ scala_library(
|
|||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
||||||
|
@ -35,8 +35,8 @@ object FeedbackFatigueScorer
|
|||||||
override def onlyIf(query: PipelineQuery): Boolean =
|
override def onlyIf(query: PipelineQuery): Boolean =
|
||||||
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
|
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
|
||||||
|
|
||||||
private val DurationForFiltering = 14.days
|
val DurationForFiltering = 14.days
|
||||||
private val DurationForDiscounting = 140.days
|
val DurationForDiscounting = 140.days
|
||||||
private val ScoreMultiplierLowerBound = 0.2
|
private val ScoreMultiplierLowerBound = 0.2
|
||||||
private val ScoreMultiplierUpperBound = 1.0
|
private val ScoreMultiplierUpperBound = 1.0
|
||||||
private val ScoreMultiplierIncrementsCount = 4
|
private val ScoreMultiplierIncrementsCount = 4
|
||||||
@ -76,8 +76,27 @@ object FeedbackFatigueScorer
|
|||||||
feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty))
|
feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty))
|
||||||
|
|
||||||
val featureMaps = candidates.map { candidate =>
|
val featureMaps = candidates.map { candidate =>
|
||||||
|
val multiplier = getScoreMultiplier(
|
||||||
|
candidate,
|
||||||
|
authorsToDiscount,
|
||||||
|
likersToDiscount,
|
||||||
|
followersToDiscount,
|
||||||
|
retweetersToDiscount
|
||||||
|
)
|
||||||
val score = candidate.features.getOrElse(ScoreFeature, None)
|
val score = candidate.features.getOrElse(ScoreFeature, None)
|
||||||
|
FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
|
||||||
|
}
|
||||||
|
|
||||||
|
Stitch.value(featureMaps)
|
||||||
|
}
|
||||||
|
|
||||||
|
def getScoreMultiplier(
|
||||||
|
candidate: CandidateWithFeatures[TweetCandidate],
|
||||||
|
authorsToDiscount: Map[Long, Double],
|
||||||
|
likersToDiscount: Map[Long, Double],
|
||||||
|
followersToDiscount: Map[Long, Double],
|
||||||
|
retweetersToDiscount: Map[Long, Double],
|
||||||
|
): Double = {
|
||||||
val originalAuthorId =
|
val originalAuthorId =
|
||||||
CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
|
CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
|
||||||
val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
|
val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
|
||||||
@ -92,7 +111,8 @@ object FeedbackFatigueScorer
|
|||||||
val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
|
val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
|
||||||
val followerMultipliers = followers.flatMap(followersToDiscount.get)
|
val followerMultipliers = followers.flatMap(followersToDiscount.get)
|
||||||
val followerMultiplier =
|
val followerMultiplier =
|
||||||
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size)
|
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size &&
|
||||||
|
likers.isEmpty)
|
||||||
followerMultipliers.max
|
followerMultipliers.max
|
||||||
else 1.0
|
else 1.0
|
||||||
|
|
||||||
@ -102,16 +122,10 @@ object FeedbackFatigueScorer
|
|||||||
retweetersToDiscount.getOrElse(authorId, 1.0)
|
retweetersToDiscount.getOrElse(authorId, 1.0)
|
||||||
else 1.0
|
else 1.0
|
||||||
|
|
||||||
val multiplier =
|
|
||||||
originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
|
originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
|
||||||
|
|
||||||
FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Stitch.value(featureMaps)
|
def getUserDiscounts(
|
||||||
}
|
|
||||||
|
|
||||||
private def getUserDiscounts(
|
|
||||||
queryTime: Time,
|
queryTime: Time,
|
||||||
feedbackEntries: Seq[FeedbackEntry],
|
feedbackEntries: Seq[FeedbackEntry],
|
||||||
): Map[Long, Double] = {
|
): Map[Long, Double] = {
|
||||||
|
@ -1,61 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.scorer
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorInNetworkMultiplierParam
|
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorOutOfNetworkMultiplierParam
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
|
|
||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Scales scores of tweets whose author is Blue Verified by the provided scale factor
|
|
||||||
*/
|
|
||||||
object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate] {
|
|
||||||
|
|
||||||
override val identifier: ScorerIdentifier = ScorerIdentifier("VerifiedAuthorScaling")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(ScoreFeature)
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
|
||||||
Stitch.value {
|
|
||||||
candidates.map { candidate =>
|
|
||||||
val score = candidate.features.getOrElse(ScoreFeature, None)
|
|
||||||
val updatedScore = getUpdatedScore(score, candidate, query)
|
|
||||||
FeatureMapBuilder().add(ScoreFeature, updatedScore).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* We should only be applying this multiplier if the author of the candidate is Blue Verified.
|
|
||||||
* We also treat In-Network vs Out-of-Network differently.
|
|
||||||
*/
|
|
||||||
private def getUpdatedScore(
|
|
||||||
score: Option[Double],
|
|
||||||
candidate: CandidateWithFeatures[TweetCandidate],
|
|
||||||
query: PipelineQuery
|
|
||||||
): Option[Double] = {
|
|
||||||
val isAuthorBlueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false)
|
|
||||||
|
|
||||||
if (isAuthorBlueVerified) {
|
|
||||||
val isCandidateInNetwork = candidate.features.getOrElse(InNetworkFeature, false)
|
|
||||||
|
|
||||||
val scaleFactor =
|
|
||||||
if (isCandidateInNetwork) query.params(BlueVerifiedAuthorInNetworkMultiplierParam)
|
|
||||||
else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam)
|
|
||||||
|
|
||||||
score.map(_ * scaleFactor)
|
|
||||||
} else score
|
|
||||||
}
|
|
||||||
}
|
|
@ -5,6 +5,7 @@ scala_library(
|
|||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
package com.twitter.home_mixer.functional_component.selector
|
package com.twitter.home_mixer.functional_component.selector
|
||||||
|
|
||||||
import com.twitter.home_mixer.functional_component.decorator.HomeClientEventDetailsBuilder
|
import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature
|
||||||
|
@ -7,14 +7,12 @@ scala_library(
|
|||||||
"3rdparty/jvm/javax/inject:javax.inject",
|
"3rdparty/jvm/javax/inject:javax.inject",
|
||||||
"eventbus/client/src/main/scala/com/twitter/eventbus/client",
|
"eventbus/client/src/main/scala/com/twitter/eventbus/client",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||||
@ -22,6 +20,7 @@ scala_library(
|
|||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module",
|
||||||
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
||||||
"src/scala/com/twitter/timelines/prediction/common/adapters",
|
"src/scala/com/twitter/timelines/prediction/common/adapters",
|
||||||
@ -33,13 +32,12 @@ scala_library(
|
|||||||
"src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java",
|
"src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java",
|
||||||
"src/thrift/com/twitter/timelines/timeline_logging:thrift-scala",
|
"src/thrift/com/twitter/timelines/timeline_logging:thrift-scala",
|
||||||
"src/thrift/com/twitter/user_session_store:thrift-scala",
|
"src/thrift/com/twitter/user_session_store:thrift-scala",
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/core",
|
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/uss",
|
|
||||||
"timelines/ml:kafka",
|
"timelines/ml:kafka",
|
||||||
"timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka",
|
"timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka",
|
||||||
"timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding",
|
"timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/clientconfig",
|
"timelines/src/main/scala/com/twitter/timelines/clientconfig",
|
||||||
|
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
|
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
|
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
||||||
|
@ -2,13 +2,14 @@ package com.twitter.home_mixer.functional_component.side_effect
|
|||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates
|
import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates
|
||||||
import com.twitter.home_mixer.functional_component.decorator.HomeTweetTypePredicates
|
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent
|
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent
|
||||||
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace
|
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
@ -18,15 +19,17 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|||||||
import com.twitter.timelines.injection.scribe.InjectionScribeUtil
|
import com.twitter.timelines.injection.scribe.InjectionScribeUtil
|
||||||
|
|
||||||
private[side_effect] sealed trait ClientEventsBuilder {
|
private[side_effect] sealed trait ClientEventsBuilder {
|
||||||
private val FollowingSection = Some("home_latest")
|
private val FollowingSection = Some("latest")
|
||||||
private val ForYouSection = Some("home")
|
private val ForYouSection = Some("home")
|
||||||
private val ListTweetsSection = Some("list")
|
private val ListTweetsSection = Some("list")
|
||||||
|
private val SubscribedSection = Some("subscribed")
|
||||||
|
|
||||||
protected def section(query: PipelineQuery): Option[String] = {
|
protected def section(query: PipelineQuery): Option[String] = {
|
||||||
query.product match {
|
query.product match {
|
||||||
case FollowingProduct => FollowingSection
|
case FollowingProduct => FollowingSection
|
||||||
case ForYouProduct => ForYouSection
|
case ForYouProduct => ForYouSection
|
||||||
case ListTweetsProduct => ListTweetsSection
|
case ListTweetsProduct => ListTweetsSection
|
||||||
|
case SubscribedProduct => SubscribedSection
|
||||||
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -52,6 +55,7 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
|
|||||||
private val InjectedComponent = Some("injected")
|
private val InjectedComponent = Some("injected")
|
||||||
private val PromotedComponent = Some("promoted")
|
private val PromotedComponent = Some("promoted")
|
||||||
private val WhoToFollowComponent = Some("who_to_follow")
|
private val WhoToFollowComponent = Some("who_to_follow")
|
||||||
|
private val WhoToSubscribeComponent = Some("who_to_subscribe")
|
||||||
private val WithVideoDurationComponent = Some("with_video_duration")
|
private val WithVideoDurationComponent = Some("with_video_duration")
|
||||||
private val VideoDurationSumElement = Some("video_duration_sum")
|
private val VideoDurationSumElement = Some("video_duration_sum")
|
||||||
private val NumVideosElement = Some("num_videos")
|
private val NumVideosElement = Some("num_videos")
|
||||||
@ -60,7 +64,8 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
|
|||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
injectedTweets: Seq[ItemCandidateWithDetails],
|
injectedTweets: Seq[ItemCandidateWithDetails],
|
||||||
promotedTweets: Seq[ItemCandidateWithDetails],
|
promotedTweets: Seq[ItemCandidateWithDetails],
|
||||||
whoToFollowUsers: Seq[ItemCandidateWithDetails]
|
whoToFollowUsers: Seq[ItemCandidateWithDetails],
|
||||||
|
whoToSubscribeUsers: Seq[ItemCandidateWithDetails]
|
||||||
): Seq[ClientEvent] = {
|
): Seq[ClientEvent] = {
|
||||||
val baseEventNamespace = EventNamespace(
|
val baseEventNamespace = EventNamespace(
|
||||||
section = section(query),
|
section = section(query),
|
||||||
@ -77,6 +82,9 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
|
|||||||
ClientEvent(
|
ClientEvent(
|
||||||
baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction),
|
baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction),
|
||||||
eventValue = count(whoToFollowUsers)),
|
eventValue = count(whoToFollowUsers)),
|
||||||
|
ClientEvent(
|
||||||
|
baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction),
|
||||||
|
eventValue = count(whoToSubscribeUsers)),
|
||||||
)
|
)
|
||||||
|
|
||||||
val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map {
|
val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map {
|
||||||
|
@ -5,6 +5,7 @@ import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
|||||||
import com.twitter.home_mixer.util.CandidatesUtil
|
import com.twitter.home_mixer.util.CandidatesUtil
|
||||||
import com.twitter.logpipeline.client.common.EventPublisher
|
import com.twitter.logpipeline.client.common.EventPublisher
|
||||||
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect
|
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect
|
||||||
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||||
@ -15,16 +16,30 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|||||||
* Side effect that logs served tweet metrics to Scribe as client events.
|
* Side effect that logs served tweet metrics to Scribe as client events.
|
||||||
*/
|
*/
|
||||||
case class HomeScribeClientEventSideEffect(
|
case class HomeScribeClientEventSideEffect(
|
||||||
|
enableScribeClientEvents: Boolean,
|
||||||
override val logPipelinePublisher: EventPublisher[LogEvent],
|
override val logPipelinePublisher: EventPublisher[LogEvent],
|
||||||
injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier],
|
injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier],
|
||||||
adsCandidatePipelineIdentifier: CandidatePipelineIdentifier,
|
adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
|
||||||
whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
|
whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
|
||||||
) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] {
|
whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None)
|
||||||
|
extends ScribeClientEventSideEffect[PipelineQuery, Timeline]
|
||||||
|
with PipelineResultSideEffect.Conditionally[
|
||||||
|
PipelineQuery,
|
||||||
|
Timeline
|
||||||
|
] {
|
||||||
|
|
||||||
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent")
|
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent")
|
||||||
|
|
||||||
override val page = "timelinemixer"
|
override val page = "timelinemixer"
|
||||||
|
|
||||||
|
override def onlyIf(
|
||||||
|
query: PipelineQuery,
|
||||||
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
|
remainingCandidates: Seq[CandidateWithDetails],
|
||||||
|
droppedCandidates: Seq[CandidateWithDetails],
|
||||||
|
response: Timeline
|
||||||
|
): Boolean = enableScribeClientEvents
|
||||||
|
|
||||||
override def buildClientEvents(
|
override def buildClientEvents(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
selectedCandidates: Seq[CandidateWithDetails],
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
@ -37,13 +52,15 @@ case class HomeScribeClientEventSideEffect(
|
|||||||
val sources = itemCandidates.groupBy(_.source)
|
val sources = itemCandidates.groupBy(_.source)
|
||||||
val injectedTweets =
|
val injectedTweets =
|
||||||
injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty))
|
injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty))
|
||||||
val promotedTweets = sources.getOrElse(adsCandidatePipelineIdentifier, Seq.empty)
|
val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
|
||||||
|
|
||||||
// WhoToFollow module is not required for all home-mixer products, e.g. list tweets timeline.
|
// WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline.
|
||||||
val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
|
val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
|
||||||
|
val whoToSubscribeUsers =
|
||||||
|
whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
|
||||||
|
|
||||||
val servedEvents = ServedEventsBuilder
|
val servedEvents = ServedEventsBuilder
|
||||||
.build(query, injectedTweets, promotedTweets, whoToFollowUsers)
|
.build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers)
|
||||||
|
|
||||||
val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets)
|
val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets)
|
||||||
|
|
||||||
|
@ -0,0 +1,245 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.side_effect
|
||||||
|
|
||||||
|
import com.twitter.finagle.tracing.Trace
|
||||||
|
import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller
|
||||||
|
import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller
|
||||||
|
import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
|
||||||
|
import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
|
||||||
|
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||||
|
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
||||||
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
|
import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag
|
||||||
|
import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam
|
||||||
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
|
import com.twitter.inject.annotations.Flag
|
||||||
|
import com.twitter.logpipeline.client.common.EventPublisher
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
|
||||||
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
||||||
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
|
||||||
|
import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect
|
||||||
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
||||||
|
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||||
|
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
||||||
|
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
|
||||||
|
import com.twitter.util.Time
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Side effect that logs home timeline served candidates to Scribe.
|
||||||
|
*/
|
||||||
|
@Singleton
|
||||||
|
class HomeScribeServedCandidatesSideEffect @Inject() (
|
||||||
|
@Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean,
|
||||||
|
scribeEventPublisher: EventPublisher[thrift.ServedEntry])
|
||||||
|
extends ScribeLogEventSideEffect[
|
||||||
|
thrift.ServedEntry,
|
||||||
|
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
||||||
|
Timeline
|
||||||
|
]
|
||||||
|
with PipelineResultSideEffect.Conditionally[
|
||||||
|
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
||||||
|
Timeline
|
||||||
|
] {
|
||||||
|
|
||||||
|
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates")
|
||||||
|
|
||||||
|
override def onlyIf(
|
||||||
|
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
||||||
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
|
remainingCandidates: Seq[CandidateWithDetails],
|
||||||
|
droppedCandidates: Seq[CandidateWithDetails],
|
||||||
|
response: Timeline
|
||||||
|
): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam)
|
||||||
|
|
||||||
|
override def buildLogEvents(
|
||||||
|
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
||||||
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
|
remainingCandidates: Seq[CandidateWithDetails],
|
||||||
|
droppedCandidates: Seq[CandidateWithDetails],
|
||||||
|
response: Timeline
|
||||||
|
): Seq[thrift.ServedEntry] = {
|
||||||
|
val timelineType = query.product match {
|
||||||
|
case FollowingProduct => thrift.TimelineType.HomeLatest
|
||||||
|
case ForYouProduct => thrift.TimelineType.Home
|
||||||
|
case SubscribedProduct => thrift.TimelineType.HomeSubscribed
|
||||||
|
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
||||||
|
}
|
||||||
|
val requestProvenance = query.deviceContext.map { deviceContext =>
|
||||||
|
deviceContext.requestContextValue match {
|
||||||
|
case RequestContext.Foreground => thrift.RequestProvenance.Foreground
|
||||||
|
case RequestContext.Launch => thrift.RequestProvenance.Launch
|
||||||
|
case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
|
||||||
|
case _ => thrift.RequestProvenance.Other
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val queryType = query.features.map { featureMap =>
|
||||||
|
if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
|
||||||
|
else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
|
||||||
|
else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
|
||||||
|
else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
|
||||||
|
else thrift.QueryType.Other
|
||||||
|
}
|
||||||
|
val requestInfo = thrift.RequestInfo(
|
||||||
|
requestTimeMs = query.queryTime.inMilliseconds,
|
||||||
|
traceId = Trace.id.traceId.toLong,
|
||||||
|
userId = query.getOptionalUserId,
|
||||||
|
clientAppId = query.clientContext.appId,
|
||||||
|
hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
|
||||||
|
parentId = Some(Trace.id.parentId.toLong),
|
||||||
|
spanId = Some(Trace.id.spanId.toLong),
|
||||||
|
timelineType = Some(timelineType),
|
||||||
|
ipAddress = query.clientContext.ipAddress,
|
||||||
|
userAgent = query.clientContext.userAgent,
|
||||||
|
queryType = queryType,
|
||||||
|
requestProvenance = requestProvenance,
|
||||||
|
languageCode = query.clientContext.languageCode,
|
||||||
|
countryCode = query.clientContext.countryCode,
|
||||||
|
requestEndTimeMs = Some(Time.now.inMilliseconds),
|
||||||
|
servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
|
||||||
|
requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None))
|
||||||
|
)
|
||||||
|
|
||||||
|
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
||||||
|
selectedCandidates.flatMap {
|
||||||
|
case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
|
||||||
|
Seq((item.candidateIdLong, item))
|
||||||
|
case module: ModuleCandidateWithDetails
|
||||||
|
if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
|
||||||
|
module.candidates.map(item => (item.candidateIdLong, item))
|
||||||
|
case _ => Seq.empty
|
||||||
|
}.toMap
|
||||||
|
|
||||||
|
val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
||||||
|
selectedCandidates.flatMap {
|
||||||
|
case module: ModuleCandidateWithDetails
|
||||||
|
if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
|
||||||
|
module.candidates.map { item =>
|
||||||
|
(item.candidateIdLong, item)
|
||||||
|
}
|
||||||
|
case _ => Seq.empty
|
||||||
|
}.toMap
|
||||||
|
|
||||||
|
response.instructions.zipWithIndex
|
||||||
|
.collect {
|
||||||
|
case (AddEntriesTimelineInstruction(entries), index) =>
|
||||||
|
entries.collect {
|
||||||
|
case entry: TweetItem if entry.promotedMetadata.isDefined =>
|
||||||
|
val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index)
|
||||||
|
Seq(
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = entry.id,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = entry.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.PromotedTweet,
|
||||||
|
sortIndex = entry.sortIndex,
|
||||||
|
verticalSize = Some(1),
|
||||||
|
displayType = Some(entry.displayType.toString),
|
||||||
|
details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
case entry: TweetItem =>
|
||||||
|
val candidate = tweetIdToItemCandidateMap(entry.id)
|
||||||
|
val tweetDetails = TweetDetailsMarshaller(entry, candidate)
|
||||||
|
Seq(
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = candidate.candidateIdLong,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = entry.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.Tweet,
|
||||||
|
sortIndex = entry.sortIndex,
|
||||||
|
verticalSize = Some(1),
|
||||||
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
||||||
|
displayType = Some(entry.displayType.toString),
|
||||||
|
details = Some(thrift.ItemDetails.TweetDetails(tweetDetails))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
case module: TimelineModule
|
||||||
|
if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
|
||||||
|
module.items.collect {
|
||||||
|
case ModuleItem(entry: UserItem, _, _) =>
|
||||||
|
val candidate = userIdToItemCandidateMap(entry.id)
|
||||||
|
val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate)
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = entry.id,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = module.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.WhoToFollowModule,
|
||||||
|
sortIndex = module.sortIndex,
|
||||||
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
||||||
|
displayType = Some(entry.displayType.toString),
|
||||||
|
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case module: TimelineModule
|
||||||
|
if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
|
||||||
|
module.items.collect {
|
||||||
|
case ModuleItem(entry: UserItem, _, _) =>
|
||||||
|
val candidate = userIdToItemCandidateMap(entry.id)
|
||||||
|
val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate)
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = entry.id,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = module.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.WhoToSubscribeModule,
|
||||||
|
sortIndex = module.sortIndex,
|
||||||
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
||||||
|
displayType = Some(entry.displayType.toString),
|
||||||
|
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case module: TimelineModule
|
||||||
|
if module.sortIndex.isDefined && module.items.headOption.exists(
|
||||||
|
_.item.isInstanceOf[TweetItem]) =>
|
||||||
|
module.items.collect {
|
||||||
|
case ModuleItem(entry: TweetItem, _, _) =>
|
||||||
|
val candidate = tweetIdToItemCandidateMap(entry.id)
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = entry.id,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = module.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.ConversationModule,
|
||||||
|
sortIndex = module.sortIndex,
|
||||||
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
||||||
|
displayType = Some(entry.displayType.toString)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case _ => Seq.empty
|
||||||
|
}.flatten
|
||||||
|
// Other instructions
|
||||||
|
case _ => Seq.empty[thrift.EntryInfo]
|
||||||
|
}.flatten.map { entryInfo =>
|
||||||
|
thrift.ServedEntry(
|
||||||
|
entry = Some(entryInfo),
|
||||||
|
request = requestInfo
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] =
|
||||||
|
scribeEventPublisher
|
||||||
|
|
||||||
|
override val alerts = Seq(
|
||||||
|
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
|
||||||
|
)
|
||||||
|
}
|
@ -1,212 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.side_effect
|
|
||||||
|
|
||||||
import com.twitter.finagle.tracing.Trace
|
|
||||||
import com.twitter.home_mixer.marshaller.timeline_logging.ConversationEntryMarshaller
|
|
||||||
import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetEntryMarshaller
|
|
||||||
import com.twitter.home_mixer.marshaller.timeline_logging.TweetEntryMarshaller
|
|
||||||
import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowEntryMarshaller
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
|
|
||||||
import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
|
|
||||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
|
||||||
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
|
||||||
import com.twitter.logpipeline.client.common.EventPublisher
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
|
|
||||||
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
|
|
||||||
import com.twitter.util.Time
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Side effect that logs home timeline served entries to Scribe.
|
|
||||||
*/
|
|
||||||
@Singleton
|
|
||||||
class HomeScribeServedEntriesSideEffect @Inject() (
|
|
||||||
scribeEventPublisher: EventPublisher[thrift.Timeline])
|
|
||||||
extends PipelineResultSideEffect[
|
|
||||||
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
||||||
Timeline
|
|
||||||
] {
|
|
||||||
|
|
||||||
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedEntries")
|
|
||||||
|
|
||||||
final override def apply(
|
|
||||||
inputs: PipelineResultSideEffect.Inputs[
|
|
||||||
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
||||||
Timeline
|
|
||||||
]
|
|
||||||
): Stitch[Unit] = {
|
|
||||||
val timelineThrift = buildTimeline(inputs)
|
|
||||||
Stitch.callFuture(scribeEventPublisher.publish(timelineThrift)).unit
|
|
||||||
}
|
|
||||||
|
|
||||||
def buildTimeline(
|
|
||||||
inputs: PipelineResultSideEffect.Inputs[
|
|
||||||
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
||||||
Timeline
|
|
||||||
]
|
|
||||||
): thrift.Timeline = {
|
|
||||||
val timelineType = inputs.query.product match {
|
|
||||||
case FollowingProduct => thrift.TimelineType.HomeLatest
|
|
||||||
case ForYouProduct => thrift.TimelineType.Home
|
|
||||||
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
|
||||||
}
|
|
||||||
val requestProvenance = inputs.query.deviceContext.map { deviceContext =>
|
|
||||||
deviceContext.requestContextValue match {
|
|
||||||
case RequestContext.Foreground => thrift.RequestProvenance.Foreground
|
|
||||||
case RequestContext.Launch => thrift.RequestProvenance.Launch
|
|
||||||
case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
|
|
||||||
case _ => thrift.RequestProvenance.Other
|
|
||||||
}
|
|
||||||
}
|
|
||||||
val queryType = inputs.query.features.map { featureMap =>
|
|
||||||
if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
|
|
||||||
else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
|
|
||||||
else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
|
|
||||||
else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
|
|
||||||
else thrift.QueryType.Other
|
|
||||||
}
|
|
||||||
|
|
||||||
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
|
||||||
inputs.selectedCandidates.flatMap {
|
|
||||||
case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
|
|
||||||
Seq((item.candidateIdLong, item))
|
|
||||||
case module: ModuleCandidateWithDetails
|
|
||||||
if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
|
|
||||||
module.candidates.map(item => (item.candidateIdLong, item))
|
|
||||||
case _ => Seq.empty
|
|
||||||
}.toMap
|
|
||||||
|
|
||||||
val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
|
||||||
inputs.selectedCandidates.flatMap {
|
|
||||||
case module: ModuleCandidateWithDetails
|
|
||||||
if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
|
|
||||||
module.candidates.map { item =>
|
|
||||||
(item.candidateIdLong, item)
|
|
||||||
}
|
|
||||||
case _ => Seq.empty
|
|
||||||
}.toMap
|
|
||||||
|
|
||||||
val timelineEntries = inputs.response.instructions.zipWithIndex.collect {
|
|
||||||
case (AddEntriesTimelineInstruction(entries), index) =>
|
|
||||||
entries.collect {
|
|
||||||
case entry: TweetItem if entry.promotedMetadata.isDefined =>
|
|
||||||
val promotedTweetEntry = PromotedTweetEntryMarshaller(entry, index)
|
|
||||||
Seq(
|
|
||||||
thrift.TimelineEntry(
|
|
||||||
content = thrift.Content.PromotedTweetEntry(promotedTweetEntry),
|
|
||||||
position = index.shortValue(),
|
|
||||||
entryId = entry.entryIdentifier,
|
|
||||||
entryType = thrift.EntryType.PromotedTweet,
|
|
||||||
sortIndex = entry.sortIndex,
|
|
||||||
verticalSize = Some(1)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case entry: TweetItem =>
|
|
||||||
val candidate = tweetIdToItemCandidateMap(entry.id)
|
|
||||||
val tweetEntry = TweetEntryMarshaller(entry, candidate)
|
|
||||||
Seq(
|
|
||||||
thrift.TimelineEntry(
|
|
||||||
content = thrift.Content.TweetEntry(tweetEntry),
|
|
||||||
position = index.shortValue(),
|
|
||||||
entryId = entry.entryIdentifier,
|
|
||||||
entryType = thrift.EntryType.Tweet,
|
|
||||||
sortIndex = entry.sortIndex,
|
|
||||||
verticalSize = Some(1)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case module: TimelineModule
|
|
||||||
if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
|
|
||||||
val whoToFollowEntries = module.items.collect {
|
|
||||||
case ModuleItem(entry: UserItem, _, _) =>
|
|
||||||
val candidate = userIdToItemCandidateMap(entry.id)
|
|
||||||
val whoToFollowEntry = WhoToFollowEntryMarshaller(entry, candidate)
|
|
||||||
thrift.AtomicEntry.WtfEntry(whoToFollowEntry)
|
|
||||||
}
|
|
||||||
Seq(
|
|
||||||
thrift.TimelineEntry(
|
|
||||||
content = thrift.Content.Entries(whoToFollowEntries),
|
|
||||||
position = index.shortValue(),
|
|
||||||
entryId = module.entryIdentifier,
|
|
||||||
entryType = thrift.EntryType.WhoToFollowModule,
|
|
||||||
sortIndex = module.sortIndex
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case module: TimelineModule
|
|
||||||
if module.sortIndex.isDefined && module.items.headOption.exists(
|
|
||||||
_.item.isInstanceOf[TweetItem]) =>
|
|
||||||
val conversationTweetEntries = module.items.collect {
|
|
||||||
case ModuleItem(entry: TweetItem, _, _) =>
|
|
||||||
val candidate = tweetIdToItemCandidateMap(entry.id)
|
|
||||||
val conversationEntry = ConversationEntryMarshaller(entry, candidate)
|
|
||||||
thrift.AtomicEntry.ConversationEntry(conversationEntry)
|
|
||||||
}
|
|
||||||
Seq(
|
|
||||||
thrift.TimelineEntry(
|
|
||||||
content = thrift.Content.Entries(conversationTweetEntries),
|
|
||||||
position = index.shortValue(),
|
|
||||||
entryId = module.entryIdentifier,
|
|
||||||
entryType = thrift.EntryType.ConversationModule,
|
|
||||||
sortIndex = module.sortIndex
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case _ => Seq.empty
|
|
||||||
}.flatten
|
|
||||||
// Other instructions
|
|
||||||
case _ => Seq.empty[thrift.TimelineEntry]
|
|
||||||
}.flatten
|
|
||||||
|
|
||||||
thrift.Timeline(
|
|
||||||
timelineEntries = timelineEntries,
|
|
||||||
requestTimeMs = inputs.query.queryTime.inMilliseconds,
|
|
||||||
traceId = Trace.id.traceId.toLong,
|
|
||||||
userId = inputs.query.getOptionalUserId,
|
|
||||||
clientAppId = inputs.query.clientContext.appId,
|
|
||||||
sourceJobInstance = None,
|
|
||||||
hasDarkRequest = inputs.query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
|
|
||||||
parentId = Some(Trace.id.parentId.toLong),
|
|
||||||
spanId = Some(Trace.id.spanId.toLong),
|
|
||||||
timelineType = Some(timelineType),
|
|
||||||
ipAddress = inputs.query.clientContext.ipAddress,
|
|
||||||
userAgent = inputs.query.clientContext.userAgent,
|
|
||||||
queryType = queryType,
|
|
||||||
requestProvenance = requestProvenance,
|
|
||||||
sessionId = None,
|
|
||||||
timeZone = None,
|
|
||||||
browserNotificationPermission = None,
|
|
||||||
lastNonePollingTimeMs = None,
|
|
||||||
languageCode = inputs.query.clientContext.languageCode,
|
|
||||||
countryCode = inputs.query.clientContext.countryCode,
|
|
||||||
requestEndTimeMs = Some(Time.now.inMilliseconds),
|
|
||||||
servedRequestId = inputs.query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
|
|
||||||
requestJoinId = inputs.query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)),
|
|
||||||
requestSeenTweetIds = inputs.query.seenTweetIds
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
override val alerts = Seq(
|
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
|
|
||||||
)
|
|
||||||
}
|
|
@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.side_effect
|
|||||||
import com.twitter.eventbus.client.EventBusPublisher
|
import com.twitter.eventbus.client.EventBusPublisher
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
@ -22,6 +23,7 @@ import javax.inject.Singleton
|
|||||||
object PublishClientSentImpressionsEventBusSideEffect {
|
object PublishClientSentImpressionsEventBusSideEffect {
|
||||||
val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline))
|
val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline))
|
||||||
val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline))
|
val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline))
|
||||||
|
val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed))
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -56,6 +58,7 @@ class PublishClientSentImpressionsEventBusSideEffect @Inject() (
|
|||||||
val surfaceArea = query.product match {
|
val surfaceArea = query.product match {
|
||||||
case ForYouProduct => HomeSurfaceArea
|
case ForYouProduct => HomeSurfaceArea
|
||||||
case FollowingProduct => HomeLatestSurfaceArea
|
case FollowingProduct => HomeLatestSurfaceArea
|
||||||
|
case SubscribedProduct => HomeSubscribedSurfaceArea
|
||||||
case _ => None
|
case _ => None
|
||||||
}
|
}
|
||||||
query.seenTweetIds.map { seenTweetIds =>
|
query.seenTweetIds.map { seenTweetIds =>
|
||||||
|
@ -1,39 +1,46 @@
|
|||||||
package com.twitter.home_mixer.functional_component.side_effect
|
package com.twitter.home_mixer.functional_component.side_effect
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
|
||||||
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
||||||
|
import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
|
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
|
||||||
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
|
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBloomFilter)
|
class PublishImpressionBloomFilterSideEffect @Inject() (
|
||||||
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, Timeline]
|
bloomFilterClient: ManhattanStoreClient[
|
||||||
with PipelineResultSideEffect.Conditionally[PipelineQuery with HasSeenTweetIds, Timeline] {
|
blm.ImpressionBloomFilterKey,
|
||||||
|
blm.ImpressionBloomFilterSeq
|
||||||
private val SurfaceArea = t.SurfaceArea.HomeTimeline
|
]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling]
|
||||||
|
with PipelineResultSideEffect.Conditionally[
|
||||||
|
PipelineQuery with HasSeenTweetIds,
|
||||||
|
HasMarshalling
|
||||||
|
] {
|
||||||
|
|
||||||
override val identifier: SideEffectIdentifier =
|
override val identifier: SideEffectIdentifier =
|
||||||
SideEffectIdentifier("UpdateImpressionBloomFilter")
|
SideEffectIdentifier("PublishImpressionBloomFilter")
|
||||||
|
|
||||||
|
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
|
||||||
|
|
||||||
override def onlyIf(
|
override def onlyIf(
|
||||||
query: PipelineQuery with HasSeenTweetIds,
|
query: PipelineQuery with HasSeenTweetIds,
|
||||||
selectedCandidates: Seq[CandidateWithDetails],
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
remainingCandidates: Seq[CandidateWithDetails],
|
remainingCandidates: Seq[CandidateWithDetails],
|
||||||
droppedCandidates: Seq[CandidateWithDetails],
|
droppedCandidates: Seq[CandidateWithDetails],
|
||||||
response: Timeline
|
response: HasMarshalling
|
||||||
): Boolean = query.seenTweetIds.exists(_.nonEmpty)
|
): Boolean =
|
||||||
|
query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty)
|
||||||
|
|
||||||
def buildEvents(query: PipelineQuery): Option[t.ImpressionBloomFilterSeq] = {
|
def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = {
|
||||||
query.features.flatMap { featureMap =>
|
query.features.flatMap { featureMap =>
|
||||||
val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature)
|
val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature)
|
||||||
if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq)
|
if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq)
|
||||||
@ -42,19 +49,17 @@ class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBl
|
|||||||
}
|
}
|
||||||
|
|
||||||
override def apply(
|
override def apply(
|
||||||
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, Timeline]
|
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling]
|
||||||
): Stitch[Unit] = {
|
): Stitch[Unit] = {
|
||||||
buildEvents(inputs.query)
|
buildEvents(inputs.query)
|
||||||
.map { updatedBloomFilter =>
|
.map { updatedBloomFilterSeq =>
|
||||||
bloomFilter.writeBloomFilterSeq(
|
bloomFilterClient.write(
|
||||||
userId = inputs.query.getRequiredUserId,
|
blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea),
|
||||||
surfaceArea = SurfaceArea,
|
updatedBloomFilterSeq)
|
||||||
impressionBloomFilterSeq = updatedBloomFilter)
|
|
||||||
}.getOrElse(Stitch.Unit)
|
}.getOrElse(Stitch.Unit)
|
||||||
}
|
}
|
||||||
|
|
||||||
override val alerts = Seq(
|
override val alerts = Seq(
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8),
|
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8)
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(30.millis)
|
|
||||||
)
|
)
|
||||||
}
|
}
|
@ -1,80 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.side_effect
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.AuthorListForStatsParam
|
|
||||||
import com.twitter.home_mixer.util.CandidatesUtil
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver)
|
|
||||||
extends PipelineResultSideEffect[PipelineQuery, Timeline] {
|
|
||||||
|
|
||||||
override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats")
|
|
||||||
|
|
||||||
private val baseStatsReceiver = statsReceiver.scope(identifier.toString)
|
|
||||||
private val authorStatsReceiver = baseStatsReceiver.scope("Author")
|
|
||||||
private val candidateSourceStatsReceiver = baseStatsReceiver.scope("CandidateSource")
|
|
||||||
private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance")
|
|
||||||
private val inNetworkStatsCounter = contentBalanceStatsReceiver.counter("InNetwork")
|
|
||||||
private val outOfNetworkStatsCounter = contentBalanceStatsReceiver.counter("OutOfNetwork")
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
|
|
||||||
): Stitch[Unit] = {
|
|
||||||
val tweetCandidates = CandidatesUtil
|
|
||||||
.getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]())
|
|
||||||
|
|
||||||
recordAuthorStats(tweetCandidates, inputs.query.params(AuthorListForStatsParam))
|
|
||||||
recordCandidateSourceStats(tweetCandidates)
|
|
||||||
recordContentBalanceStats(tweetCandidates)
|
|
||||||
Stitch.Unit
|
|
||||||
}
|
|
||||||
|
|
||||||
def recordAuthorStats(candidates: Seq[CandidateWithDetails], authors: Set[Long]): Unit = {
|
|
||||||
candidates
|
|
||||||
.filter { candidate =>
|
|
||||||
candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) &&
|
|
||||||
// Only include original tweets
|
|
||||||
(!candidate.features.getOrElse(IsRetweetFeature, false)) &&
|
|
||||||
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
|
|
||||||
}
|
|
||||||
.groupBy { candidate =>
|
|
||||||
(getCandidateSourceId(candidate), candidate.features.get(AuthorIdFeature).get)
|
|
||||||
}
|
|
||||||
.foreach {
|
|
||||||
case ((candidateSourceId, authorId), authorCandidates) =>
|
|
||||||
authorStatsReceiver
|
|
||||||
.scope(authorId.toString).counter(candidateSourceId).incr(authorCandidates.size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def recordCandidateSourceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
|
|
||||||
candidates.groupBy(getCandidateSourceId).foreach {
|
|
||||||
case (candidateSourceId, candidateSourceCandidates) =>
|
|
||||||
candidateSourceStatsReceiver.counter(candidateSourceId).incr(candidateSourceCandidates.size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def recordContentBalanceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
|
|
||||||
val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true))
|
|
||||||
inNetworkStatsCounter.incr(in.size)
|
|
||||||
outOfNetworkStatsCounter.incr(oon.size)
|
|
||||||
}
|
|
||||||
|
|
||||||
private def getCandidateSourceId(candidate: CandidateWithDetails): String =
|
|
||||||
candidate.features.getOrElse(CandidateSourceIdFeature, None).map(_.name).getOrElse("None")
|
|
||||||
}
|
|
@ -3,8 +3,10 @@ package com.twitter.home_mixer.functional_component.side_effect
|
|||||||
import com.twitter.home_mixer.model.HomeFeatures._
|
import com.twitter.home_mixer.model.HomeFeatures._
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
||||||
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
||||||
@ -97,12 +99,14 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
|
|||||||
val entries = inputs.response.instructions.collect {
|
val entries = inputs.response.instructions.collect {
|
||||||
case AddEntriesTimelineInstruction(entries) =>
|
case AddEntriesTimelineInstruction(entries) =>
|
||||||
entries.collect {
|
entries.collect {
|
||||||
// includes both tweets and promoted tweets
|
// includes tweets, tweet previews, and promoted tweets
|
||||||
case entry: TweetItem if entry.sortIndex.isDefined =>
|
case entry: TweetItem if entry.sortIndex.isDefined => {
|
||||||
Seq(
|
Seq(
|
||||||
buildTweetEntryWithItemIds(
|
buildTweetEntryWithItemIds(
|
||||||
tweetIdToItemCandidateMap(entry.id),
|
tweetIdToItemCandidateMap(entry.id),
|
||||||
entry.sortIndex.get))
|
entry.sortIndex.get
|
||||||
|
))
|
||||||
|
}
|
||||||
// tweet conversation modules are flattened to individual tweets in the persistence store
|
// tweet conversation modules are flattened to individual tweets in the persistence store
|
||||||
case module: TimelineModule
|
case module: TimelineModule
|
||||||
if module.sortIndex.isDefined && module.items.headOption.exists(
|
if module.sortIndex.isDefined && module.items.headOption.exists(
|
||||||
@ -125,6 +129,19 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
|
|||||||
size = module.items.size.toShort,
|
size = module.items.size.toShort,
|
||||||
itemIds = Some(userIds)
|
itemIds = Some(userIds)
|
||||||
))
|
))
|
||||||
|
case module: TimelineModule
|
||||||
|
if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
|
||||||
|
val userIds = module.items
|
||||||
|
.map(item =>
|
||||||
|
UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId =
|
||||||
|
Some(item.item.id.asInstanceOf[Long])))
|
||||||
|
Seq(
|
||||||
|
EntryWithItemIds(
|
||||||
|
entityIdType = EntityIdType.WhoToSubscribe,
|
||||||
|
sortIndex = module.sortIndex.get,
|
||||||
|
size = module.items.size.toShort,
|
||||||
|
itemIds = Some(userIds)
|
||||||
|
))
|
||||||
}.flatten
|
}.flatten
|
||||||
case ShowCoverInstruction(cover) =>
|
case ShowCoverInstruction(cover) =>
|
||||||
Seq(
|
Seq(
|
||||||
@ -216,8 +233,11 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
|
|||||||
userId = None
|
userId = None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false)
|
||||||
|
val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet
|
||||||
|
|
||||||
EntryWithItemIds(
|
EntryWithItemIds(
|
||||||
entityIdType = EntityIdType.Tweet,
|
entityIdType = entityType,
|
||||||
sortIndex = sortIndex,
|
sortIndex = sortIndex,
|
||||||
size = 1.toShort,
|
size = 1.toShort,
|
||||||
itemIds = Some(Seq(itemIds))
|
itemIds = Some(Seq(itemIds))
|
||||||
|
@ -4,14 +4,12 @@ scala_library(
|
|||||||
strict_deps = True,
|
strict_deps = True,
|
||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||||
],
|
],
|
||||||
exports = [
|
exports = [
|
||||||
"dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
||||||
|
@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProductContext
|
|||||||
import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext
|
import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext
|
||||||
import com.twitter.home_mixer.model.request.ListTweetsProductContext
|
import com.twitter.home_mixer.model.request.ListTweetsProductContext
|
||||||
import com.twitter.home_mixer.model.request.ScoredTweetsProductContext
|
import com.twitter.home_mixer.model.request.ScoredTweetsProductContext
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProductContext
|
||||||
import com.twitter.home_mixer.{thriftscala => t}
|
import com.twitter.home_mixer.{thriftscala => t}
|
||||||
import com.twitter.product_mixer.core.model.marshalling.request.ProductContext
|
import com.twitter.product_mixer.core.model.marshalling.request.ProductContext
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
@ -26,15 +26,17 @@ class HomeMixerProductContextUnmarshaller @Inject() (
|
|||||||
ForYouProductContext(
|
ForYouProductContext(
|
||||||
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
||||||
seenTweetIds = p.seenTweetIds,
|
seenTweetIds = p.seenTweetIds,
|
||||||
dspClientContext = p.dspClientContext
|
dspClientContext = p.dspClientContext,
|
||||||
|
pushToHomeTweetId = p.pushToHomeTweetId
|
||||||
)
|
)
|
||||||
case t.ProductContext.Realtime(p) =>
|
case t.ProductContext.ListManagement(p) =>
|
||||||
throw new UnsupportedOperationException(s"This product is no longer used")
|
throw new UnsupportedOperationException(s"This product is no longer used")
|
||||||
case t.ProductContext.ScoredTweets(p) =>
|
case t.ProductContext.ScoredTweets(p) =>
|
||||||
ScoredTweetsProductContext(
|
ScoredTweetsProductContext(
|
||||||
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
||||||
seenTweetIds = p.seenTweetIds,
|
seenTweetIds = p.seenTweetIds,
|
||||||
servedTweetIds = p.servedTweetIds
|
servedTweetIds = p.servedTweetIds,
|
||||||
|
backfillTweetIds = p.backfillTweetIds
|
||||||
)
|
)
|
||||||
case t.ProductContext.ListTweets(p) =>
|
case t.ProductContext.ListTweets(p) =>
|
||||||
ListTweetsProductContext(
|
ListTweetsProductContext(
|
||||||
@ -46,7 +48,13 @@ class HomeMixerProductContextUnmarshaller @Inject() (
|
|||||||
ListRecommendedUsersProductContext(
|
ListRecommendedUsersProductContext(
|
||||||
listId = p.listId,
|
listId = p.listId,
|
||||||
selectedUserIds = p.selectedUserIds,
|
selectedUserIds = p.selectedUserIds,
|
||||||
excludedUserIds = p.excludedUserIds
|
excludedUserIds = p.excludedUserIds,
|
||||||
|
listName = p.listName
|
||||||
|
)
|
||||||
|
case t.ProductContext.Subscribed(p) =>
|
||||||
|
SubscribedProductContext(
|
||||||
|
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
||||||
|
seenTweetIds = p.seenTweetIds,
|
||||||
)
|
)
|
||||||
case t.ProductContext.UnknownUnionField(field) =>
|
case t.ProductContext.UnknownUnionField(field) =>
|
||||||
throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}")
|
throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}")
|
||||||
|
@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProduct
|
|||||||
import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct
|
import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct
|
||||||
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
||||||
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
import com.twitter.home_mixer.{thriftscala => t}
|
import com.twitter.home_mixer.{thriftscala => t}
|
||||||
import com.twitter.product_mixer.core.model.marshalling.request.Product
|
import com.twitter.product_mixer.core.model.marshalling.request.Product
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
@ -17,11 +17,12 @@ class HomeMixerProductUnmarshaller @Inject() () {
|
|||||||
def apply(product: t.Product): Product = product match {
|
def apply(product: t.Product): Product = product match {
|
||||||
case t.Product.Following => FollowingProduct
|
case t.Product.Following => FollowingProduct
|
||||||
case t.Product.ForYou => ForYouProduct
|
case t.Product.ForYou => ForYouProduct
|
||||||
case t.Product.Realtime =>
|
case t.Product.ListManagement =>
|
||||||
throw new UnsupportedOperationException(s"This product is no longer used")
|
throw new UnsupportedOperationException(s"This product is no longer used")
|
||||||
case t.Product.ScoredTweets => ScoredTweetsProduct
|
case t.Product.ScoredTweets => ScoredTweetsProduct
|
||||||
case t.Product.ListTweets => ListTweetsProduct
|
case t.Product.ListTweets => ListTweetsProduct
|
||||||
case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct
|
case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct
|
||||||
|
case t.Product.Subscribed => SubscribedProduct
|
||||||
case t.Product.EnumUnknownProduct(value) =>
|
case t.Product.EnumUnknownProduct(value) =>
|
||||||
throw new UnsupportedOperationException(s"Unknown product: $value")
|
throw new UnsupportedOperationException(s"Unknown product: $value")
|
||||||
}
|
}
|
||||||
|
@ -1,16 +0,0 @@
|
|||||||
package com.twitter.home_mixer.marshaller.timeline_logging
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
|
||||||
|
|
||||||
object ConversationEntryMarshaller {
|
|
||||||
|
|
||||||
def apply(entry: TweetItem, candidate: ItemCandidateWithDetails): thriftlog.ConversationEntry =
|
|
||||||
thriftlog.ConversationEntry(
|
|
||||||
displayedTweetId = entry.id,
|
|
||||||
displayType = Some(entry.displayType.toString),
|
|
||||||
score = candidate.features.getOrElse(ScoreFeature, None)
|
|
||||||
)
|
|
||||||
}
|
|
@ -3,15 +3,13 @@ package com.twitter.home_mixer.marshaller.timeline_logging
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
||||||
|
|
||||||
object PromotedTweetEntryMarshaller {
|
object PromotedTweetDetailsMarshaller {
|
||||||
|
|
||||||
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetEntry = {
|
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetDetails = {
|
||||||
thriftlog.PromotedTweetEntry(
|
thriftlog.PromotedTweetDetails(
|
||||||
id = entry.id,
|
advertiserId = Some(entry.promotedMetadata.map(_.advertiserId).getOrElse(0L)),
|
||||||
advertiserId = entry.promotedMetadata.map(_.advertiserId).getOrElse(0L),
|
insertPosition = Some(position),
|
||||||
insertPosition = position,
|
impressionId = entry.promotedMetadata.flatMap(_.impressionString)
|
||||||
impressionId = entry.promotedMetadata.flatMap(_.impressionString),
|
|
||||||
displayType = Some(entry.displayType.toString)
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -0,0 +1,47 @@
|
|||||||
|
package com.twitter.home_mixer.marshaller.timeline_logging
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation
|
||||||
|
import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation
|
||||||
|
import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller
|
||||||
|
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ConversationGeneralContextType
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext
|
||||||
|
import com.twitter.timelines.service.{thriftscala => tst}
|
||||||
|
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
||||||
|
|
||||||
|
object TweetDetailsMarshaller {
|
||||||
|
|
||||||
|
private val generalContextTypeMarshaller = new GeneralContextTypeMarshaller()
|
||||||
|
|
||||||
|
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetDetails = {
|
||||||
|
val socialContext = candidate.presentation.flatMap {
|
||||||
|
case _ @UrtItemPresentation(timelineItem: TweetItem, _) => timelineItem.socialContext
|
||||||
|
case _ @UrtModulePresentation(timelineModule) =>
|
||||||
|
timelineModule.items.head.item match {
|
||||||
|
case timelineItem: TweetItem => timelineItem.socialContext
|
||||||
|
case _ => Some(ConversationGeneralContextType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val socialContextType = socialContext match {
|
||||||
|
case Some(GeneralContext(contextType, _, _, _, _)) =>
|
||||||
|
Some(generalContextTypeMarshaller(contextType).value.toShort)
|
||||||
|
case Some(TopicContext(_, _)) => Some(tst.ContextType.Topic.value.toShort)
|
||||||
|
case _ => None
|
||||||
|
}
|
||||||
|
|
||||||
|
thriftlog.TweetDetails(
|
||||||
|
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
|
||||||
|
socialContextType = socialContextType,
|
||||||
|
suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name),
|
||||||
|
authorId = candidate.features.getOrElse(AuthorIdFeature, None),
|
||||||
|
sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
@ -1,29 +0,0 @@
|
|||||||
package com.twitter.home_mixer.marshaller.timeline_logging
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SocialContextFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
|
||||||
import com.twitter.timelines.service.{thriftscala => tst}
|
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
|
||||||
|
|
||||||
object TweetEntryMarshaller {
|
|
||||||
|
|
||||||
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetEntry = {
|
|
||||||
val socialContextType = candidate.features.getOrElse(SocialContextFeature, None) match {
|
|
||||||
case Some(tst.SocialContext.GeneralContext(tst.GeneralContext(contextType, _, _, _, _))) =>
|
|
||||||
Some(contextType.value.toShort)
|
|
||||||
case Some(tst.SocialContext.TopicContext(_)) =>
|
|
||||||
Some(tst.ContextType.Topic.value.toShort)
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
thriftlog.TweetEntry(
|
|
||||||
id = candidate.candidateIdLong,
|
|
||||||
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
|
|
||||||
displayType = Some(entry.displayType.toString),
|
|
||||||
score = candidate.features.getOrElse(ScoreFeature, None),
|
|
||||||
socialContextType = socialContextType
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,17 +1,13 @@
|
|||||||
package com.twitter.home_mixer.marshaller.timeline_logging
|
package com.twitter.home_mixer.marshaller.timeline_logging
|
||||||
|
|
||||||
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.ScoreFeature
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
||||||
|
|
||||||
object WhoToFollowEntryMarshaller {
|
object WhoToFollowDetailsMarshaller {
|
||||||
|
|
||||||
def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowEntry =
|
def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowDetails =
|
||||||
thriftlog.WhoToFollowEntry(
|
thriftlog.WhoToFollowDetails(
|
||||||
userId = entry.id,
|
|
||||||
displayType = Some(entry.displayType.toString),
|
|
||||||
score = candidate.features.getOrElse(ScoreFeature, None),
|
|
||||||
enableReactiveBlending = entry.enableReactiveBlending,
|
enableReactiveBlending = entry.enableReactiveBlending,
|
||||||
impressionId = entry.promotedMetadata.flatMap(_.impressionString),
|
impressionId = entry.promotedMetadata.flatMap(_.impressionString),
|
||||||
advertiserId = entry.promotedMetadata.map(_.advertiserId)
|
advertiserId = entry.promotedMetadata.map(_.advertiserId)
|
@ -4,13 +4,8 @@ scala_library(
|
|||||||
strict_deps = True,
|
strict_deps = True,
|
||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"3rdparty/jvm/javax/inject:javax.inject",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt",
|
|
||||||
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
|
||||||
"src/thrift/com/twitter/timelineservice:thrift-scala",
|
"src/thrift/com/twitter/timelineservice:thrift-scala",
|
||||||
"timelineservice/common:model",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user