mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-12-22 18:21:51 +01:00
Compare commits
3 Commits
2204b4c508
...
14d62a80c7
Author | SHA1 | Date | |
---|---|---|---|
|
14d62a80c7 | ||
|
72eda9a24f | ||
|
948a1950ce |
@ -62,6 +62,9 @@ The core components of Recommended Notifications included in this repository are
|
|||||||
|
|
||||||
We include Bazel BUILD files for most components, but not a top-level BUILD or WORKSPACE file. We plan to add a more complete build and test system in the future.
|
We include Bazel BUILD files for most components, but not a top-level BUILD or WORKSPACE file. We plan to add a more complete build and test system in the future.
|
||||||
|
|
||||||
|
# Summary
|
||||||
|
The Twitter Recommendation Algorithm is a system that recommends content to users by using different components like community detection, knowledge graph embeddings, and models for detecting abusive content. It also provides recommendations for accounts to follow and Tweets to see. The system ranks Tweets based on various signals, and filters content to ensure legal compliance and user trust. Overall, the algorithm uses advanced machine learning and data processing techniques to deliver personalized content to Twitter users.
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
We invite the community to submit GitHub issues and pull requests for suggestions on improving the recommendation algorithm. We are working on tools to manage these suggestions and sync changes to our internal repository. Any security concerns or issues should be routed to our official [bug bounty program](https://hackerone.com/twitter) through HackerOne. We hope to benefit from the collective intelligence and expertise of the global community in helping us identify issues and suggest improvements, ultimately leading to a better Twitter.
|
We invite the community to submit GitHub issues and pull requests for suggestions on improving the recommendation algorithm. We are working on tools to manage these suggestions and sync changes to our internal repository. Any security concerns or issues should be routed to our official [bug bounty program](https://hackerone.com/twitter) through HackerOne. We hope to benefit from the collective intelligence and expertise of the global community in helping us identify issues and suggest improvements, ultimately leading to a better Twitter.
|
||||||
|
@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
|
|||||||
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
|
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
|
||||||
- Fetch Tweet Candidates
|
- Fetch Tweet Candidates
|
||||||
- ScoredTweetsInNetworkCandidatePipelineConfig
|
- ScoredTweetsInNetworkCandidatePipelineConfig
|
||||||
- ScoredTweetsCrMixerCandidatePipelineConfig
|
- ScoredTweetsTweetMixerCandidatePipelineConfig
|
||||||
- ScoredTweetsUtegCandidatePipelineConfig
|
- ScoredTweetsUtegCandidatePipelineConfig
|
||||||
- ScoredTweetsFrsCandidatePipelineConfig
|
- ScoredTweetsFrsCandidatePipelineConfig
|
||||||
- Feature Hydration and Scoring
|
- Feature Hydration and Scoring
|
||||||
@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer.
|
|||||||
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
|
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
|
||||||
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
|
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
|
||||||
- ListTweetsAdsCandidatePipelineConfig (fetch ads)
|
- ListTweetsAdsCandidatePipelineConfig (fetch ads)
|
||||||
|
|
||||||
|
@ -21,6 +21,7 @@ scala_library(
|
|||||||
"finatra/inject/inject-utils/src/main/scala",
|
"finatra/inject/inject-utils/src/main/scala",
|
||||||
"home-mixer/server/src/main/resources",
|
"home-mixer/server/src/main/resources",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
|
||||||
@ -31,6 +32,10 @@ scala_library(
|
|||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
|
||||||
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
|
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
||||||
|
"strato/config/columns/auth-context:auth-context-strato-client",
|
||||||
|
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/fed",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||||
"stringcenter/client",
|
"stringcenter/client",
|
||||||
"stringcenter/client/src/main/java",
|
"stringcenter/client/src/main/java",
|
||||||
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",
|
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",
|
||||||
|
@ -2,7 +2,7 @@ package com.twitter.home_mixer
|
|||||||
|
|
||||||
import com.twitter.finatra.http.routing.HttpWarmup
|
import com.twitter.finatra.http.routing.HttpWarmup
|
||||||
import com.twitter.finatra.httpclient.RequestBuilder._
|
import com.twitter.finatra.httpclient.RequestBuilder._
|
||||||
import com.twitter.inject.Logging
|
import com.twitter.util.logging.Logging
|
||||||
import com.twitter.inject.utils.Handler
|
import com.twitter.inject.utils.Handler
|
||||||
import com.twitter.util.Try
|
import com.twitter.util.Try
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
|
@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer
|
|||||||
import com.twitter.finatra.thrift.filters._
|
import com.twitter.finatra.thrift.filters._
|
||||||
import com.twitter.finatra.thrift.routing.ThriftRouter
|
import com.twitter.finatra.thrift.routing.ThriftRouter
|
||||||
import com.twitter.home_mixer.controller.HomeThriftController
|
import com.twitter.home_mixer.controller.HomeThriftController
|
||||||
|
import com.twitter.home_mixer.federated.HomeMixerColumn
|
||||||
import com.twitter.home_mixer.module._
|
import com.twitter.home_mixer.module._
|
||||||
import com.twitter.home_mixer.param.GlobalParamConfigModule
|
import com.twitter.home_mixer.param.GlobalParamConfigModule
|
||||||
import com.twitter.home_mixer.product.HomeMixerProductModule
|
import com.twitter.home_mixer.product.HomeMixerProductModule
|
||||||
import com.twitter.home_mixer.{thriftscala => st}
|
import com.twitter.home_mixer.{thriftscala => st}
|
||||||
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
|
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
|
||||||
import com.twitter.product_mixer.component_library.module.CrMixerClientModule
|
|
||||||
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
|
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
|
||||||
import com.twitter.product_mixer.component_library.module.EarlybirdModule
|
import com.twitter.product_mixer.component_library.module.EarlybirdModule
|
||||||
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
|
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
|
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
|
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
|
||||||
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
|
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
|
||||||
import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
|
|
||||||
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
|
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
|
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
|
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
|
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
|
||||||
|
import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
|
||||||
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
|
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
|
||||||
import com.twitter.product_mixer.core.controllers.ProductMixerController
|
import com.twitter.product_mixer.core.controllers.ProductMixerController
|
||||||
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
|
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
|
||||||
import com.twitter.product_mixer.core.module.ProductMixerModule
|
import com.twitter.product_mixer.core.module.ProductMixerModule
|
||||||
import com.twitter.product_mixer.core.module.StratoClientModule
|
|
||||||
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
|
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
|
||||||
|
import com.twitter.strato.fed.StratoFed
|
||||||
|
import com.twitter.strato.fed.server.StratoFedServer
|
||||||
|
|
||||||
object HomeMixerServerMain extends HomeMixerServer
|
object HomeMixerServerMain extends HomeMixerServer
|
||||||
|
|
||||||
class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
|
class HomeMixerServer
|
||||||
|
extends StratoFedServer
|
||||||
|
with ThriftServer
|
||||||
|
with Mtls
|
||||||
|
with HttpServer
|
||||||
|
with HttpMtls {
|
||||||
override val name = "home-mixer-server"
|
override val name = "home-mixer-server"
|
||||||
|
|
||||||
override val modules: Seq[Module] = Seq(
|
override val modules: Seq[Module] = Seq(
|
||||||
AccountRecommendationsMixerModule,
|
AccountRecommendationsMixerModule,
|
||||||
AdvertiserBrandSafetySettingsStoreModule,
|
AdvertiserBrandSafetySettingsStoreModule,
|
||||||
|
BlenderClientModule,
|
||||||
ClientSentImpressionsPublisherModule,
|
ClientSentImpressionsPublisherModule,
|
||||||
ConversationServiceModule,
|
ConversationServiceModule,
|
||||||
CrMixerClientModule,
|
|
||||||
EarlybirdModule,
|
EarlybirdModule,
|
||||||
ExploreRankerClientModule,
|
ExploreRankerClientModule,
|
||||||
|
FeedbackHistoryClientModule,
|
||||||
GizmoduckClientModule,
|
GizmoduckClientModule,
|
||||||
GlobalParamConfigModule,
|
GlobalParamConfigModule,
|
||||||
HomeAdsCandidateSourceModule,
|
HomeAdsCandidateSourceModule,
|
||||||
HomeMixerFlagsModule,
|
HomeMixerFlagsModule,
|
||||||
HomeMixerProductModule,
|
HomeMixerProductModule,
|
||||||
HomeMixerResourcesModule,
|
HomeMixerResourcesModule,
|
||||||
HomeNaviModelClientModule,
|
|
||||||
ImpressionBloomFilterModule,
|
ImpressionBloomFilterModule,
|
||||||
InjectionHistoryClientModule,
|
InjectionHistoryClientModule,
|
||||||
FeedbackHistoryClientModule,
|
|
||||||
ManhattanClientsModule,
|
ManhattanClientsModule,
|
||||||
ManhattanFeatureRepositoryModule,
|
ManhattanFeatureRepositoryModule,
|
||||||
ManhattanTweetImpressionStoreModule,
|
ManhattanTweetImpressionStoreModule,
|
||||||
MemcachedFeatureRepositoryModule,
|
MemcachedFeatureRepositoryModule,
|
||||||
|
NaviModelClientModule,
|
||||||
OnboardingTaskServiceModule,
|
OnboardingTaskServiceModule,
|
||||||
OptimizedStratoClientModule,
|
OptimizedStratoClientModule,
|
||||||
PeopleDiscoveryServiceModule,
|
PeopleDiscoveryServiceModule,
|
||||||
@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
|
|||||||
SimClustersRecentEngagementsClientModule,
|
SimClustersRecentEngagementsClientModule,
|
||||||
SocialGraphServiceModule,
|
SocialGraphServiceModule,
|
||||||
StaleTweetsCacheModule,
|
StaleTweetsCacheModule,
|
||||||
StratoClientModule,
|
|
||||||
ThriftFeatureRepositoryModule,
|
ThriftFeatureRepositoryModule,
|
||||||
TimelineMixerClientModule,
|
|
||||||
TimelineRankerClientModule,
|
TimelineRankerClientModule,
|
||||||
TimelineScorerClientModule,
|
TimelineScorerClientModule,
|
||||||
TimelineServiceClientModule,
|
TimelineServiceClientModule,
|
||||||
TimelinesPersistenceStoreClientModule,
|
TimelinesPersistenceStoreClientModule,
|
||||||
|
TopicSocialProofClientModule,
|
||||||
TweetImpressionStoreModule,
|
TweetImpressionStoreModule,
|
||||||
TweetyPieClientModule,
|
TweetMixerClientModule,
|
||||||
|
TweetypieClientModule,
|
||||||
TweetypieStaticEntitiesCacheClientModule,
|
TweetypieStaticEntitiesCacheClientModule,
|
||||||
UserMetadataStoreModule,
|
|
||||||
UserSessionStoreModule,
|
UserSessionStoreModule,
|
||||||
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
|
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
|
||||||
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
|
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
|
||||||
new ProductScopeStringCenterModule()
|
new ProductScopeStringCenterModule()
|
||||||
)
|
)
|
||||||
|
|
||||||
def configureThrift(router: ThriftRouter): Unit = {
|
override def configureThrift(router: ThriftRouter): Unit = {
|
||||||
router
|
router
|
||||||
.filter[LoggingMDCFilter]
|
.filter[LoggingMDCFilter]
|
||||||
.filter[TraceIdMDCFilter]
|
.filter[TraceIdMDCFilter]
|
||||||
@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
|
|||||||
this.injector,
|
this.injector,
|
||||||
st.HomeMixer.ExecutePipeline))
|
st.HomeMixer.ExecutePipeline))
|
||||||
|
|
||||||
|
override val dest: String = "/s/home-mixer/home-mixer:strato"
|
||||||
|
|
||||||
|
override val columns: Seq[Class[_ <: StratoFed.Column]] =
|
||||||
|
Seq(classOf[HomeMixerColumn])
|
||||||
|
|
||||||
override protected def warmup(): Unit = {
|
override protected def warmup(): Unit = {
|
||||||
handle[HomeMixerThriftServerWarmupHandler]()
|
handle[HomeMixerThriftServerWarmupHandler]()
|
||||||
handle[HomeMixerHttpServerWarmupHandler]()
|
handle[HomeMixerHttpServerWarmupHandler]()
|
||||||
|
@ -3,7 +3,7 @@ package com.twitter.home_mixer
|
|||||||
import com.twitter.finagle.thrift.ClientId
|
import com.twitter.finagle.thrift.ClientId
|
||||||
import com.twitter.finatra.thrift.routing.ThriftWarmup
|
import com.twitter.finatra.thrift.routing.ThriftWarmup
|
||||||
import com.twitter.home_mixer.{thriftscala => st}
|
import com.twitter.home_mixer.{thriftscala => st}
|
||||||
import com.twitter.inject.Logging
|
import com.twitter.util.logging.Logging
|
||||||
import com.twitter.inject.utils.Handler
|
import com.twitter.inject.utils.Handler
|
||||||
import com.twitter.product_mixer.core.{thriftscala => pt}
|
import com.twitter.product_mixer.core.{thriftscala => pt}
|
||||||
import com.twitter.scrooge.Request
|
import com.twitter.scrooge.Request
|
||||||
|
@ -5,19 +5,13 @@ scala_library(
|
|||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
||||||
@ -25,10 +19,6 @@ scala_library(
|
|||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
|
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
|
||||||
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
|
||||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
|
||||||
],
|
],
|
||||||
exports = [
|
exports = [
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
|
||||||
|
@ -1,18 +1,23 @@
|
|||||||
package com.twitter.home_mixer.candidate_pipeline
|
package com.twitter.home_mixer.candidate_pipeline
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
|
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
|
||||||
import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
|
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
|
||||||
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
|
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
|
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
|
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
|
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
|
||||||
import com.twitter.product_mixer.component_library.filter.FeatureFilter
|
import com.twitter.product_mixer.component_library.filter.FeatureFilter
|
||||||
|
import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
|
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
|
||||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||||
@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
|
|||||||
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
||||||
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
||||||
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
||||||
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
|
|
||||||
namesFeatureHydrator: NamesFeatureHydrator,
|
namesFeatureHydrator: NamesFeatureHydrator,
|
||||||
|
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
|
||||||
override val gates: Seq[BaseGate[Query]],
|
override val gates: Seq[BaseGate[Query]],
|
||||||
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
|
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
|
||||||
extends DependentCandidatePipelineConfig[
|
extends DependentCandidatePipelineConfig[
|
||||||
@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
|||||||
val tweetsWithConversationMetadata = candidates.map { candidate =>
|
val tweetsWithConversationMetadata = candidates.map { candidate =>
|
||||||
TweetWithConversationMetadata(
|
TweetWithConversationMetadata(
|
||||||
tweetId = candidate.candidateIdLong,
|
tweetId = candidate.candidateIdLong,
|
||||||
userId = None,
|
userId = candidate.features.getOrElse(AuthorIdFeature, None),
|
||||||
sourceTweetId = None,
|
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
|
||||||
sourceUserId = None,
|
sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
|
||||||
inReplyToTweetId = None,
|
inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
|
||||||
conversationId = None,
|
conversationId = None,
|
||||||
ancestors = Seq.empty
|
ancestors = Seq.empty
|
||||||
)
|
)
|
||||||
@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
|||||||
|
|
||||||
override val preFilterFeatureHydrationPhase1: Seq[
|
override val preFilterFeatureHydrationPhase1: Seq[
|
||||||
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
|
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
|
||||||
] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
|
] = Seq(
|
||||||
|
tweetypieFeatureHydrator,
|
||||||
|
InNetworkFeatureHydrator,
|
||||||
|
)
|
||||||
|
|
||||||
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
|
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
|
||||||
RetweetDeduplicationFilter,
|
RetweetDeduplicationFilter,
|
||||||
@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
|
|||||||
FilterIdentifier(QuotedTweetDroppedFilterId),
|
FilterIdentifier(QuotedTweetDroppedFilterId),
|
||||||
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
|
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
|
||||||
),
|
),
|
||||||
|
invalidSubscriptionTweetFilter,
|
||||||
InvalidConversationModuleFilter
|
InvalidConversationModuleFilter
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -1,9 +1,9 @@
|
|||||||
package com.twitter.home_mixer.candidate_pipeline
|
package com.twitter.home_mixer.candidate_pipeline
|
||||||
|
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
|
||||||
|
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
|
||||||
|
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
|
||||||
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
|
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
|
||||||
@ -15,7 +15,7 @@ import javax.inject.Singleton
|
|||||||
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
|
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
|
||||||
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
conversationServiceCandidateSource: ConversationServiceCandidateSource,
|
||||||
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
|
||||||
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
|
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
|
||||||
namesFeatureHydrator: NamesFeatureHydrator) {
|
namesFeatureHydrator: NamesFeatureHydrator) {
|
||||||
|
|
||||||
def build(
|
def build(
|
||||||
@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
|
|||||||
new ConversationServiceCandidatePipelineConfig(
|
new ConversationServiceCandidatePipelineConfig(
|
||||||
conversationServiceCandidateSource,
|
conversationServiceCandidateSource,
|
||||||
tweetypieFeatureHydrator,
|
tweetypieFeatureHydrator,
|
||||||
socialGraphServiceFeatureHydrator,
|
|
||||||
namesFeatureHydrator,
|
namesFeatureHydrator,
|
||||||
|
invalidSubscriptionTweetFilter,
|
||||||
gates,
|
gates,
|
||||||
decorator
|
decorator
|
||||||
)
|
)
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
package com.twitter.home_mixer.candidate_pipeline
|
package com.twitter.home_mixer.candidate_pipeline
|
||||||
|
|
||||||
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
|
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
|
||||||
import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
|
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
|
||||||
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
|
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
|
@ -13,8 +13,5 @@ scala_library(
|
|||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
|
||||||
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
||||||
"src/thrift/com/twitter/context:twitter-context-scala",
|
|
||||||
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
|
||||||
"twitter-context/src/main/scala",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService
|
|||||||
import com.twitter.home_mixer.{thriftscala => t}
|
import com.twitter.home_mixer.{thriftscala => t}
|
||||||
import com.twitter.product_mixer.core.controllers.DebugTwitterContext
|
import com.twitter.product_mixer.core.controllers.DebugTwitterContext
|
||||||
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
|
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
|
||||||
|
import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
|
||||||
import com.twitter.product_mixer.core.service.urt.UrtService
|
import com.twitter.product_mixer.core.service.urt.UrtService
|
||||||
import com.twitter.snowflake.id.SnowflakeId
|
import com.twitter.snowflake.id.SnowflakeId
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
|
@ -0,0 +1,24 @@
|
|||||||
|
scala_library(
|
||||||
|
sources = ["*.scala"],
|
||||||
|
compiler_option_sets = ["fatal_warnings"],
|
||||||
|
strict_deps = True,
|
||||||
|
tags = ["bazel-compatible"],
|
||||||
|
dependencies = [
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
|
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||||
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||||
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
|
||||||
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
|
||||||
|
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
|
||||||
|
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||||
|
"src/thrift/com/twitter/timelines/render:thrift-scala",
|
||||||
|
"stitch/stitch-repo/src/main/scala",
|
||||||
|
"strato/config/columns/auth-context:auth-context-strato-client",
|
||||||
|
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
|
||||||
|
"strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/callcontext",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/fed",
|
||||||
|
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||||
|
],
|
||||||
|
)
|
@ -0,0 +1,217 @@
|
|||||||
|
package com.twitter.home_mixer.federated
|
||||||
|
|
||||||
|
import com.twitter.gizmoduck.{thriftscala => gd}
|
||||||
|
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
|
||||||
|
import com.twitter.home_mixer.model.request.HomeMixerRequest
|
||||||
|
import com.twitter.home_mixer.{thriftscala => hm}
|
||||||
|
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
|
||||||
|
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
|
||||||
|
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
|
||||||
|
import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
|
||||||
|
import com.twitter.product_mixer.core.{thriftscala => pm}
|
||||||
|
import com.twitter.stitch.Arrow
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
import com.twitter.strato.callcontext.CallContext
|
||||||
|
import com.twitter.strato.catalog.OpMetadata
|
||||||
|
import com.twitter.strato.config._
|
||||||
|
import com.twitter.strato.data._
|
||||||
|
import com.twitter.strato.fed.StratoFed
|
||||||
|
import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
|
||||||
|
import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
|
||||||
|
import com.twitter.strato.graphql.timelines.{thriftscala => gql}
|
||||||
|
import com.twitter.strato.thrift.ScroogeConv
|
||||||
|
import com.twitter.timelines.render.{thriftscala => tr}
|
||||||
|
import com.twitter.util.Try
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
class HomeMixerColumn @Inject() (
|
||||||
|
homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
|
||||||
|
compositeOnUserClientColumn: CompositeOnUserClientColumn,
|
||||||
|
auditIpClientColumn: AuditIpClientColumn,
|
||||||
|
paramsBuilder: ParamsBuilder,
|
||||||
|
productPipelineRegistry: ProductPipelineRegistry)
|
||||||
|
extends StratoFed.Column(HomeMixerColumn.Path)
|
||||||
|
with StratoFed.Fetch.Arrow {
|
||||||
|
|
||||||
|
override val contactInfo: ContactInfo = ContactInfo(
|
||||||
|
contactEmail = "",
|
||||||
|
ldapGroup = "",
|
||||||
|
slackRoomId = ""
|
||||||
|
)
|
||||||
|
|
||||||
|
override val metadata: OpMetadata =
|
||||||
|
OpMetadata(
|
||||||
|
lifecycle = Some(Lifecycle.Production),
|
||||||
|
description =
|
||||||
|
Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
|
||||||
|
)
|
||||||
|
|
||||||
|
private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
|
||||||
|
private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
|
||||||
|
ServiceIdentifierPattern(
|
||||||
|
role = "",
|
||||||
|
service = "",
|
||||||
|
env = "",
|
||||||
|
zone = Seq(""))
|
||||||
|
)
|
||||||
|
|
||||||
|
override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
|
||||||
|
|
||||||
|
override type Key = gql.TimelineKey
|
||||||
|
override type View = gql.HomeTimelineView
|
||||||
|
override type Value = tr.Timeline
|
||||||
|
|
||||||
|
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
|
||||||
|
override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
|
||||||
|
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
|
||||||
|
|
||||||
|
private def createHomeMixerRequestArrow(
|
||||||
|
compositeOnUserClientColumn: CompositeOnUserClientColumn,
|
||||||
|
auditIpClientColumn: AuditIpClientColumn
|
||||||
|
): Arrow[(Key, View), hm.HomeMixerRequest] = {
|
||||||
|
|
||||||
|
val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
|
||||||
|
val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
|
||||||
|
(gd.LookupContext(), Set(gd.QueryFields.Roles))
|
||||||
|
|
||||||
|
val populateUserRoles = Arrow
|
||||||
|
.flatMap[(Key, View), Option[Set[String]]] { _ =>
|
||||||
|
Stitch.collect {
|
||||||
|
CallContext.twitterUserId.map { userId =>
|
||||||
|
compositeOnUserClientColumn.fetcher
|
||||||
|
.callStack(HomeMixerColumn.FetchCallstack)
|
||||||
|
.fetch(userId, gizmoduckView).map(_.v)
|
||||||
|
.map {
|
||||||
|
_.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val populateIpAddress = Arrow
|
||||||
|
.flatMap[(Key, View), Option[String]](_ =>
|
||||||
|
auditIpClientColumn.fetcher
|
||||||
|
.callStack(HomeMixerColumn.FetchCallstack)
|
||||||
|
.fetch((), ()).map(_.v))
|
||||||
|
|
||||||
|
Arrow.join(
|
||||||
|
populateUserRoles,
|
||||||
|
populateIpAddress
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
Arrow.zipWithArg(populateUserRolesAndIp).map {
|
||||||
|
case ((key, view), (roles, ipAddress)) =>
|
||||||
|
val deviceContextOpt = Some(
|
||||||
|
hm.DeviceContext(
|
||||||
|
isPolling = CallContext.isPolling,
|
||||||
|
requestContext = view.requestContext,
|
||||||
|
latestControlAvailable = view.latestControlAvailable,
|
||||||
|
autoplayEnabled = view.autoplayEnabled
|
||||||
|
))
|
||||||
|
val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
|
||||||
|
|
||||||
|
val (product, productContext) = key match {
|
||||||
|
case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
|
||||||
|
(
|
||||||
|
hm.Product.ForYou,
|
||||||
|
hm.ProductContext.ForYou(
|
||||||
|
hm.ForYou(
|
||||||
|
deviceContextOpt,
|
||||||
|
seenTweetIds,
|
||||||
|
view.dspClientContext,
|
||||||
|
view.pushToHomeTweetId
|
||||||
|
)
|
||||||
|
))
|
||||||
|
case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
|
||||||
|
(
|
||||||
|
hm.Product.Following,
|
||||||
|
hm.ProductContext.Following(
|
||||||
|
hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
|
||||||
|
case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
|
||||||
|
(
|
||||||
|
hm.Product.Subscribed,
|
||||||
|
hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
|
||||||
|
case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
|
||||||
|
}
|
||||||
|
|
||||||
|
val clientContext = pm.ClientContext(
|
||||||
|
userId = CallContext.twitterUserId,
|
||||||
|
guestId = CallContext.guestId,
|
||||||
|
guestIdAds = CallContext.guestIdAds,
|
||||||
|
guestIdMarketing = CallContext.guestIdMarketing,
|
||||||
|
appId = CallContext.clientApplicationId,
|
||||||
|
ipAddress = ipAddress,
|
||||||
|
userAgent = CallContext.userAgent,
|
||||||
|
countryCode = CallContext.requestCountryCode,
|
||||||
|
languageCode = CallContext.requestLanguageCode,
|
||||||
|
isTwoffice = CallContext.isInternalOrTwoffice,
|
||||||
|
userRoles = roles,
|
||||||
|
deviceId = CallContext.deviceId,
|
||||||
|
mobileDeviceId = CallContext.mobileDeviceId,
|
||||||
|
mobileDeviceAdId = CallContext.adId,
|
||||||
|
limitAdTracking = CallContext.limitAdTracking
|
||||||
|
)
|
||||||
|
|
||||||
|
hm.HomeMixerRequest(
|
||||||
|
clientContext = clientContext,
|
||||||
|
product = product,
|
||||||
|
productContext = Some(productContext),
|
||||||
|
maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
|
||||||
|
cursor = view.cursor.filter(_.nonEmpty)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override val fetch: Arrow[(Key, View), Result[Value]] = {
|
||||||
|
val transformThriftIntoPipelineRequest: Arrow[
|
||||||
|
(Key, View),
|
||||||
|
ProductPipelineRequest[HomeMixerRequest]
|
||||||
|
] = {
|
||||||
|
Arrow
|
||||||
|
.identity[(Key, View)]
|
||||||
|
.andThen {
|
||||||
|
createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
|
||||||
|
}
|
||||||
|
.map {
|
||||||
|
case thriftRequest =>
|
||||||
|
val request = homeMixerRequestUnmarshaller(thriftRequest)
|
||||||
|
val params = paramsBuilder.build(
|
||||||
|
clientContext = request.clientContext,
|
||||||
|
product = request.product,
|
||||||
|
featureOverrides =
|
||||||
|
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
|
||||||
|
)
|
||||||
|
ProductPipelineRequest(request, params)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val underlyingProduct: Arrow[
|
||||||
|
ProductPipelineRequest[HomeMixerRequest],
|
||||||
|
ProductPipelineResult[tr.TimelineResponse]
|
||||||
|
] = Arrow
|
||||||
|
.identity[ProductPipelineRequest[HomeMixerRequest]]
|
||||||
|
.map { pipelineRequest =>
|
||||||
|
val pipelineArrow = productPipelineRegistry
|
||||||
|
.getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
|
||||||
|
pipelineRequest.request.product)
|
||||||
|
.arrow
|
||||||
|
(pipelineArrow, pipelineRequest)
|
||||||
|
}.applyArrow
|
||||||
|
|
||||||
|
transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
|
||||||
|
_.result match {
|
||||||
|
case Some(result) => found(result.timeline)
|
||||||
|
case _ => missing
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
object HomeMixerColumn {
|
||||||
|
val Path = "home-mixer/homeMixer.Timeline"
|
||||||
|
private val FetchCallstack = s"$Path:fetch"
|
||||||
|
private val MaxCount: Option[Int] = Some(100)
|
||||||
|
}
|
@ -10,11 +10,8 @@ scala_library(
|
|||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||||
"src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
|
|
||||||
"src/thrift/com/twitter/search:earlybird-scala",
|
"src/thrift/com/twitter/search:earlybird-scala",
|
||||||
"stitch/stitch-timelineservice/src/main/scala",
|
"stitch/stitch-timelineservice/src/main/scala",
|
||||||
"strato/config/columns/recommendations/similarity:similarity-strato-client",
|
|
||||||
"strato/src/main/scala/com/twitter/strato/client",
|
|
||||||
],
|
],
|
||||||
exports = [
|
exports = [
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
|
||||||
|
@ -6,13 +6,11 @@ scala_library(
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"finagle/finagle-core/src/main",
|
"finagle/finagle-core/src/main",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"joinkey/src/main/scala/com/twitter/joinkey/context",
|
|
||||||
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
|
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
|
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
|
||||||
@ -25,8 +23,6 @@ scala_library(
|
|||||||
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
|
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
|
||||||
"stringcenter/client",
|
"stringcenter/client",
|
||||||
"stringcenter/client/src/main/java",
|
"stringcenter/client/src/main/java",
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
|
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
|
|
||||||
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -1,6 +1,8 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator
|
||||||
|
|
||||||
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
|
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
|
||||||
|
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
|
||||||
|
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
|
||||||
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
|
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
|
||||||
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator
|
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator
|
||||||
|
@ -8,7 +8,9 @@ scala_library(
|
|||||||
"finagle/finagle-core/src/main",
|
"finagle/finagle-core/src/main",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
|
"joinkey/src/main/scala/com/twitter/joinkey/context",
|
||||||
|
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
|
||||||
@ -18,6 +20,7 @@ scala_library(
|
|||||||
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
|
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
|
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
|
||||||
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.builder
|
||||||
|
|
||||||
import com.twitter.finagle.tracing.Trace
|
import com.twitter.finagle.tracing.Trace
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.builder
|
||||||
|
|
||||||
import com.twitter.bijection.Base64String
|
import com.twitter.bijection.Base64String
|
||||||
import com.twitter.bijection.scrooge.BinaryScalaCodec
|
import com.twitter.bijection.scrooge.BinaryScalaCodec
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient
|
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient
|
@ -1,8 +1,10 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.builder
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.home_mixer.model.HomeFeatures._
|
import com.twitter.home_mixer.model.HomeFeatures._
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
|
||||||
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
|
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
|
||||||
import com.twitter.tweetypie.{thriftscala => tpt}
|
import com.twitter.tweetypie.{thriftscala => tpt}
|
||||||
|
|
||||||
@ -25,74 +27,76 @@ object HomeTweetTypePredicates {
|
|||||||
(
|
(
|
||||||
"has_exclusive_conversation_author_id",
|
"has_exclusive_conversation_author_id",
|
||||||
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
|
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
|
||||||
("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)),
|
("is_eligible_for_connect_boost", _ => false),
|
||||||
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
|
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
|
||||||
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
|
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
|
||||||
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
|
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
|
||||||
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
|
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
|
||||||
(
|
|
||||||
"is_self_thread_tweet",
|
|
||||||
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
|
|
||||||
("get_initial", _.getOrElse(GetInitialFeature, false)),
|
("get_initial", _.getOrElse(GetInitialFeature, false)),
|
||||||
("get_newer", _.getOrElse(GetNewerFeature, false)),
|
("get_newer", _.getOrElse(GetNewerFeature, false)),
|
||||||
("get_middle", _.getOrElse(GetMiddleFeature, false)),
|
("get_middle", _.getOrElse(GetMiddleFeature, false)),
|
||||||
("get_older", _.getOrElse(GetOlderFeature, false)),
|
("get_older", _.getOrElse(GetOlderFeature, false)),
|
||||||
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
|
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
|
||||||
("polling", _.getOrElse(PollingFeature, false)),
|
("polling", _.getOrElse(PollingFeature, false)),
|
||||||
("tls_size_20_plus", _ => false),
|
("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)),
|
||||||
("near_empty", _ => false),
|
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
|
||||||
("ranked_request", _ => false),
|
|
||||||
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
|
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
|
||||||
|
(
|
||||||
|
"less_than_10_mins_since_lnpt",
|
||||||
|
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
|
||||||
|
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
|
||||||
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
|
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
|
||||||
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
|
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
|
||||||
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
|
|
||||||
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
|
|
||||||
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
|
|
||||||
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
|
|
||||||
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
|
|
||||||
(
|
(
|
||||||
"is_signup_request",
|
"conversation_module_has_2_displayed_tweets",
|
||||||
candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
|
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
|
||||||
("empty_request", _ => false),
|
("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)),
|
||||||
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
|
|
||||||
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
|
|
||||||
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
|
|
||||||
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
|
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
|
||||||
(
|
(
|
||||||
"served_size_between_50_and_100",
|
"served_size_between_50_and_100",
|
||||||
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
|
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
|
||||||
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
|
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
|
||||||
|
(
|
||||||
|
"is_self_thread_tweet",
|
||||||
|
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
|
||||||
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
|
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
|
||||||
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
|
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
|
||||||
|
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
|
||||||
|
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
|
||||||
|
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
|
||||||
(
|
(
|
||||||
"account_age_less_than_30_minutes",
|
"account_age_less_than_30_minutes",
|
||||||
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
|
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
|
||||||
|
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
|
||||||
|
(
|
||||||
|
"directed_at_user_is_in_first_degree",
|
||||||
|
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
|
||||||
|
(
|
||||||
|
"has_semantic_core_annotation",
|
||||||
|
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
|
||||||
|
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
|
||||||
(
|
(
|
||||||
"account_age_less_than_1_day",
|
"account_age_less_than_1_day",
|
||||||
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
|
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
|
||||||
(
|
(
|
||||||
"account_age_less_than_7_days",
|
"account_age_less_than_7_days",
|
||||||
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
|
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
|
||||||
(
|
|
||||||
"directed_at_user_is_in_first_degree",
|
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
|
|
||||||
("root_user_is_in_first_degree", _ => false),
|
|
||||||
(
|
|
||||||
"has_semantic_core_annotation",
|
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
|
|
||||||
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
|
|
||||||
(
|
(
|
||||||
"part_of_utt",
|
"part_of_utt",
|
||||||
_.getOrElse(EarlybirdFeature, None)
|
_.getOrElse(EarlybirdFeature, None)
|
||||||
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
|
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
|
||||||
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
|
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
|
||||||
|
(
|
||||||
|
"has_home_latest_request_past_week",
|
||||||
|
_.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
|
||||||
|
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
|
||||||
|
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
|
||||||
|
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
|
||||||
|
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
|
||||||
|
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
|
||||||
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
|
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
|
||||||
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
|
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
|
||||||
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
|
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
|
||||||
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
|
|
||||||
("viewer_is_employee", _ => false),
|
|
||||||
("viewer_is_timelines_employee", _ => false),
|
|
||||||
("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
|
|
||||||
(
|
(
|
||||||
"has_ancestor_authored_by_viewer",
|
"has_ancestor_authored_by_viewer",
|
||||||
candidate =>
|
candidate =>
|
||||||
@ -100,11 +104,6 @@ object HomeTweetTypePredicates {
|
|||||||
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
|
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
|
||||||
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
|
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
|
||||||
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
|
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
|
||||||
(
|
|
||||||
"root_ancestor",
|
|
||||||
candidate =>
|
|
||||||
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
|
|
||||||
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
|
|
||||||
(
|
(
|
||||||
"deep_reply",
|
"deep_reply",
|
||||||
candidate =>
|
candidate =>
|
||||||
@ -119,23 +118,22 @@ object HomeTweetTypePredicates {
|
|||||||
"tweet_age_less_than_15_seconds",
|
"tweet_age_less_than_15_seconds",
|
||||||
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
.exists(_.untilNow <= 15.seconds)),
|
.exists(_.untilNow <= 15.seconds)),
|
||||||
("is_followed_topic_tweet", _ => false),
|
(
|
||||||
("is_recommended_topic_tweet", _ => false),
|
"less_than_1_hour_since_lnpt",
|
||||||
("is_topic_tweet", _ => false),
|
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)),
|
||||||
("preferred_language_matches_tweet_language", _ => false),
|
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
|
||||||
(
|
(
|
||||||
"device_language_matches_tweet_language",
|
"device_language_matches_tweet_language",
|
||||||
candidate =>
|
candidate =>
|
||||||
candidate.getOrElse(TweetLanguageFeature, None) ==
|
candidate.getOrElse(TweetLanguageFeature, None) ==
|
||||||
candidate.getOrElse(DeviceLanguageFeature, None)),
|
candidate.getOrElse(DeviceLanguageFeature, None)),
|
||||||
|
(
|
||||||
|
"root_ancestor",
|
||||||
|
candidate =>
|
||||||
|
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
|
||||||
|
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
|
||||||
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
|
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
|
||||||
("in_network", _.getOrElse(FromInNetworkSourceFeature, true)),
|
("in_network", _.getOrElse(InNetworkFeature, true)),
|
||||||
("viewer_follows_original_author", _ => false),
|
|
||||||
("has_account_follow_prompt", _ => false),
|
|
||||||
("has_relevance_prompt", _ => false),
|
|
||||||
("has_topic_annotation_haug_prompt", _ => false),
|
|
||||||
("has_topic_annotation_random_precision_prompt", _ => false),
|
|
||||||
("has_topic_annotation_prompt", _ => false),
|
|
||||||
(
|
(
|
||||||
"has_political_annotation",
|
"has_political_annotation",
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(
|
_.getOrElse(EarlybirdFeature, None).exists(
|
||||||
@ -153,9 +151,14 @@ object HomeTweetTypePredicates {
|
|||||||
_.getOrElse(EarlybirdFeature, None)
|
_.getOrElse(EarlybirdFeature, None)
|
||||||
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
|
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
|
||||||
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
|
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
|
||||||
("is_viewer_not_invited_to_reply", _ => false),
|
(
|
||||||
("is_viewer_invited_to_reply", _ => false),
|
"is_followed_topic_tweet",
|
||||||
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
|
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
|
||||||
|
.exists(_ == BasicTopicContextFunctionalityType)),
|
||||||
|
(
|
||||||
|
"is_recommended_topic_tweet",
|
||||||
|
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
|
||||||
|
.exists(_ == RecommendationTopicContextFunctionalityType)),
|
||||||
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
|
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
|
||||||
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
|
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
|
||||||
(
|
(
|
||||||
@ -164,8 +167,6 @@ object HomeTweetTypePredicates {
|
|||||||
(
|
(
|
||||||
"has_gte_100k_favs",
|
"has_gte_100k_favs",
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
|
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
|
||||||
("above_neighbor_is_topic_tweet", _ => false),
|
|
||||||
("is_topic_tweet_with_neighbor_below", _ => false),
|
|
||||||
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
|
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
|
||||||
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
|
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
|
||||||
(
|
(
|
||||||
@ -187,6 +188,7 @@ object HomeTweetTypePredicates {
|
|||||||
(
|
(
|
||||||
"has_toxicity_score_above_threshold",
|
"has_toxicity_score_above_threshold",
|
||||||
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
|
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
|
||||||
|
("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
|
||||||
(
|
(
|
||||||
"text_only",
|
"text_only",
|
||||||
candidate =>
|
candidate =>
|
||||||
@ -204,23 +206,50 @@ object HomeTweetTypePredicates {
|
|||||||
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
|
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
|
||||||
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
|
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
|
||||||
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
|
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
|
||||||
("3_or_more_consecutive_not_in_network", _ => false),
|
|
||||||
("2_or_more_consecutive_not_in_network", _ => false),
|
|
||||||
("5_out_of_7_not_in_network", _ => false),
|
|
||||||
("7_out_of_7_not_in_network", _ => false),
|
|
||||||
("5_out_of_5_not_in_network", _ => false),
|
|
||||||
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
|
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
|
||||||
("has_liked_by_social_context", _ => false),
|
|
||||||
("has_followed_by_social_context", _ => false),
|
|
||||||
("has_topic_social_context", _ => false),
|
|
||||||
("timeline_entry_has_banner", _ => false),
|
|
||||||
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
|
|
||||||
(
|
(
|
||||||
"conversation_module_has_2_displayed_tweets",
|
"has_liked_by_social_context",
|
||||||
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
|
candidateFeatures =>
|
||||||
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
|
candidateFeatures
|
||||||
("served_in_recap_tweet_candidate_module_injection", _ => false),
|
.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
|
||||||
("served_in_threaded_conversation_module", _ => false)
|
.exists(candidateFeatures
|
||||||
|
.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
|
||||||
|
(
|
||||||
|
"has_followed_by_social_context",
|
||||||
|
_.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
|
||||||
|
(
|
||||||
|
"has_topic_social_context",
|
||||||
|
candidateFeatures =>
|
||||||
|
candidateFeatures
|
||||||
|
.getOrElse(TopicIdSocialContextFeature, None)
|
||||||
|
.isDefined &&
|
||||||
|
candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
|
||||||
|
("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
|
||||||
|
(
|
||||||
|
"video_bt_10_60_sec",
|
||||||
|
_.getOrElse(VideoDurationMsFeature, None).exists(duration =>
|
||||||
|
duration > 10000 && duration <= 60000)),
|
||||||
|
("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
|
||||||
|
(
|
||||||
|
"tweet_age_lte_30_minutes",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow <= 30.minutes)),
|
||||||
|
(
|
||||||
|
"tweet_age_lte_1_hour",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow <= 1.hour)),
|
||||||
|
(
|
||||||
|
"tweet_age_lte_6_hours",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow <= 6.hours)),
|
||||||
|
(
|
||||||
|
"tweet_age_lte_12_hours",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow <= 12.hours)),
|
||||||
|
(
|
||||||
|
"tweet_age_gte_24_hours",
|
||||||
|
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
|
||||||
|
.exists(_.untilNow >= 24.hours)),
|
||||||
)
|
)
|
||||||
|
|
||||||
val PredicateMap = CandidatePredicates.toMap
|
val PredicateMap = CandidatePredicates.toMap
|
@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
|
|||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||||
|
|
||||||
object ListClientEventDetailsBuilder
|
case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
|
||||||
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
|
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
|
||||||
|
|
||||||
override def apply(
|
override def apply(
|
||||||
@ -20,7 +20,7 @@ object ListClientEventDetailsBuilder
|
|||||||
conversationDetails = None,
|
conversationDetails = None,
|
||||||
timelinesDetails = Some(
|
timelinesDetails = Some(
|
||||||
TimelinesDetails(
|
TimelinesDetails(
|
||||||
injectionType = Some(st.SuggestType.OrganicListTweet.name),
|
injectionType = Some(suggestType.name),
|
||||||
controllerData = None,
|
controllerData = None,
|
||||||
sourceData = None)),
|
sourceData = None)),
|
||||||
articleDetails = None,
|
articleDetails = None,
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
@ -9,7 +9,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
|
|||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
import com.twitter.timelines.service.{thriftscala => t}
|
import com.twitter.timelines.service.{thriftscala => t}
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -4,9 +4,16 @@ scala_library(
|
|||||||
strict_deps = True,
|
strict_deps = True,
|
||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
|
||||||
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
|
||||||
"src/thrift/com/twitter/timelines/service:thrift-scala",
|
"src/thrift/com/twitter/timelines/service:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||||
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||||
@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
@ -17,7 +17,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
|
|||||||
import com.twitter.timelineservice.model.FeedbackInfo
|
import com.twitter.timelineservice.model.FeedbackInfo
|
||||||
import com.twitter.timelineservice.model.FeedbackMetadata
|
import com.twitter.timelineservice.model.FeedbackMetadata
|
||||||
import com.twitter.timelineservice.{thriftscala => tls}
|
import com.twitter.timelineservice.{thriftscala => tls}
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
|
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature
|
@ -0,0 +1,18 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
|
import com.twitter.stringcenter.client.ExternalStringRegistry
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Provider
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
class FeedbackStrings @Inject() (
|
||||||
|
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
|
||||||
|
private val externalStringRegistry = externalStringRegistryProvider.get()
|
||||||
|
|
||||||
|
val seeLessOftenFeedbackString =
|
||||||
|
externalStringRegistry.createProdString("Feedback.seeLessOften")
|
||||||
|
val seeLessOftenConfirmationFeedbackString =
|
||||||
|
externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
|
||||||
|
}
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
|
|||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.timelines.service.{thriftscala => t}
|
import com.twitter.timelines.service.{thriftscala => t}
|
||||||
import com.twitter.timelines.util.FeedbackMetadataSerializer
|
import com.twitter.timelines.util.FeedbackMetadataSerializer
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
|
||||||
@ -14,10 +14,13 @@ import javax.inject.Singleton
|
|||||||
@Singleton
|
@Singleton
|
||||||
case class HomeTweetSocialContextBuilder @Inject() (
|
case class HomeTweetSocialContextBuilder @Inject() (
|
||||||
likedBySocialContextBuilder: LikedBySocialContextBuilder,
|
likedBySocialContextBuilder: LikedBySocialContextBuilder,
|
||||||
|
listsSocialContextBuilder: ListsSocialContextBuilder,
|
||||||
followedBySocialContextBuilder: FollowedBySocialContextBuilder,
|
followedBySocialContextBuilder: FollowedBySocialContextBuilder,
|
||||||
topicSocialContextBuilder: TopicSocialContextBuilder,
|
topicSocialContextBuilder: TopicSocialContextBuilder,
|
||||||
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
|
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
|
||||||
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
|
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder,
|
||||||
|
popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
|
||||||
|
popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
|
||||||
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
def apply(
|
def apply(
|
||||||
@ -31,6 +34,9 @@ case class HomeTweetSocialContextBuilder @Inject() (
|
|||||||
likedBySocialContextBuilder(query, candidate, features)
|
likedBySocialContextBuilder(query, candidate, features)
|
||||||
.orElse(followedBySocialContextBuilder(query, candidate, features))
|
.orElse(followedBySocialContextBuilder(query, candidate, features))
|
||||||
.orElse(topicSocialContextBuilder(query, candidate, features))
|
.orElse(topicSocialContextBuilder(query, candidate, features))
|
||||||
|
.orElse(popularVideoSocialContextBuilder(query, candidate, features))
|
||||||
|
.orElse(listsSocialContextBuilder(query, candidate, features))
|
||||||
|
.orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
|
||||||
case Some(_) =>
|
case Some(_) =>
|
||||||
val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
|
val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
|
||||||
// Only hydrate the social context into the root tweet in a conversation module
|
// Only hydrate the social context into the root tweet in a conversation module
|
@ -5,7 +5,6 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
|
|||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.ExternalStringRegistry
|
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Provider
|
import javax.inject.Provider
|
||||||
@ -32,12 +31,13 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
|
|||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
|
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
|
||||||
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
|
feedbackStrings: FeedbackStrings,
|
||||||
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
|
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
|
||||||
|
|
||||||
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
|
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
|
||||||
externalStringRegistry = externalStringRegistryProvider.get(),
|
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
|
||||||
|
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
|
||||||
stringCenter = stringCenterProvider.get(),
|
stringCenter = stringCenterProvider.get(),
|
||||||
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
|
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
|
||||||
)
|
)
|
||||||
|
@ -0,0 +1,52 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
|
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
|
||||||
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Provider
|
||||||
|
import javax.inject.Singleton
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.timelines.service.{thriftscala => tl}
|
||||||
|
import com.twitter.timelines.util.FeedbackRequestSerializer
|
||||||
|
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
|
||||||
|
import com.twitter.timelineservice.thriftscala.FeedbackType
|
||||||
|
|
||||||
|
object HomeWhoToSubscribeFeedbackActionInfoBuilder {
|
||||||
|
private val FeedbackMetadata = tl.FeedbackMetadata(
|
||||||
|
injectionType = Some(SuggestType.WhoToSubscribe),
|
||||||
|
engagementType = None,
|
||||||
|
entityIds = Seq.empty,
|
||||||
|
ttlMs = None
|
||||||
|
)
|
||||||
|
private val FeedbackRequest =
|
||||||
|
tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
|
||||||
|
private val EncodedFeedbackRequest =
|
||||||
|
FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
|
||||||
|
feedbackStrings: FeedbackStrings,
|
||||||
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
|
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
|
||||||
|
|
||||||
|
private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
|
||||||
|
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
|
||||||
|
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
|
||||||
|
stringCenter = stringCenterProvider.get(),
|
||||||
|
encodedFeedbackRequest =
|
||||||
|
Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
|
||||||
|
)
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidate: UserCandidate,
|
||||||
|
candidateFeatures: FeatureMap
|
||||||
|
): Option[FeedbackActionInfo] =
|
||||||
|
whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
|
||||||
|
}
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature
|
@ -0,0 +1,50 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
|
||||||
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
import com.twitter.timelineservice.suggests.{thriftscala => t}
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Provider
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
/**
|
||||||
|
* "Your Lists" will be rendered for the context and a url link for your lists.
|
||||||
|
*/
|
||||||
|
@Singleton
|
||||||
|
case class ListsSocialContextBuilder @Inject() (
|
||||||
|
externalStrings: HomeMixerExternalStrings,
|
||||||
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
|
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
|
private val stringCenter = stringCenterProvider.get()
|
||||||
|
private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
|
||||||
|
|
||||||
|
def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidate: TweetCandidate,
|
||||||
|
candidateFeatures: FeatureMap
|
||||||
|
): Option[SocialContext] = {
|
||||||
|
candidateFeatures.get(SuggestTypeFeature) match {
|
||||||
|
case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
|
||||||
|
val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
|
||||||
|
Some(
|
||||||
|
GeneralContext(
|
||||||
|
contextType = ListGeneralContextType,
|
||||||
|
text = stringCenter.prepare(listString),
|
||||||
|
url = userName.map(name => ""),
|
||||||
|
contextImageUrls = None,
|
||||||
|
landingUrl = None
|
||||||
|
))
|
||||||
|
case _ => None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||||
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
|
||||||
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
@ -12,7 +12,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
|
|||||||
import com.twitter.timelineservice.model.FeedbackInfo
|
import com.twitter.timelineservice.model.FeedbackInfo
|
||||||
import com.twitter.timelineservice.model.FeedbackMetadata
|
import com.twitter.timelineservice.model.FeedbackMetadata
|
||||||
import com.twitter.timelineservice.{thriftscala => tlst}
|
import com.twitter.timelineservice.{thriftscala => tlst}
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -0,0 +1,43 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Provider
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
case class PopularInYourAreaSocialContextBuilder @Inject() (
|
||||||
|
externalStrings: HomeMixerExternalStrings,
|
||||||
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
|
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
|
private val stringCenter = stringCenterProvider.get()
|
||||||
|
private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
|
||||||
|
|
||||||
|
def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidate: TweetCandidate,
|
||||||
|
candidateFeatures: FeatureMap
|
||||||
|
): Option[SocialContext] = {
|
||||||
|
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
|
||||||
|
if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
|
||||||
|
Some(
|
||||||
|
GeneralContext(
|
||||||
|
contextType = LocationGeneralContextType,
|
||||||
|
text = stringCenter.prepare(popularInYourAreaString),
|
||||||
|
url = None,
|
||||||
|
contextImageUrls = None,
|
||||||
|
landingUrl = None
|
||||||
|
))
|
||||||
|
} else None
|
||||||
|
}
|
||||||
|
}
|
@ -1,50 +1,48 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
|
||||||
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
|
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Provider
|
import javax.inject.Provider
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
/**
|
|
||||||
* Renders a fixed 'You Might Like' string above all OON Tweets.
|
|
||||||
*/
|
|
||||||
@Singleton
|
@Singleton
|
||||||
case class YouMightLikeSocialContextBuilder @Inject() (
|
case class PopularVideoSocialContextBuilder @Inject() (
|
||||||
externalStrings: HomeMixerExternalStrings,
|
externalStrings: HomeMixerExternalStrings,
|
||||||
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
@ProductScoped stringCenterProvider: Provider[StringCenter])
|
||||||
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
private val stringCenter = stringCenterProvider.get()
|
private val stringCenter = stringCenterProvider.get()
|
||||||
private val youMightLikeString = externalStrings.socialContextYouMightLikeString
|
private val popularVideoString = externalStrings.socialContextPopularVideoString
|
||||||
|
|
||||||
def apply(
|
def apply(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
candidate: TweetCandidate,
|
candidate: TweetCandidate,
|
||||||
candidateFeatures: FeatureMap
|
candidateFeatures: FeatureMap
|
||||||
): Option[SocialContext] = {
|
): Option[SocialContext] = {
|
||||||
val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true)
|
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
|
||||||
val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false)
|
if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) {
|
||||||
if (!isInNetwork && !isRetweet) {
|
|
||||||
Some(
|
Some(
|
||||||
GeneralContext(
|
GeneralContext(
|
||||||
contextType = SparkleGeneralContextType,
|
contextType = SparkleGeneralContextType,
|
||||||
text = stringCenter.prepare(youMightLikeString),
|
text = stringCenter.prepare(popularVideoString),
|
||||||
url = None,
|
url = None,
|
||||||
contextImageUrls = None,
|
contextImageUrls = None,
|
||||||
landingUrl = None
|
landingUrl = Some(
|
||||||
|
Url(
|
||||||
|
urlType = DeepLink,
|
||||||
|
url = ""
|
||||||
|
)
|
||||||
|
)
|
||||||
))
|
))
|
||||||
} else {
|
} else None
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
@ -8,7 +8,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||||
@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
|
|||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
import com.twitter.timelines.service.{thriftscala => t}
|
import com.twitter.timelines.service.{thriftscala => t}
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
|
@ -1,4 +1,4 @@
|
|||||||
package com.twitter.home_mixer.functional_component.decorator
|
package com.twitter.home_mixer.functional_component.decorator.urt.builder
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
|
||||||
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
|
||||||
import com.twitter.stringcenter.client.StringCenter
|
import com.twitter.stringcenter.client.StringCenter
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
@ -4,95 +4,56 @@ scala_library(
|
|||||||
strict_deps = True,
|
strict_deps = True,
|
||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
|
||||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||||
"configapi/configapi-decider",
|
"configapi/configapi-decider",
|
||||||
"finatra/inject/inject-core/src/main/scala",
|
"finatra/inject/inject-core/src/main/scala",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
|
|
||||||
"joinkey/src/main/scala/com/twitter/joinkey/context",
|
"joinkey/src/main/scala/com/twitter/joinkey/context",
|
||||||
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
|
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
|
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
|
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
|
||||||
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
|
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
|
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
||||||
"representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
|
|
||||||
"representation-scorer/server/src/main/thrift:thrift-scala",
|
|
||||||
"servo/repo/src/main/scala",
|
|
||||||
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
||||||
"src/java/com/twitter/ml/api/constant",
|
|
||||||
"src/java/com/twitter/search/common/util/lang",
|
"src/java/com/twitter/search/common/util/lang",
|
||||||
"src/scala/com/twitter/ml/api/util",
|
"src/scala/com/twitter/timelines/prediction/adapters/request_context",
|
||||||
"src/scala/com/twitter/timelines/prediction/adapters/real_graph",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/adapters/twistly",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/common/util",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/features/common",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/features/recap",
|
|
||||||
"src/scala/com/twitter/timelines/prediction/features/time_features",
|
|
||||||
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||||
"src/thrift/com/twitter/ml/api:data-java",
|
|
||||||
"src/thrift/com/twitter/ml/api:embedding-java",
|
|
||||||
"src/thrift/com/twitter/onboarding/relevance/features:features-java",
|
|
||||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||||
"src/thrift/com/twitter/search:earlybird-scala",
|
"src/thrift/com/twitter/search:earlybird-scala",
|
||||||
"src/thrift/com/twitter/search/common:constants-java",
|
"src/thrift/com/twitter/search/common:constants-java",
|
||||||
"src/thrift/com/twitter/socialgraph:thrift-scala",
|
"src/thrift/com/twitter/socialgraph:thrift-scala",
|
||||||
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
|
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
|
||||||
"src/thrift/com/twitter/timelineranker:thrift-scala",
|
"src/thrift/com/twitter/timelineranker:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelines/author_features:thrift-java",
|
|
||||||
"src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
|
|
||||||
"src/thrift/com/twitter/timelines/impression:thrift-scala",
|
"src/thrift/com/twitter/timelines/impression:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
|
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
|
||||||
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
|
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
|
||||||
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
|
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
|
||||||
"src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
|
|
||||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||||
"src/thrift/com/twitter/user_session_store:thrift-java",
|
"src/thrift/com/twitter/user_session_store:thrift-java",
|
||||||
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
|
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
|
||||||
"src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
|
|
||||||
"stitch/stitch-core",
|
"stitch/stitch-core",
|
||||||
"stitch/stitch-gizmoduck",
|
"stitch/stitch-gizmoduck",
|
||||||
"stitch/stitch-socialgraph",
|
"stitch/stitch-socialgraph",
|
||||||
"stitch/stitch-timelineservice",
|
"stitch/stitch-timelineservice",
|
||||||
"stitch/stitch-tweetypie",
|
"stitch/stitch-tweetypie",
|
||||||
"strato/config/columns/topic-signals/tsp",
|
|
||||||
"strato/config/columns/topic-signals/tsp:tsp-strato-client",
|
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
|
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
|
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
|
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly",
|
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
|
|
||||||
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
|
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
|
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
|
||||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
||||||
"topic-social-proof/server/src/main/thrift:thrift-scala",
|
|
||||||
"topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
|
|
||||||
"tweetconvosvc/thrift/src/main/thrift:thrift-scala",
|
|
||||||
"twitter-config/yaml",
|
|
||||||
"user_session_store/src/main/scala/com/twitter/user_session_store",
|
"user_session_store/src/main/scala/com/twitter/user_session_store",
|
||||||
"util/util-core",
|
"util/util-core",
|
||||||
],
|
],
|
||||||
|
@ -1,12 +1,10 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
package com.twitter.home_mixer.functional_component.feature_hydrator
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
|
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
import com.twitter.product_mixer.core.feature.Feature
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
@ -17,16 +15,12 @@ import javax.inject.Singleton
|
|||||||
@Singleton
|
@Singleton
|
||||||
case class FeedbackHistoryQueryFeatureHydrator @Inject() (
|
case class FeedbackHistoryQueryFeatureHydrator @Inject() (
|
||||||
feedbackHistoryClient: FeedbackHistoryManhattanClient)
|
feedbackHistoryClient: FeedbackHistoryManhattanClient)
|
||||||
extends QueryFeatureHydrator[PipelineQuery]
|
extends QueryFeatureHydrator[PipelineQuery] {
|
||||||
with Conditionally[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
|
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
|
||||||
|
|
||||||
override def onlyIf(query: PipelineQuery): Boolean =
|
|
||||||
query.params(EnableFeedbackFatigueParam)
|
|
||||||
|
|
||||||
override def hydrate(
|
override def hydrate(
|
||||||
query: PipelineQuery
|
query: PipelineQuery
|
||||||
): Stitch[FeatureMap] =
|
): Stitch[FeatureMap] =
|
||||||
|
@ -1,41 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
|
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
|
||||||
import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
case class FollowedTopicsQueryFeatureHydrator @Inject() (
|
|
||||||
followedTopicsCandidateSource: FollowedTopicsCandidateSource)
|
|
||||||
extends QueryFeatureHydrator[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
|
|
||||||
|
|
||||||
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
|
|
||||||
val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
|
|
||||||
followedTopicsCandidateSource(request)
|
|
||||||
.map { topics =>
|
|
||||||
FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
|
|
||||||
}.handle {
|
|
||||||
case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override val alerts = Seq(
|
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
|
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
|
|
||||||
)
|
|
||||||
}
|
|
@ -1,58 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.gizmoduck.{thriftscala => gt}
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
|
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.stitch.gizmoduck.Gizmoduck
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
|
|
||||||
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
|
|
||||||
with Conditionally[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier =
|
|
||||||
FeatureHydratorIdentifier("GizmoduckAuthorSafety")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
|
|
||||||
|
|
||||||
override def onlyIf(query: PipelineQuery): Boolean =
|
|
||||||
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
|
|
||||||
|
|
||||||
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidate: TweetCandidate,
|
|
||||||
existingFeatures: FeatureMap
|
|
||||||
): Stitch[FeatureMap] = {
|
|
||||||
val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
|
|
||||||
|
|
||||||
val blueVerifiedStitch = authorIdOption
|
|
||||||
.map { authorId =>
|
|
||||||
gizmoduck
|
|
||||||
.getUserById(
|
|
||||||
userId = authorId,
|
|
||||||
queryFields = queryFields
|
|
||||||
)
|
|
||||||
.map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
|
|
||||||
}.getOrElse(Stitch.False)
|
|
||||||
|
|
||||||
blueVerifiedStitch.map { isBlueVerified =>
|
|
||||||
FeatureMapBuilder()
|
|
||||||
.add(AuthorIsBlueVerifiedFeature, isBlueVerified)
|
|
||||||
.build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
|
|||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
|
||||||
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
||||||
|
import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
import com.twitter.product_mixer.core.feature.Feature
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
@ -11,7 +12,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
|
|||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
|
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
|
||||||
|
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
|
||||||
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
|
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
@ -19,32 +21,35 @@ import javax.inject.Singleton
|
|||||||
@Singleton
|
@Singleton
|
||||||
case class ImpressionBloomFilterQueryFeatureHydrator[
|
case class ImpressionBloomFilterQueryFeatureHydrator[
|
||||||
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
|
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
|
||||||
bloomFilter: ImpressionBloomFilter)
|
bloomFilterClient: ManhattanStoreClient[
|
||||||
extends QueryFeatureHydrator[Query] {
|
blm.ImpressionBloomFilterKey,
|
||||||
|
blm.ImpressionBloomFilterSeq
|
||||||
|
]) extends QueryFeatureHydrator[Query] {
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||||
"ImpressionBloomFilter")
|
"ImpressionBloomFilter")
|
||||||
|
|
||||||
private val ImpressionBloomFilterTTL = 7.day
|
private val ImpressionBloomFilterTTL = 7.day
|
||||||
private val ImpressionBloomFilterFalsePositiveRate = 0.002
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
|
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
|
||||||
|
|
||||||
private val SurfaceArea = t.SurfaceArea.HomeTimeline
|
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
|
||||||
|
|
||||||
override def hydrate(query: Query): Stitch[FeatureMap] = {
|
override def hydrate(query: Query): Stitch[FeatureMap] = {
|
||||||
val userId = query.getRequiredUserId
|
val userId = query.getRequiredUserId
|
||||||
bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq =>
|
bloomFilterClient
|
||||||
|
.get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
|
||||||
|
.map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
|
||||||
|
.map { bloomFilterSeq =>
|
||||||
val updatedBloomFilterSeq =
|
val updatedBloomFilterSeq =
|
||||||
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
|
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
|
||||||
else {
|
else {
|
||||||
bloomFilter.addElements(
|
ImpressionBloomFilter.addSeenTweetIds(
|
||||||
userId = userId,
|
|
||||||
surfaceArea = SurfaceArea,
|
surfaceArea = SurfaceArea,
|
||||||
tweetIds = query.seenTweetIds.get,
|
tweetIds = query.seenTweetIds.get,
|
||||||
bloomFilterEntrySeq = bloomFilterSeq,
|
bloomFilterSeq = bloomFilterSeq,
|
||||||
timeToLive = ImpressionBloomFilterTTL,
|
timeToLive = ImpressionBloomFilterTTL,
|
||||||
falsePositiveRate = ImpressionBloomFilterFalsePositiveRate
|
falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
|
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
|
||||||
|
@ -0,0 +1,41 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.feature_hydrator
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
|
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.feature.Feature
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||||
|
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
|
||||||
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
|
||||||
|
object InNetworkFeatureHydrator
|
||||||
|
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
|
||||||
|
|
||||||
|
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
|
): Stitch[Seq[FeatureMap]] = {
|
||||||
|
val viewerId = query.getRequiredUserId
|
||||||
|
val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
|
||||||
|
|
||||||
|
val featureMaps = candidates.map { candidate =>
|
||||||
|
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
|
||||||
|
val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
|
||||||
|
// Users cannot follow themselves but this is in network by definition
|
||||||
|
val isSelfTweet = authorId == viewerId
|
||||||
|
isSelfTweet || followedUserIds.contains(authorId)
|
||||||
|
}
|
||||||
|
FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
|
||||||
|
}
|
||||||
|
Stitch.value(featureMaps)
|
||||||
|
}
|
||||||
|
}
|
@ -2,8 +2,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
|
|||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
|
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
|
||||||
|
import com.twitter.finagle.stats.StatsReceiver
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
|
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
@ -27,17 +29,27 @@ import javax.inject.Singleton
|
|||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
case class PersistenceStoreQueryFeatureHydrator @Inject() (
|
case class PersistenceStoreQueryFeatureHydrator @Inject() (
|
||||||
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
|
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3],
|
||||||
|
statsReceiver: StatsReceiver)
|
||||||
extends QueryFeatureHydrator[PipelineQuery] {
|
extends QueryFeatureHydrator[PipelineQuery] {
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
|
||||||
|
|
||||||
|
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
|
||||||
|
private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
|
||||||
|
|
||||||
private val WhoToFollowExcludedUserIdsLimit = 1000
|
private val WhoToFollowExcludedUserIdsLimit = 1000
|
||||||
private val ServedTweetIdsDuration = 1.hour
|
private val ServedTweetIdsDuration = 10.minutes
|
||||||
private val ServedTweetIdsLimit = 100
|
private val ServedTweetIdsLimit = 100
|
||||||
|
private val ServedTweetPreviewIdsDuration = 10.hours
|
||||||
|
private val ServedTweetPreviewIdsLimit = 10
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] =
|
override val features: Set[Feature[_, _]] =
|
||||||
Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature)
|
Set(
|
||||||
|
ServedTweetIdsFeature,
|
||||||
|
ServedTweetPreviewIdsFeature,
|
||||||
|
PersistenceEntriesFeature,
|
||||||
|
WhoToFollowExcludedUserIdsFeature)
|
||||||
|
|
||||||
private val supportedClients = Seq(
|
private val supportedClients = Seq(
|
||||||
ClientPlatform.IPhone,
|
ClientPlatform.IPhone,
|
||||||
@ -80,8 +92,19 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
|
|||||||
.flatMap(
|
.flatMap(
|
||||||
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
|
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
|
||||||
|
|
||||||
|
servedTweetIdsSizeStat.add(servedTweetIds.size)
|
||||||
|
|
||||||
|
val servedTweetPreviewIds = timelineResponses
|
||||||
|
.filter(_.clientPlatform == clientPlatform)
|
||||||
|
.filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
|
||||||
|
.sortBy(-_.servedTime.inMilliseconds)
|
||||||
|
.flatMap(_.entries
|
||||||
|
.filter(_.entityIdType == EntityIdType.TweetPreview)
|
||||||
|
.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
|
||||||
|
|
||||||
FeatureMapBuilder()
|
FeatureMapBuilder()
|
||||||
.add(ServedTweetIdsFeature, servedTweetIds)
|
.add(ServedTweetIdsFeature, servedTweetIds)
|
||||||
|
.add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
|
||||||
.add(PersistenceEntriesFeature, timelineResponses)
|
.add(PersistenceEntriesFeature, timelineResponses)
|
||||||
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
|
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
|
||||||
.build()
|
.build()
|
||||||
|
@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
|
|||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.util.OffloadFuturePools
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.stitch.timelineservice.TimelineService
|
import com.twitter.stitch.timelineservice.TimelineService
|
||||||
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
|
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
|
||||||
@ -37,10 +38,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
|
|||||||
override def apply(
|
override def apply(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
|
||||||
val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
|
val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
|
||||||
candidate.features
|
candidate.features
|
||||||
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
|
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
|
||||||
.map(favoritedBy => favoritedBy -> candidate.candidate.id)
|
.map(favoritedBy => favoritedBy -> candidate.candidate.id)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -59,7 +60,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
|
|||||||
|
|
||||||
candidates.map { candidate =>
|
candidates.map { candidate =>
|
||||||
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
|
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
|
||||||
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
|
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
|
||||||
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
|
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
|
||||||
|
|
||||||
FeatureMapBuilder()
|
FeatureMapBuilder()
|
||||||
|
@ -32,11 +32,15 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
|
|||||||
val realGraphScoresFeatures = realGraphFollowedUsers
|
val realGraphScoresFeatures = realGraphFollowedUsers
|
||||||
.getOrElse(Seq.empty)
|
.getOrElse(Seq.empty)
|
||||||
.sortBy(-_.score)
|
.sortBy(-_.score)
|
||||||
.map(candidate => candidate.userId -> candidate.score)
|
.map(candidate => candidate.userId -> scaleScore(candidate.score))
|
||||||
.take(RealGraphCandidateCount)
|
.take(RealGraphCandidateCount)
|
||||||
.toMap
|
.toMap
|
||||||
|
|
||||||
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
|
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
|
||||||
|
private def scaleScore(score: Double): Double =
|
||||||
|
if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
|
||||||
}
|
}
|
||||||
|
@ -17,9 +17,13 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
|
||||||
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
|
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
|
||||||
|
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||||
import com.twitter.search.common.util.lang.ThriftLanguageUtil
|
import com.twitter.search.common.util.lang.ThriftLanguageUtil
|
||||||
import com.twitter.snowflake.id.SnowflakeId
|
import com.twitter.snowflake.id.SnowflakeId
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
|
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
|
||||||
|
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
|
||||||
import java.util.UUID
|
import java.util.UUID
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
@ -45,6 +49,9 @@ class RequestQueryFeatureHydrator[
|
|||||||
PullToRefreshFeature,
|
PullToRefreshFeature,
|
||||||
RequestJoinIdFeature,
|
RequestJoinIdFeature,
|
||||||
ServedRequestIdFeature,
|
ServedRequestIdFeature,
|
||||||
|
TimestampFeature,
|
||||||
|
TimestampGMTDowFeature,
|
||||||
|
TimestampGMTHourFeature,
|
||||||
ViewerIdFeature
|
ViewerIdFeature
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -67,6 +74,7 @@ class RequestQueryFeatureHydrator[
|
|||||||
override def hydrate(query: Query): Stitch[FeatureMap] = {
|
override def hydrate(query: Query): Stitch[FeatureMap] = {
|
||||||
val requestContext = query.deviceContext.flatMap(_.requestContextValue)
|
val requestContext = query.deviceContext.flatMap(_.requestContextValue)
|
||||||
val servedRequestId = UUID.randomUUID.getMostSignificantBits
|
val servedRequestId = UUID.randomUUID.getMostSignificantBits
|
||||||
|
val timestamp = query.queryTime.inMilliseconds
|
||||||
|
|
||||||
val featureMap = FeatureMapBuilder()
|
val featureMap = FeatureMapBuilder()
|
||||||
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
|
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
|
||||||
@ -97,8 +105,15 @@ class RequestQueryFeatureHydrator[
|
|||||||
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
|
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
|
||||||
.add(ServedRequestIdFeature, Some(servedRequestId))
|
.add(ServedRequestIdFeature, Some(servedRequestId))
|
||||||
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
|
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
|
||||||
|
.add(TimestampFeature, timestamp)
|
||||||
|
.add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
|
||||||
|
.add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
|
||||||
.add(HasDarkRequestFeature, hasDarkRequest)
|
.add(HasDarkRequestFeature, hasDarkRequest)
|
||||||
.add(ViewerIdFeature, query.getRequiredUserId)
|
.add(
|
||||||
|
ViewerIdFeature,
|
||||||
|
query.getOptionalUserId
|
||||||
|
.orElse(query.getGuestId).getOrElse(
|
||||||
|
throw PipelineFailure(BadRequest, "Missing viewer id")))
|
||||||
.build()
|
.build()
|
||||||
|
|
||||||
Stitch.value(featureMap)
|
Stitch.value(featureMap)
|
||||||
|
@ -1,46 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.socialgraph.{thriftscala => sg}
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
|
|
||||||
socialGraphStitchClient: SocialGraphStitchClient)
|
|
||||||
extends QueryFeatureHydrator[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier =
|
|
||||||
FeatureHydratorIdentifier("SGSFollowedUsers")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
|
|
||||||
|
|
||||||
private val SocialGraphLimit = 14999
|
|
||||||
|
|
||||||
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
|
|
||||||
val userId = query.getRequiredUserId
|
|
||||||
|
|
||||||
val request = sg.IdsRequest(
|
|
||||||
relationships = Seq(
|
|
||||||
sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
|
|
||||||
sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
|
|
||||||
),
|
|
||||||
pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
|
|
||||||
)
|
|
||||||
|
|
||||||
socialGraphStitchClient
|
|
||||||
.ids(request).map(_.ids)
|
|
||||||
.map { followedUsers =>
|
|
||||||
FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
|
|||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.product_mixer.core.util.OffloadFuturePools
|
||||||
import com.twitter.socialgraph.{thriftscala => sg}
|
import com.twitter.socialgraph.{thriftscala => sg}
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.stitch.socialgraph.SocialGraph
|
import com.twitter.stitch.socialgraph.SocialGraph
|
||||||
@ -41,8 +42,7 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
|
|||||||
override def apply(
|
override def apply(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
|
||||||
|
|
||||||
val allSocialContextUserIds =
|
val allSocialContextUserIds =
|
||||||
candidates.flatMap { candidate =>
|
candidates.flatMap { candidate =>
|
||||||
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++
|
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++
|
||||||
|
@ -1,67 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.socialgraph.{thriftscala => sg}
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
|
|
||||||
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier =
|
|
||||||
FeatureHydratorIdentifier("SocialGraphService")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
|
||||||
val viewerId = query.getRequiredUserId
|
|
||||||
|
|
||||||
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
|
|
||||||
val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
|
|
||||||
val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
|
|
||||||
|
|
||||||
val idsRequest = createIdsRequest(
|
|
||||||
userId = viewerId,
|
|
||||||
relationshipTypes = Set(sg.RelationshipType.Following),
|
|
||||||
targetIds = Some(distinctNonSelfAuthorIds)
|
|
||||||
)
|
|
||||||
|
|
||||||
socialGraphStitchClient
|
|
||||||
.ids(request = idsRequest, requestContext = None)
|
|
||||||
.map { idResult =>
|
|
||||||
authorIds.map { authorId =>
|
|
||||||
// Users cannot follow themselves but this is in network by definition
|
|
||||||
val isSelfTweet = authorId == viewerId
|
|
||||||
val inNetworkAuthorIds = idResult.ids.toSet
|
|
||||||
val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
|
|
||||||
FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def createIdsRequest(
|
|
||||||
userId: Long,
|
|
||||||
relationshipTypes: Set[sg.RelationshipType],
|
|
||||||
targetIds: Option[Seq[Long]] = None
|
|
||||||
): sg.IdsRequest = sg.IdsRequest(
|
|
||||||
relationshipTypes.map { relationshipType =>
|
|
||||||
sg.SrcRelationship(userId, relationshipType, targets = targetIds)
|
|
||||||
}.toSeq,
|
|
||||||
Some(sg.PageRequest(selectAll = Some(true)))
|
|
||||||
)
|
|
||||||
}
|
|
@ -1,162 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.contentrecommender.{thriftscala => cr}
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
|
|
||||||
import com.twitter.ml.api.DataRecord
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
|
||||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
|
|
||||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
|
|
||||||
import com.twitter.topiclisting.TopicListingViewerContext
|
|
||||||
import com.twitter.tsp.{thriftscala => tsp}
|
|
||||||
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
import scala.collection.JavaConverters._
|
|
||||||
|
|
||||||
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
|
|
||||||
object TSPInferredTopicDataRecordFeature
|
|
||||||
extends DataRecordInAFeature[TweetCandidate]
|
|
||||||
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
|
|
||||||
override def defaultValue: DataRecord = new DataRecord()
|
|
||||||
}
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class TSPInferredTopicFeatureHydrator @Inject() (
|
|
||||||
topicSocialProofClientColumn: TopicSocialProofClientColumn,
|
|
||||||
statsReceiver: StatsReceiver,
|
|
||||||
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] =
|
|
||||||
Set(
|
|
||||||
TSPInferredTopicFeature,
|
|
||||||
TSPInferredTopicDataRecordFeature,
|
|
||||||
TopicIdSocialContextFeature,
|
|
||||||
TopicContextFunctionalityTypeFeature)
|
|
||||||
|
|
||||||
private val topK = 3
|
|
||||||
|
|
||||||
private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
|
|
||||||
sid.CandidateTweetSourceId.Simcluster,
|
|
||||||
sid.CandidateTweetSourceId.CroonTweet
|
|
||||||
)
|
|
||||||
|
|
||||||
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
|
|
||||||
private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
|
|
||||||
private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
|
|
||||||
private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
|
|
||||||
|
|
||||||
private val DefaultFeatureMap = FeatureMapBuilder()
|
|
||||||
.add(TSPInferredTopicFeature, Map.empty[Long, Double])
|
|
||||||
.add(TSPInferredTopicDataRecordFeature, new DataRecord())
|
|
||||||
.add(TopicIdSocialContextFeature, None)
|
|
||||||
.add(TopicContextFunctionalityTypeFeature, None)
|
|
||||||
.build()
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
|
||||||
val tags = candidates.collect {
|
|
||||||
case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
|
|
||||||
candidate.candidate.id -> candidate.features
|
|
||||||
.getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
|
|
||||||
}.toMap
|
|
||||||
|
|
||||||
val topicSocialProofRequest =
|
|
||||||
tsp.TopicSocialProofRequest(
|
|
||||||
userId = query.getRequiredUserId,
|
|
||||||
tweetIds = candidates.map(_.candidate.id).toSet,
|
|
||||||
displayLocation = cr.DisplayLocation.HomeTimeline,
|
|
||||||
topicListingSetting = tsp.TopicListingSetting.Followable,
|
|
||||||
context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
|
|
||||||
bypassModes = None,
|
|
||||||
// Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
|
|
||||||
tags = if (tags.isEmpty) None else Some(tags)
|
|
||||||
)
|
|
||||||
|
|
||||||
topicSocialProofClientColumn.fetcher
|
|
||||||
.fetch(topicSocialProofRequest)
|
|
||||||
.map(_.v)
|
|
||||||
.map {
|
|
||||||
case Some(response) =>
|
|
||||||
candidates.map { candidate =>
|
|
||||||
val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
|
|
||||||
if (topicWithScores.nonEmpty) {
|
|
||||||
keyFoundCounter.incr()
|
|
||||||
val (socialProofId, socialProofFunctionalityType) =
|
|
||||||
if (candidate.features
|
|
||||||
.getOrElse(CandidateSourceIdFeature, None)
|
|
||||||
.exists(sourcesToSetSocialProof.contains)) {
|
|
||||||
getSocialProof(topicWithScores)
|
|
||||||
} else {
|
|
||||||
(None, None)
|
|
||||||
}
|
|
||||||
val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
|
|
||||||
val inferredTopicDataRecord =
|
|
||||||
InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
|
|
||||||
FeatureMapBuilder()
|
|
||||||
.add(TSPInferredTopicFeature, inferredTopicFeatures)
|
|
||||||
.add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
|
|
||||||
.add(TopicIdSocialContextFeature, socialProofId)
|
|
||||||
.add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
|
|
||||||
.build()
|
|
||||||
} else {
|
|
||||||
keyLossCounter.incr()
|
|
||||||
DefaultFeatureMap
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case _ =>
|
|
||||||
requestFailCounter.incr()
|
|
||||||
candidates.map { _ =>
|
|
||||||
DefaultFeatureMap
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def getSocialProof(
|
|
||||||
topicWithScores: Seq[tsp.TopicWithScore]
|
|
||||||
): (Option[Long], Option[TopicContextFunctionalityType]) = {
|
|
||||||
val followingTopicId = topicWithScores
|
|
||||||
.collectFirst {
|
|
||||||
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
|
|
||||||
topicId
|
|
||||||
}
|
|
||||||
if (followingTopicId.nonEmpty) {
|
|
||||||
return (followingTopicId, Some(BasicTopicContextFunctionalityType))
|
|
||||||
}
|
|
||||||
val implicitFollowingId = topicWithScores.collectFirst {
|
|
||||||
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
|
|
||||||
topicId
|
|
||||||
}
|
|
||||||
if (implicitFollowingId.nonEmpty) {
|
|
||||||
return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
|
|
||||||
}
|
|
||||||
(None, None)
|
|
||||||
}
|
|
||||||
|
|
||||||
private def convertTopicWithScores(
|
|
||||||
topicWithScores: Seq[tsp.TopicWithScore],
|
|
||||||
): Map[Long, Double] = {
|
|
||||||
topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,251 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
|
||||||
import com.twitter.ml.api.DataRecord
|
|
||||||
import com.twitter.ml.api.RichDataRecord
|
|
||||||
import com.twitter.ml.api.util.FDsl._
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
|
||||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.search.common.features.{thriftscala => sc}
|
|
||||||
import com.twitter.snowflake.id.SnowflakeId
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
|
|
||||||
import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
|
|
||||||
import com.twitter.timelines.prediction.features.time_features.TimeFeatures
|
|
||||||
import com.twitter.util.Duration
|
|
||||||
import scala.collection.Searching._
|
|
||||||
|
|
||||||
object TimeFeaturesDataRecordFeature
|
|
||||||
extends DataRecordInAFeature[TweetCandidate]
|
|
||||||
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
|
|
||||||
override def defaultValue: DataRecord = new DataRecord()
|
|
||||||
}
|
|
||||||
|
|
||||||
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidate: TweetCandidate,
|
|
||||||
existingFeatures: FeatureMap
|
|
||||||
): Stitch[FeatureMap] = {
|
|
||||||
Stitch.value {
|
|
||||||
val richDataRecord = new RichDataRecord()
|
|
||||||
setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
|
|
||||||
FeatureMapBuilder()
|
|
||||||
.add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
|
|
||||||
.build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def setTimeFeatures(
|
|
||||||
richDataRecord: RichDataRecord,
|
|
||||||
candidate: TweetCandidate,
|
|
||||||
existingFeatures: FeatureMap,
|
|
||||||
query: PipelineQuery,
|
|
||||||
): Unit = {
|
|
||||||
val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
|
|
||||||
timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
|
|
||||||
}
|
|
||||||
|
|
||||||
private[feature_hydrator] def getTimeFeatures(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidate: TweetCandidate,
|
|
||||||
existingFeatures: FeatureMap,
|
|
||||||
): Option[TimeFeatures] = {
|
|
||||||
for {
|
|
||||||
requestTimestampMs <- Some(query.queryTime.inMilliseconds)
|
|
||||||
tweetId <- Some(candidate.id)
|
|
||||||
viewerId <- query.getOptionalUserId
|
|
||||||
tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
|
|
||||||
timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
|
|
||||||
accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
|
|
||||||
Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
|
|
||||||
}
|
|
||||||
timeSinceSourceTweetCreation =
|
|
||||||
existingFeatures
|
|
||||||
.getOrElse(SourceTweetIdFeature, None)
|
|
||||||
.flatMap { sourceTweetId =>
|
|
||||||
timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
|
|
||||||
requestTimestampMs - sourceTweetCreationTimeMs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.getOrElse(timeSinceTweetCreation)
|
|
||||||
if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
|
|
||||||
} yield {
|
|
||||||
val timeFeatures = TimeFeatures(
|
|
||||||
timeSinceTweetCreation = timeSinceTweetCreation,
|
|
||||||
timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
|
|
||||||
timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
|
|
||||||
isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
|
|
||||||
isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
|
|
||||||
accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
|
|
||||||
isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
|
|
||||||
)
|
|
||||||
|
|
||||||
val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
|
|
||||||
existingFeatures.getOrElse(EarlybirdFeature, None),
|
|
||||||
timeFeatures,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
).getOrElse(timeFeatures)
|
|
||||||
|
|
||||||
val nonPollingTimestampsMs =
|
|
||||||
query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
|
|
||||||
val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
|
|
||||||
timeFeaturesWithLastEngagement,
|
|
||||||
requestTimestampMs,
|
|
||||||
tweetCreationTimeMs,
|
|
||||||
nonPollingTimestampsMs
|
|
||||||
)
|
|
||||||
timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
|
|
||||||
if (SnowflakeId.isSnowflakeId(tweetOrUserId))
|
|
||||||
Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
|
|
||||||
else None
|
|
||||||
}
|
|
||||||
|
|
||||||
private def addLastEngagementTimeFeatures(
|
|
||||||
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
|
|
||||||
timeFeatures: TimeFeatures,
|
|
||||||
timeSinceSourceTweetCreation: Long
|
|
||||||
): Option[TimeFeatures] = {
|
|
||||||
tweetFeaturesOpt.map { tweetFeatures =>
|
|
||||||
val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
|
|
||||||
val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
|
|
||||||
val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
|
|
||||||
val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
|
|
||||||
|
|
||||||
timeFeatures.copy(
|
|
||||||
lastFavSinceCreationHrs = lastFavSinceCreationHrs,
|
|
||||||
lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
|
|
||||||
lastReplySinceCreationHrs = lastReplySinceCreationHrs,
|
|
||||||
lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
|
|
||||||
timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
|
|
||||||
lastFavSinceCreationHrs,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
),
|
|
||||||
timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
|
|
||||||
lastRetweetSinceCreationHrs,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
),
|
|
||||||
timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
|
|
||||||
lastReplySinceCreationHrs,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
),
|
|
||||||
timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
|
|
||||||
lastQuoteSinceCreationHrs,
|
|
||||||
timeSinceSourceTweetCreation
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def addNonPollingTimeFeatures(
|
|
||||||
timeFeatures: TimeFeatures,
|
|
||||||
requestTimestampMs: Long,
|
|
||||||
creationTimeMs: Long,
|
|
||||||
nonPollingTimestampsMs: Option[Seq[Long]]
|
|
||||||
): Option[TimeFeatures] = {
|
|
||||||
for {
|
|
||||||
nonPollingTimestampsMs <- nonPollingTimestampsMs
|
|
||||||
lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
|
|
||||||
earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
|
|
||||||
} yield {
|
|
||||||
val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
|
|
||||||
val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
|
|
||||||
1.0,
|
|
||||||
timeFeatures.timeSinceTweetCreation
|
|
||||||
)
|
|
||||||
/*
|
|
||||||
* Non-polling timestamps are stored in chronological order.
|
|
||||||
* The latest timestamps occur first, therefore we need to explicitly search in reverse order.
|
|
||||||
*/
|
|
||||||
val nonPollingRequestsSinceTweetCreation =
|
|
||||||
if (nonPollingTimestampsMs.nonEmpty) {
|
|
||||||
nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
* Calculate the average time between non-polling requests; include
|
|
||||||
* request time in this calculation as latest timestamp.
|
|
||||||
*/
|
|
||||||
val timeBetweenNonPollingRequestsAvg =
|
|
||||||
(requestTimestampMs - earliestNonPollingTimestampMs) / math
|
|
||||||
.max(1.0, nonPollingTimestampsMs.size)
|
|
||||||
val timeFeaturesWithNonPolling = timeFeatures.copy(
|
|
||||||
timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
|
|
||||||
timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
|
|
||||||
nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
|
|
||||||
tweetAgeRatio = Some(tweetAgeRatio)
|
|
||||||
)
|
|
||||||
timeFeaturesWithNonPolling
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private[this] def getTimeSinceLastEngagementHrs(
|
|
||||||
lastEngagementTimeSinceCreationHrsOpt: Option[Double],
|
|
||||||
timeSinceTweetCreation: Long
|
|
||||||
): Option[Double] = {
|
|
||||||
lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
|
|
||||||
val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
|
|
||||||
timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
|
|
||||||
val record = richDataRecord.getRecord
|
|
||||||
.setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
|
|
||||||
.setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
|
|
||||||
.setFeatureValueFromOption(
|
|
||||||
TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
|
|
||||||
features.timeSinceViewerAccountCreationSecs)
|
|
||||||
.setFeatureValue(
|
|
||||||
USER_ID_IS_SNOWFLAKE_ID,
|
|
||||||
features.timeSinceViewerAccountCreationSecs.isDefined
|
|
||||||
)
|
|
||||||
.setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
|
|
||||||
.setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
|
|
||||||
.setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
|
|
||||||
.setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
|
|
||||||
.setFeatureValueFromOption(
|
|
||||||
LAST_RETWEET_SINCE_CREATION_HRS,
|
|
||||||
features.lastRetweetSinceCreationHrs
|
|
||||||
)
|
|
||||||
.setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
|
|
||||||
.setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
|
|
||||||
.setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
|
|
||||||
.setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
|
|
||||||
.setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
|
|
||||||
.setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
|
|
||||||
/*
|
|
||||||
* set features whose values are optional as some users do not have non-polling timestamps
|
|
||||||
*/
|
|
||||||
features.timeBetweenNonPollingRequestsAvg.foreach(
|
|
||||||
record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
|
|
||||||
)
|
|
||||||
features.timeSinceLastNonPollingRequest.foreach(
|
|
||||||
record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
|
|
||||||
)
|
|
||||||
features.nonPollingRequestsSinceTweetCreation.foreach(
|
|
||||||
record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
|
|
||||||
)
|
|
||||||
features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
|
|
||||||
}
|
|
||||||
}
|
|
@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
|
|||||||
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
|
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
|
||||||
extends QueryFeatureHydrator[Query] {
|
extends QueryFeatureHydrator[Query] {
|
||||||
|
|
||||||
private val TweetImpressionTTL = 1.day
|
private val TweetImpressionTTL = 2.days
|
||||||
private val TweetImpressionCap = 3000
|
private val TweetImpressionCap = 5000
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")
|
||||||
|
|
||||||
|
@ -1,6 +1,9 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator
|
package com.twitter.home_mixer.functional_component.feature_hydrator
|
||||||
|
|
||||||
|
import com.twitter.finagle.stats.StatsReceiver
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
|
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
|
||||||
@ -10,11 +13,13 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
|
|||||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
|
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
||||||
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
import com.twitter.home_mixer.util.tweetypie.RequestFields
|
import com.twitter.home_mixer.util.tweetypie.RequestFields
|
||||||
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
|
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
|
||||||
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
|
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
|
||||||
@ -29,17 +34,22 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
|
|||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
|
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
|
||||||
import com.twitter.tweetypie.{thriftscala => tp}
|
import com.twitter.tweetypie.{thriftscala => tp}
|
||||||
|
import com.twitter.util.logging.Logging
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
|
class TweetypieFeatureHydrator @Inject() (
|
||||||
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
|
tweetypieStitchClient: TweetypieStitchClient,
|
||||||
|
statsReceiver: StatsReceiver)
|
||||||
|
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
|
||||||
|
with Logging {
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(
|
override val features: Set[Feature[_, _]] = Set(
|
||||||
AuthorIdFeature,
|
AuthorIdFeature,
|
||||||
|
ExclusiveConversationAuthorIdFeature,
|
||||||
InReplyToTweetIdFeature,
|
InReplyToTweetIdFeature,
|
||||||
IsHydratedFeature,
|
IsHydratedFeature,
|
||||||
IsNsfw,
|
IsNsfw,
|
||||||
@ -51,6 +61,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
SourceTweetIdFeature,
|
SourceTweetIdFeature,
|
||||||
SourceUserIdFeature,
|
SourceUserIdFeature,
|
||||||
TweetTextFeature,
|
TweetTextFeature,
|
||||||
|
TweetLanguageFeature,
|
||||||
VisibilityReason
|
VisibilityReason
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -70,9 +81,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
): Stitch[FeatureMap] = {
|
): Stitch[FeatureMap] = {
|
||||||
val safetyLevel = query.product match {
|
val safetyLevel = query.product match {
|
||||||
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
|
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
|
||||||
case ForYouProduct => rtf.SafetyLevel.TimelineHome
|
case ForYouProduct =>
|
||||||
|
val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
|
||||||
|
if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
|
||||||
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
|
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
|
||||||
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
|
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
|
||||||
|
case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
|
||||||
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
|
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -82,9 +96,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
includeQuotedTweet = true,
|
includeQuotedTweet = true,
|
||||||
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
|
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
|
||||||
safetyLevel = Some(safetyLevel),
|
safetyLevel = Some(safetyLevel),
|
||||||
forUserId = Some(query.getRequiredUserId)
|
forUserId = query.getOptionalUserId
|
||||||
)
|
)
|
||||||
|
|
||||||
|
val exclusiveAuthorIdOpt =
|
||||||
|
existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
|
||||||
|
|
||||||
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
|
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
|
||||||
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
|
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
|
||||||
val coreData = found.tweet.coreData
|
val coreData = found.tweet.coreData
|
||||||
@ -106,6 +123,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
|
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
|
||||||
|
|
||||||
val tweetText = coreData.map(_.text)
|
val tweetText = coreData.map(_.text)
|
||||||
|
val tweetLanguage = found.tweet.language.map(_.language)
|
||||||
|
|
||||||
val tweetAuthorId = coreData.map(_.userId)
|
val tweetAuthorId = coreData.map(_.userId)
|
||||||
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
|
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
|
||||||
@ -127,6 +145,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
|
|
||||||
FeatureMapBuilder()
|
FeatureMapBuilder()
|
||||||
.add(AuthorIdFeature, tweetAuthorId)
|
.add(AuthorIdFeature, tweetAuthorId)
|
||||||
|
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
|
||||||
.add(InReplyToTweetIdFeature, inReplyToTweetId)
|
.add(InReplyToTweetIdFeature, inReplyToTweetId)
|
||||||
.add(IsHydratedFeature, true)
|
.add(IsHydratedFeature, true)
|
||||||
.add(IsNsfw, Some(isNsfw))
|
.add(IsNsfw, Some(isNsfw))
|
||||||
@ -137,20 +156,24 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
|
|||||||
.add(QuotedUserIdFeature, quotedTweetUserId)
|
.add(QuotedUserIdFeature, quotedTweetUserId)
|
||||||
.add(SourceTweetIdFeature, retweetedTweetId)
|
.add(SourceTweetIdFeature, retweetedTweetId)
|
||||||
.add(SourceUserIdFeature, retweetedTweetUserId)
|
.add(SourceUserIdFeature, retweetedTweetUserId)
|
||||||
|
.add(TweetLanguageFeature, tweetLanguage)
|
||||||
.add(TweetTextFeature, tweetText)
|
.add(TweetTextFeature, tweetText)
|
||||||
.add(VisibilityReason, found.suppressReason)
|
.add(VisibilityReason, found.suppressReason)
|
||||||
.build()
|
.build()
|
||||||
|
|
||||||
// If no tweet result found, return default and pre-existing features
|
// If no tweet result found, return default and pre-existing features
|
||||||
case _ =>
|
case _ =>
|
||||||
DefaultFeatureMap +
|
DefaultFeatureMap ++ FeatureMapBuilder()
|
||||||
(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) +
|
.add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
|
||||||
(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) +
|
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
|
||||||
(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) +
|
.add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
|
||||||
(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) +
|
.add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
|
||||||
(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) +
|
.add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
|
||||||
(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) +
|
.add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
|
||||||
(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
|
.add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
|
||||||
|
.add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
|
||||||
|
.add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
|
||||||
|
.build()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,59 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
|
|
||||||
|
|
||||||
import com.twitter.ml.api.DataRecordMerger
|
|
||||||
import com.twitter.ml.api.Feature
|
|
||||||
import com.twitter.ml.api.FeatureContext
|
|
||||||
import com.twitter.ml.api.RichDataRecord
|
|
||||||
import com.twitter.ml.api.util.CompactDataRecordConverter
|
|
||||||
import com.twitter.ml.api.util.FDsl._
|
|
||||||
import com.twitter.timelines.author_features.v1.{thriftjava => af}
|
|
||||||
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
|
|
||||||
import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
|
|
||||||
import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
|
|
||||||
|
|
||||||
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
|
|
||||||
|
|
||||||
private val originalAuthorAggregatesFeatures =
|
|
||||||
TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
|
|
||||||
.buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
|
|
||||||
private val authorFeatures = originalAuthorAggregatesFeatures ++
|
|
||||||
Seq(
|
|
||||||
UserHealthFeatures.AuthorState,
|
|
||||||
UserHealthFeatures.NumAuthorFollowers,
|
|
||||||
UserHealthFeatures.NumAuthorConnectDays,
|
|
||||||
UserHealthFeatures.NumAuthorConnect)
|
|
||||||
private val featureContext = new FeatureContext(authorFeatures: _*)
|
|
||||||
|
|
||||||
override def getFeatureContext: FeatureContext = featureContext
|
|
||||||
|
|
||||||
override val commonFeatures: Set[Feature[_]] = Set.empty
|
|
||||||
|
|
||||||
private val compactDataRecordConverter = new CompactDataRecordConverter()
|
|
||||||
private val drMerger = new DataRecordMerger()
|
|
||||||
|
|
||||||
override def setFeatures(
|
|
||||||
authorFeaturesOpt: Option[af.AuthorFeatures],
|
|
||||||
richDataRecord: RichDataRecord
|
|
||||||
): Unit = {
|
|
||||||
authorFeaturesOpt.foreach { authorFeatures =>
|
|
||||||
val dataRecord = richDataRecord.getRecord
|
|
||||||
|
|
||||||
dataRecord.setFeatureValue(
|
|
||||||
UserHealthFeatures.AuthorState,
|
|
||||||
authorFeatures.user_health.user_state.getValue.toLong)
|
|
||||||
dataRecord.setFeatureValue(
|
|
||||||
UserHealthFeatures.NumAuthorFollowers,
|
|
||||||
authorFeatures.user_health.num_followers.toDouble)
|
|
||||||
dataRecord.setFeatureValue(
|
|
||||||
UserHealthFeatures.NumAuthorConnectDays,
|
|
||||||
authorFeatures.user_health.num_connect_days.toDouble)
|
|
||||||
dataRecord.setFeatureValue(
|
|
||||||
UserHealthFeatures.NumAuthorConnect,
|
|
||||||
authorFeatures.user_health.num_connect.toDouble)
|
|
||||||
|
|
||||||
val originalAuthorAggregatesDataRecord =
|
|
||||||
compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
|
|
||||||
drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,22 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator {
|
|
||||||
|
|
||||||
override val identifier: FeatureHydratorIdentifier =
|
|
||||||
FeatureHydratorIdentifier("Phase2EdgeAggregate")
|
|
||||||
|
|
||||||
override val aggregateFeatures: Set[BaseEdgeAggregateFeature] =
|
|
||||||
Set(
|
|
||||||
UserEngagerAggregateFeature,
|
|
||||||
UserEngagerGoodClickAggregateFeature,
|
|
||||||
UserInferredTopicAggregateFeature,
|
|
||||||
UserTopicAggregateFeature,
|
|
||||||
UserMediaUnderstandingAnnotationAggregateFeature
|
|
||||||
)
|
|
||||||
}
|
|
@ -4,6 +4,7 @@ scala_library(
|
|||||||
strict_deps = True,
|
strict_deps = True,
|
||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
|
||||||
@ -16,9 +17,11 @@ scala_library(
|
|||||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||||
"stitch/stitch-core",
|
"stitch/stitch-core",
|
||||||
|
"stitch/stitch-socialgraph",
|
||||||
"stitch/stitch-tweetypie",
|
"stitch/stitch-tweetypie",
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
||||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
||||||
|
"util/util-slf4j-api/src/main/scala",
|
||||||
],
|
],
|
||||||
exports = [
|
exports = [
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
||||||
|
@ -72,7 +72,7 @@ object FeedbackFatigueFilter
|
|||||||
|
|
||||||
originalAuthorId.exists(authorsToFilter.contains) ||
|
originalAuthorId.exists(authorsToFilter.contains) ||
|
||||||
(likers.nonEmpty && eligibleLikers.isEmpty) ||
|
(likers.nonEmpty && eligibleLikers.isEmpty) ||
|
||||||
(followers.nonEmpty && eligibleFollowers.isEmpty) ||
|
(followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) ||
|
||||||
(candidate.features.getOrElse(IsRetweetFeature, false) &&
|
(candidate.features.getOrElse(IsRetweetFeature, false) &&
|
||||||
authorId.exists(retweetersToFilter.contains))
|
authorId.exists(retweetersToFilter.contains))
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,70 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.filter
|
||||||
|
|
||||||
|
import com.twitter.finagle.stats.StatsReceiver
|
||||||
|
import com.twitter.finagle.tracing.Trace
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||||
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.socialgraph.{thriftscala => sg}
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
import com.twitter.stitch.socialgraph.SocialGraph
|
||||||
|
import com.twitter.util.logging.Logging
|
||||||
|
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author
|
||||||
|
*
|
||||||
|
* If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for
|
||||||
|
* subscription tweets, so we explicitly filter those tweets out.
|
||||||
|
*/
|
||||||
|
@Singleton
|
||||||
|
case class InvalidSubscriptionTweetFilter @Inject() (
|
||||||
|
socialGraphClient: SocialGraph,
|
||||||
|
statsReceiver: StatsReceiver)
|
||||||
|
extends Filter[PipelineQuery, TweetCandidate]
|
||||||
|
with Logging {
|
||||||
|
|
||||||
|
override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet")
|
||||||
|
|
||||||
|
private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
|
||||||
|
private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet")
|
||||||
|
private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet")
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
|
): Stitch[FilterResult[TweetCandidate]] = Stitch
|
||||||
|
.traverse(candidates) { candidate =>
|
||||||
|
val exclusiveAuthorId =
|
||||||
|
candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None)
|
||||||
|
|
||||||
|
if (exclusiveAuthorId.isDefined) {
|
||||||
|
val request = sg.ExistsRequest(
|
||||||
|
source = query.getRequiredUserId,
|
||||||
|
target = exclusiveAuthorId.get,
|
||||||
|
relationships =
|
||||||
|
Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)),
|
||||||
|
)
|
||||||
|
socialGraphClient.exists(request).map(_.exists).map { valid =>
|
||||||
|
if (!valid) invalidCounter.incr() else validCounter.incr()
|
||||||
|
valid
|
||||||
|
}
|
||||||
|
} else Stitch.value(true)
|
||||||
|
}.map { validResults =>
|
||||||
|
val (kept, removed) = candidates
|
||||||
|
.map(_.candidate)
|
||||||
|
.zip(validResults)
|
||||||
|
.partition { case (candidate, valid) => valid }
|
||||||
|
|
||||||
|
val keptCandidates = kept.map { case (candidate, _) => candidate }
|
||||||
|
val removedCandidates = removed.map { case (candidate, _) => candidate }
|
||||||
|
|
||||||
|
FilterResult(kept = keptCandidates, removed = removedCandidates)
|
||||||
|
}
|
||||||
|
}
|
@ -1,59 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.filter
|
|
||||||
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
|
||||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
|
||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
|
||||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Predicate which will be applied to each candidate. True indicates that the candidate will be
|
|
||||||
* @tparam Candidate - the type of the candidate
|
|
||||||
*/
|
|
||||||
trait ShouldKeepCandidate {
|
|
||||||
def apply(features: FeatureMap): Boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
object PredicateFeatureFilter {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity,
|
|
||||||
* we recommend including the name of the shouldKeepCandidate parameter.
|
|
||||||
*
|
|
||||||
* @param identifier A FilterIdentifier for the new filter
|
|
||||||
* @param shouldKeepCandidate A predicate function. Candidates will be kept when
|
|
||||||
* this function returns True.
|
|
||||||
*/
|
|
||||||
def fromPredicate[Candidate <: UniversalNoun[Any]](
|
|
||||||
identifier: FilterIdentifier,
|
|
||||||
shouldKeepCandidate: ShouldKeepCandidate
|
|
||||||
): Filter[PipelineQuery, Candidate] = {
|
|
||||||
val i = identifier
|
|
||||||
|
|
||||||
new Filter[PipelineQuery, Candidate] {
|
|
||||||
override val identifier: FilterIdentifier = i
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Filter the list of candidates
|
|
||||||
*
|
|
||||||
* @return a FilterResult including both the list of kept candidate and the list of removed candidates
|
|
||||||
*/
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
|
||||||
): Stitch[FilterResult[Candidate]] = {
|
|
||||||
val allowedIds = candidates
|
|
||||||
.filter(candidate => shouldKeepCandidate(candidate.features)).map(_.candidate.id).toSet
|
|
||||||
|
|
||||||
val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition {
|
|
||||||
candidate => allowedIds.contains(candidate.id)
|
|
||||||
}
|
|
||||||
|
|
||||||
Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,8 +1,6 @@
|
|||||||
package com.twitter.home_mixer.functional_component.filter
|
package com.twitter.home_mixer.functional_component.filter
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||||
@ -11,24 +9,20 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
|||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
|
|
||||||
object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] {
|
object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||||
|
|
||||||
override val identifier: FilterIdentifier = FilterIdentifier("KeepBestOutOfNetworkTweetPerAuthor")
|
override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews")
|
||||||
|
|
||||||
override def apply(
|
override def apply(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
): Stitch[FilterResult[TweetCandidate]] = {
|
): Stitch[FilterResult[TweetCandidate]] = {
|
||||||
// Set containing best OON tweet for each authorId
|
|
||||||
val bestCandidatesForAuthorId = candidates
|
val servedTweetPreviewIds =
|
||||||
.filter(!_.features.getOrElse(InNetworkFeature, true))
|
query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet
|
||||||
.groupBy(_.features.getOrElse(AuthorIdFeature, None))
|
|
||||||
.values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None)))
|
|
||||||
.toSet
|
|
||||||
|
|
||||||
val (removed, kept) = candidates.partition { candidate =>
|
val (removed, kept) = candidates.partition { candidate =>
|
||||||
!candidate.features.getOrElse(InNetworkFeature, true) &&
|
servedTweetPreviewIds.contains(candidate.candidate.id)
|
||||||
!bestCandidatesForAuthorId.contains(candidate)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))
|
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))
|
@ -0,0 +1,32 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.filter
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||||
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
|
||||||
|
object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||||
|
override val identifier: FilterIdentifier = FilterIdentifier("Reply")
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
|
): Stitch[FilterResult[TweetCandidate]] = {
|
||||||
|
|
||||||
|
val (kept, removed) = candidates
|
||||||
|
.partition { candidate =>
|
||||||
|
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
|
||||||
|
}
|
||||||
|
|
||||||
|
val filterResult = FilterResult(
|
||||||
|
kept = kept.map(_.candidate),
|
||||||
|
removed = removed.map(_.candidate)
|
||||||
|
)
|
||||||
|
|
||||||
|
Stitch.value(filterResult)
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,32 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.filter
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||||
|
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||||
|
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.stitch.Stitch
|
||||||
|
|
||||||
|
object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||||
|
override val identifier: FilterIdentifier = FilterIdentifier("Retweet")
|
||||||
|
|
||||||
|
override def apply(
|
||||||
|
query: PipelineQuery,
|
||||||
|
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||||
|
): Stitch[FilterResult[TweetCandidate]] = {
|
||||||
|
|
||||||
|
val (kept, removed) = candidates
|
||||||
|
.partition { candidate =>
|
||||||
|
!candidate.features.getOrElse(IsRetweetFeature, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
val filterResult = FilterResult(
|
||||||
|
kept = kept.map(_.candidate),
|
||||||
|
removed = removed.map(_.candidate)
|
||||||
|
)
|
||||||
|
|
||||||
|
Stitch.value(filterResult)
|
||||||
|
}
|
||||||
|
}
|
@ -14,7 +14,6 @@ scala_library(
|
|||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
||||||
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||||
"stitch/stitch-socialgraph",
|
"stitch/stitch-socialgraph",
|
||||||
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
|
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
||||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
||||||
],
|
],
|
||||||
|
@ -1,14 +1,14 @@
|
|||||||
package com.twitter.home_mixer.functional_component.gate
|
package com.twitter.home_mixer.functional_component.gate
|
||||||
|
|
||||||
|
import com.twitter.conversions.DurationOps._
|
||||||
|
import com.twitter.product_mixer.core.feature.Feature
|
||||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
|
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.util.Duration
|
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
|
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
|
||||||
|
import com.twitter.util.Duration
|
||||||
|
|
||||||
object DismissFatigueGate {
|
object DismissFatigueGate {
|
||||||
// how long a dismiss action from user needs to be respected
|
// how long a dismiss action from user needs to be respected
|
||||||
|
@ -1,18 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.gate
|
|
||||||
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import scala.reflect.runtime.universe._
|
|
||||||
|
|
||||||
case class NonEmptySeqFeatureGate[T: TypeTag](
|
|
||||||
feature: Feature[PipelineQuery, Seq[T]])
|
|
||||||
extends Gate[PipelineQuery] {
|
|
||||||
|
|
||||||
override val identifier: GateIdentifier = GateIdentifier(s"NonEmptySeq$feature")
|
|
||||||
|
|
||||||
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
|
|
||||||
Stitch.value(query.features.exists(_.get(feature).nonEmpty))
|
|
||||||
}
|
|
@ -19,7 +19,7 @@ object EditedTweetsCandidatePipelineQueryTransformer
|
|||||||
override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
|
override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
|
||||||
|
|
||||||
// The time window for which a tweet remains editable after creation.
|
// The time window for which a tweet remains editable after creation.
|
||||||
private val EditTimeWindow = 30.minutes
|
private val EditTimeWindow = 60.minutes
|
||||||
|
|
||||||
override def transform(query: PipelineQuery): Seq[Long] = {
|
override def transform(query: PipelineQuery): Seq[Long] = {
|
||||||
val applicableCandidates = getApplicableCandidates(query)
|
val applicableCandidates = getApplicableCandidates(query)
|
||||||
@ -29,8 +29,8 @@ object EditedTweetsCandidatePipelineQueryTransformer
|
|||||||
// Any tweets in it could have become stale since being served.
|
// Any tweets in it could have become stale since being served.
|
||||||
val previousTimelineLoadTime = applicableCandidates.head.servedTime
|
val previousTimelineLoadTime = applicableCandidates.head.servedTime
|
||||||
|
|
||||||
// The time window for editing a tweet is 30 minutes,
|
// The time window for editing a tweet is 60 minutes,
|
||||||
// so we ignore responses older than (PTL Time - 30 mins).
|
// so we ignore responses older than (PTL Time - 60 mins).
|
||||||
val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
|
val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
|
||||||
.takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)
|
.takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)
|
||||||
|
|
||||||
|
@ -6,7 +6,6 @@ scala_library(
|
|||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
||||||
|
@ -35,8 +35,8 @@ object FeedbackFatigueScorer
|
|||||||
override def onlyIf(query: PipelineQuery): Boolean =
|
override def onlyIf(query: PipelineQuery): Boolean =
|
||||||
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
|
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
|
||||||
|
|
||||||
private val DurationForFiltering = 14.days
|
val DurationForFiltering = 14.days
|
||||||
private val DurationForDiscounting = 140.days
|
val DurationForDiscounting = 140.days
|
||||||
private val ScoreMultiplierLowerBound = 0.2
|
private val ScoreMultiplierLowerBound = 0.2
|
||||||
private val ScoreMultiplierUpperBound = 1.0
|
private val ScoreMultiplierUpperBound = 1.0
|
||||||
private val ScoreMultiplierIncrementsCount = 4
|
private val ScoreMultiplierIncrementsCount = 4
|
||||||
@ -76,8 +76,27 @@ object FeedbackFatigueScorer
|
|||||||
feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty))
|
feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty))
|
||||||
|
|
||||||
val featureMaps = candidates.map { candidate =>
|
val featureMaps = candidates.map { candidate =>
|
||||||
|
val multiplier = getScoreMultiplier(
|
||||||
|
candidate,
|
||||||
|
authorsToDiscount,
|
||||||
|
likersToDiscount,
|
||||||
|
followersToDiscount,
|
||||||
|
retweetersToDiscount
|
||||||
|
)
|
||||||
val score = candidate.features.getOrElse(ScoreFeature, None)
|
val score = candidate.features.getOrElse(ScoreFeature, None)
|
||||||
|
FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
|
||||||
|
}
|
||||||
|
|
||||||
|
Stitch.value(featureMaps)
|
||||||
|
}
|
||||||
|
|
||||||
|
def getScoreMultiplier(
|
||||||
|
candidate: CandidateWithFeatures[TweetCandidate],
|
||||||
|
authorsToDiscount: Map[Long, Double],
|
||||||
|
likersToDiscount: Map[Long, Double],
|
||||||
|
followersToDiscount: Map[Long, Double],
|
||||||
|
retweetersToDiscount: Map[Long, Double],
|
||||||
|
): Double = {
|
||||||
val originalAuthorId =
|
val originalAuthorId =
|
||||||
CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
|
CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
|
||||||
val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
|
val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
|
||||||
@ -92,7 +111,8 @@ object FeedbackFatigueScorer
|
|||||||
val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
|
val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
|
||||||
val followerMultipliers = followers.flatMap(followersToDiscount.get)
|
val followerMultipliers = followers.flatMap(followersToDiscount.get)
|
||||||
val followerMultiplier =
|
val followerMultiplier =
|
||||||
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size)
|
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size &&
|
||||||
|
likers.isEmpty)
|
||||||
followerMultipliers.max
|
followerMultipliers.max
|
||||||
else 1.0
|
else 1.0
|
||||||
|
|
||||||
@ -102,16 +122,10 @@ object FeedbackFatigueScorer
|
|||||||
retweetersToDiscount.getOrElse(authorId, 1.0)
|
retweetersToDiscount.getOrElse(authorId, 1.0)
|
||||||
else 1.0
|
else 1.0
|
||||||
|
|
||||||
val multiplier =
|
|
||||||
originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
|
originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
|
||||||
|
|
||||||
FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Stitch.value(featureMaps)
|
def getUserDiscounts(
|
||||||
}
|
|
||||||
|
|
||||||
private def getUserDiscounts(
|
|
||||||
queryTime: Time,
|
queryTime: Time,
|
||||||
feedbackEntries: Seq[FeedbackEntry],
|
feedbackEntries: Seq[FeedbackEntry],
|
||||||
): Map[Long, Double] = {
|
): Map[Long, Double] = {
|
||||||
|
@ -1,61 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.scorer
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorInNetworkMultiplierParam
|
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorOutOfNetworkMultiplierParam
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.feature.Feature
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
||||||
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
|
|
||||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Scales scores of tweets whose author is Blue Verified by the provided scale factor
|
|
||||||
*/
|
|
||||||
object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate] {
|
|
||||||
|
|
||||||
override val identifier: ScorerIdentifier = ScorerIdentifier("VerifiedAuthorScaling")
|
|
||||||
|
|
||||||
override val features: Set[Feature[_, _]] = Set(ScoreFeature)
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
query: PipelineQuery,
|
|
||||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
|
||||||
): Stitch[Seq[FeatureMap]] = {
|
|
||||||
Stitch.value {
|
|
||||||
candidates.map { candidate =>
|
|
||||||
val score = candidate.features.getOrElse(ScoreFeature, None)
|
|
||||||
val updatedScore = getUpdatedScore(score, candidate, query)
|
|
||||||
FeatureMapBuilder().add(ScoreFeature, updatedScore).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* We should only be applying this multiplier if the author of the candidate is Blue Verified.
|
|
||||||
* We also treat In-Network vs Out-of-Network differently.
|
|
||||||
*/
|
|
||||||
private def getUpdatedScore(
|
|
||||||
score: Option[Double],
|
|
||||||
candidate: CandidateWithFeatures[TweetCandidate],
|
|
||||||
query: PipelineQuery
|
|
||||||
): Option[Double] = {
|
|
||||||
val isAuthorBlueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false)
|
|
||||||
|
|
||||||
if (isAuthorBlueVerified) {
|
|
||||||
val isCandidateInNetwork = candidate.features.getOrElse(InNetworkFeature, false)
|
|
||||||
|
|
||||||
val scaleFactor =
|
|
||||||
if (isCandidateInNetwork) query.params(BlueVerifiedAuthorInNetworkMultiplierParam)
|
|
||||||
else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam)
|
|
||||||
|
|
||||||
score.map(_ * scaleFactor)
|
|
||||||
} else score
|
|
||||||
}
|
|
||||||
}
|
|
@ -5,6 +5,7 @@ scala_library(
|
|||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
package com.twitter.home_mixer.functional_component.selector
|
package com.twitter.home_mixer.functional_component.selector
|
||||||
|
|
||||||
import com.twitter.home_mixer.functional_component.decorator.HomeClientEventDetailsBuilder
|
import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature
|
||||||
|
@ -7,14 +7,12 @@ scala_library(
|
|||||||
"3rdparty/jvm/javax/inject:javax.inject",
|
"3rdparty/jvm/javax/inject:javax.inject",
|
||||||
"eventbus/client/src/main/scala/com/twitter/eventbus/client",
|
"eventbus/client/src/main/scala/com/twitter/eventbus/client",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
|
||||||
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||||
@ -22,6 +20,7 @@ scala_library(
|
|||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module",
|
||||||
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module",
|
||||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect",
|
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
||||||
"src/scala/com/twitter/timelines/prediction/common/adapters",
|
"src/scala/com/twitter/timelines/prediction/common/adapters",
|
||||||
@ -33,13 +32,12 @@ scala_library(
|
|||||||
"src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java",
|
"src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java",
|
||||||
"src/thrift/com/twitter/timelines/timeline_logging:thrift-scala",
|
"src/thrift/com/twitter/timelines/timeline_logging:thrift-scala",
|
||||||
"src/thrift/com/twitter/user_session_store:thrift-scala",
|
"src/thrift/com/twitter/user_session_store:thrift-scala",
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/core",
|
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
|
||||||
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/uss",
|
|
||||||
"timelines/ml:kafka",
|
"timelines/ml:kafka",
|
||||||
"timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka",
|
"timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka",
|
||||||
"timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding",
|
"timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/clientconfig",
|
"timelines/src/main/scala/com/twitter/timelines/clientconfig",
|
||||||
|
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
|
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
|
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
|
||||||
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
|
||||||
|
@ -2,13 +2,14 @@ package com.twitter.home_mixer.functional_component.side_effect
|
|||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
import com.twitter.conversions.DurationOps._
|
||||||
import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates
|
import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates
|
||||||
import com.twitter.home_mixer.functional_component.decorator.HomeTweetTypePredicates
|
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature
|
import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent
|
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent
|
||||||
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace
|
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
@ -18,15 +19,17 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|||||||
import com.twitter.timelines.injection.scribe.InjectionScribeUtil
|
import com.twitter.timelines.injection.scribe.InjectionScribeUtil
|
||||||
|
|
||||||
private[side_effect] sealed trait ClientEventsBuilder {
|
private[side_effect] sealed trait ClientEventsBuilder {
|
||||||
private val FollowingSection = Some("home_latest")
|
private val FollowingSection = Some("latest")
|
||||||
private val ForYouSection = Some("home")
|
private val ForYouSection = Some("home")
|
||||||
private val ListTweetsSection = Some("list")
|
private val ListTweetsSection = Some("list")
|
||||||
|
private val SubscribedSection = Some("subscribed")
|
||||||
|
|
||||||
protected def section(query: PipelineQuery): Option[String] = {
|
protected def section(query: PipelineQuery): Option[String] = {
|
||||||
query.product match {
|
query.product match {
|
||||||
case FollowingProduct => FollowingSection
|
case FollowingProduct => FollowingSection
|
||||||
case ForYouProduct => ForYouSection
|
case ForYouProduct => ForYouSection
|
||||||
case ListTweetsProduct => ListTweetsSection
|
case ListTweetsProduct => ListTweetsSection
|
||||||
|
case SubscribedProduct => SubscribedSection
|
||||||
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -52,6 +55,7 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
|
|||||||
private val InjectedComponent = Some("injected")
|
private val InjectedComponent = Some("injected")
|
||||||
private val PromotedComponent = Some("promoted")
|
private val PromotedComponent = Some("promoted")
|
||||||
private val WhoToFollowComponent = Some("who_to_follow")
|
private val WhoToFollowComponent = Some("who_to_follow")
|
||||||
|
private val WhoToSubscribeComponent = Some("who_to_subscribe")
|
||||||
private val WithVideoDurationComponent = Some("with_video_duration")
|
private val WithVideoDurationComponent = Some("with_video_duration")
|
||||||
private val VideoDurationSumElement = Some("video_duration_sum")
|
private val VideoDurationSumElement = Some("video_duration_sum")
|
||||||
private val NumVideosElement = Some("num_videos")
|
private val NumVideosElement = Some("num_videos")
|
||||||
@ -60,7 +64,8 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
|
|||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
injectedTweets: Seq[ItemCandidateWithDetails],
|
injectedTweets: Seq[ItemCandidateWithDetails],
|
||||||
promotedTweets: Seq[ItemCandidateWithDetails],
|
promotedTweets: Seq[ItemCandidateWithDetails],
|
||||||
whoToFollowUsers: Seq[ItemCandidateWithDetails]
|
whoToFollowUsers: Seq[ItemCandidateWithDetails],
|
||||||
|
whoToSubscribeUsers: Seq[ItemCandidateWithDetails]
|
||||||
): Seq[ClientEvent] = {
|
): Seq[ClientEvent] = {
|
||||||
val baseEventNamespace = EventNamespace(
|
val baseEventNamespace = EventNamespace(
|
||||||
section = section(query),
|
section = section(query),
|
||||||
@ -77,6 +82,9 @@ private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder {
|
|||||||
ClientEvent(
|
ClientEvent(
|
||||||
baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction),
|
baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction),
|
||||||
eventValue = count(whoToFollowUsers)),
|
eventValue = count(whoToFollowUsers)),
|
||||||
|
ClientEvent(
|
||||||
|
baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction),
|
||||||
|
eventValue = count(whoToSubscribeUsers)),
|
||||||
)
|
)
|
||||||
|
|
||||||
val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map {
|
val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map {
|
||||||
|
@ -5,6 +5,7 @@ import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
|||||||
import com.twitter.home_mixer.util.CandidatesUtil
|
import com.twitter.home_mixer.util.CandidatesUtil
|
||||||
import com.twitter.logpipeline.client.common.EventPublisher
|
import com.twitter.logpipeline.client.common.EventPublisher
|
||||||
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect
|
import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect
|
||||||
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||||
@ -15,16 +16,30 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|||||||
* Side effect that logs served tweet metrics to Scribe as client events.
|
* Side effect that logs served tweet metrics to Scribe as client events.
|
||||||
*/
|
*/
|
||||||
case class HomeScribeClientEventSideEffect(
|
case class HomeScribeClientEventSideEffect(
|
||||||
|
enableScribeClientEvents: Boolean,
|
||||||
override val logPipelinePublisher: EventPublisher[LogEvent],
|
override val logPipelinePublisher: EventPublisher[LogEvent],
|
||||||
injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier],
|
injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier],
|
||||||
adsCandidatePipelineIdentifier: CandidatePipelineIdentifier,
|
adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
|
||||||
whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
|
whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None,
|
||||||
) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] {
|
whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None)
|
||||||
|
extends ScribeClientEventSideEffect[PipelineQuery, Timeline]
|
||||||
|
with PipelineResultSideEffect.Conditionally[
|
||||||
|
PipelineQuery,
|
||||||
|
Timeline
|
||||||
|
] {
|
||||||
|
|
||||||
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent")
|
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent")
|
||||||
|
|
||||||
override val page = "timelinemixer"
|
override val page = "timelinemixer"
|
||||||
|
|
||||||
|
override def onlyIf(
|
||||||
|
query: PipelineQuery,
|
||||||
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
|
remainingCandidates: Seq[CandidateWithDetails],
|
||||||
|
droppedCandidates: Seq[CandidateWithDetails],
|
||||||
|
response: Timeline
|
||||||
|
): Boolean = enableScribeClientEvents
|
||||||
|
|
||||||
override def buildClientEvents(
|
override def buildClientEvents(
|
||||||
query: PipelineQuery,
|
query: PipelineQuery,
|
||||||
selectedCandidates: Seq[CandidateWithDetails],
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
@ -37,13 +52,15 @@ case class HomeScribeClientEventSideEffect(
|
|||||||
val sources = itemCandidates.groupBy(_.source)
|
val sources = itemCandidates.groupBy(_.source)
|
||||||
val injectedTweets =
|
val injectedTweets =
|
||||||
injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty))
|
injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty))
|
||||||
val promotedTweets = sources.getOrElse(adsCandidatePipelineIdentifier, Seq.empty)
|
val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
|
||||||
|
|
||||||
// WhoToFollow module is not required for all home-mixer products, e.g. list tweets timeline.
|
// WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline.
|
||||||
val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
|
val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
|
||||||
|
val whoToSubscribeUsers =
|
||||||
|
whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten
|
||||||
|
|
||||||
val servedEvents = ServedEventsBuilder
|
val servedEvents = ServedEventsBuilder
|
||||||
.build(query, injectedTweets, promotedTweets, whoToFollowUsers)
|
.build(query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers)
|
||||||
|
|
||||||
val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets)
|
val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets)
|
||||||
|
|
||||||
|
@ -0,0 +1,245 @@
|
|||||||
|
package com.twitter.home_mixer.functional_component.side_effect
|
||||||
|
|
||||||
|
import com.twitter.finagle.tracing.Trace
|
||||||
|
import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller
|
||||||
|
import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller
|
||||||
|
import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
|
||||||
|
import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
|
||||||
|
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||||
|
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
||||||
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
|
import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag
|
||||||
|
import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam
|
||||||
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
|
import com.twitter.inject.annotations.Flag
|
||||||
|
import com.twitter.logpipeline.client.common.EventPublisher
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||||
|
import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
|
||||||
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
||||||
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
|
||||||
|
import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect
|
||||||
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
||||||
|
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||||
|
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
||||||
|
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
|
||||||
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
|
import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
|
||||||
|
import com.twitter.util.Time
|
||||||
|
import javax.inject.Inject
|
||||||
|
import javax.inject.Singleton
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Side effect that logs home timeline served candidates to Scribe.
|
||||||
|
*/
|
||||||
|
@Singleton
|
||||||
|
class HomeScribeServedCandidatesSideEffect @Inject() (
|
||||||
|
@Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean,
|
||||||
|
scribeEventPublisher: EventPublisher[thrift.ServedEntry])
|
||||||
|
extends ScribeLogEventSideEffect[
|
||||||
|
thrift.ServedEntry,
|
||||||
|
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
||||||
|
Timeline
|
||||||
|
]
|
||||||
|
with PipelineResultSideEffect.Conditionally[
|
||||||
|
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
||||||
|
Timeline
|
||||||
|
] {
|
||||||
|
|
||||||
|
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates")
|
||||||
|
|
||||||
|
override def onlyIf(
|
||||||
|
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
||||||
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
|
remainingCandidates: Seq[CandidateWithDetails],
|
||||||
|
droppedCandidates: Seq[CandidateWithDetails],
|
||||||
|
response: Timeline
|
||||||
|
): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam)
|
||||||
|
|
||||||
|
override def buildLogEvents(
|
||||||
|
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
||||||
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
|
remainingCandidates: Seq[CandidateWithDetails],
|
||||||
|
droppedCandidates: Seq[CandidateWithDetails],
|
||||||
|
response: Timeline
|
||||||
|
): Seq[thrift.ServedEntry] = {
|
||||||
|
val timelineType = query.product match {
|
||||||
|
case FollowingProduct => thrift.TimelineType.HomeLatest
|
||||||
|
case ForYouProduct => thrift.TimelineType.Home
|
||||||
|
case SubscribedProduct => thrift.TimelineType.HomeSubscribed
|
||||||
|
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
||||||
|
}
|
||||||
|
val requestProvenance = query.deviceContext.map { deviceContext =>
|
||||||
|
deviceContext.requestContextValue match {
|
||||||
|
case RequestContext.Foreground => thrift.RequestProvenance.Foreground
|
||||||
|
case RequestContext.Launch => thrift.RequestProvenance.Launch
|
||||||
|
case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
|
||||||
|
case _ => thrift.RequestProvenance.Other
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val queryType = query.features.map { featureMap =>
|
||||||
|
if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
|
||||||
|
else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
|
||||||
|
else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
|
||||||
|
else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
|
||||||
|
else thrift.QueryType.Other
|
||||||
|
}
|
||||||
|
val requestInfo = thrift.RequestInfo(
|
||||||
|
requestTimeMs = query.queryTime.inMilliseconds,
|
||||||
|
traceId = Trace.id.traceId.toLong,
|
||||||
|
userId = query.getOptionalUserId,
|
||||||
|
clientAppId = query.clientContext.appId,
|
||||||
|
hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
|
||||||
|
parentId = Some(Trace.id.parentId.toLong),
|
||||||
|
spanId = Some(Trace.id.spanId.toLong),
|
||||||
|
timelineType = Some(timelineType),
|
||||||
|
ipAddress = query.clientContext.ipAddress,
|
||||||
|
userAgent = query.clientContext.userAgent,
|
||||||
|
queryType = queryType,
|
||||||
|
requestProvenance = requestProvenance,
|
||||||
|
languageCode = query.clientContext.languageCode,
|
||||||
|
countryCode = query.clientContext.countryCode,
|
||||||
|
requestEndTimeMs = Some(Time.now.inMilliseconds),
|
||||||
|
servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
|
||||||
|
requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None))
|
||||||
|
)
|
||||||
|
|
||||||
|
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
||||||
|
selectedCandidates.flatMap {
|
||||||
|
case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
|
||||||
|
Seq((item.candidateIdLong, item))
|
||||||
|
case module: ModuleCandidateWithDetails
|
||||||
|
if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
|
||||||
|
module.candidates.map(item => (item.candidateIdLong, item))
|
||||||
|
case _ => Seq.empty
|
||||||
|
}.toMap
|
||||||
|
|
||||||
|
val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
||||||
|
selectedCandidates.flatMap {
|
||||||
|
case module: ModuleCandidateWithDetails
|
||||||
|
if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
|
||||||
|
module.candidates.map { item =>
|
||||||
|
(item.candidateIdLong, item)
|
||||||
|
}
|
||||||
|
case _ => Seq.empty
|
||||||
|
}.toMap
|
||||||
|
|
||||||
|
response.instructions.zipWithIndex
|
||||||
|
.collect {
|
||||||
|
case (AddEntriesTimelineInstruction(entries), index) =>
|
||||||
|
entries.collect {
|
||||||
|
case entry: TweetItem if entry.promotedMetadata.isDefined =>
|
||||||
|
val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index)
|
||||||
|
Seq(
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = entry.id,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = entry.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.PromotedTweet,
|
||||||
|
sortIndex = entry.sortIndex,
|
||||||
|
verticalSize = Some(1),
|
||||||
|
displayType = Some(entry.displayType.toString),
|
||||||
|
details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
case entry: TweetItem =>
|
||||||
|
val candidate = tweetIdToItemCandidateMap(entry.id)
|
||||||
|
val tweetDetails = TweetDetailsMarshaller(entry, candidate)
|
||||||
|
Seq(
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = candidate.candidateIdLong,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = entry.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.Tweet,
|
||||||
|
sortIndex = entry.sortIndex,
|
||||||
|
verticalSize = Some(1),
|
||||||
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
||||||
|
displayType = Some(entry.displayType.toString),
|
||||||
|
details = Some(thrift.ItemDetails.TweetDetails(tweetDetails))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
case module: TimelineModule
|
||||||
|
if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
|
||||||
|
module.items.collect {
|
||||||
|
case ModuleItem(entry: UserItem, _, _) =>
|
||||||
|
val candidate = userIdToItemCandidateMap(entry.id)
|
||||||
|
val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate)
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = entry.id,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = module.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.WhoToFollowModule,
|
||||||
|
sortIndex = module.sortIndex,
|
||||||
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
||||||
|
displayType = Some(entry.displayType.toString),
|
||||||
|
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case module: TimelineModule
|
||||||
|
if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
|
||||||
|
module.items.collect {
|
||||||
|
case ModuleItem(entry: UserItem, _, _) =>
|
||||||
|
val candidate = userIdToItemCandidateMap(entry.id)
|
||||||
|
val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate)
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = entry.id,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = module.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.WhoToSubscribeModule,
|
||||||
|
sortIndex = module.sortIndex,
|
||||||
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
||||||
|
displayType = Some(entry.displayType.toString),
|
||||||
|
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case module: TimelineModule
|
||||||
|
if module.sortIndex.isDefined && module.items.headOption.exists(
|
||||||
|
_.item.isInstanceOf[TweetItem]) =>
|
||||||
|
module.items.collect {
|
||||||
|
case ModuleItem(entry: TweetItem, _, _) =>
|
||||||
|
val candidate = tweetIdToItemCandidateMap(entry.id)
|
||||||
|
thrift.EntryInfo(
|
||||||
|
id = entry.id,
|
||||||
|
position = index.shortValue(),
|
||||||
|
entryId = module.entryIdentifier,
|
||||||
|
entryType = thrift.EntryType.ConversationModule,
|
||||||
|
sortIndex = module.sortIndex,
|
||||||
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
||||||
|
displayType = Some(entry.displayType.toString)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case _ => Seq.empty
|
||||||
|
}.flatten
|
||||||
|
// Other instructions
|
||||||
|
case _ => Seq.empty[thrift.EntryInfo]
|
||||||
|
}.flatten.map { entryInfo =>
|
||||||
|
thrift.ServedEntry(
|
||||||
|
entry = Some(entryInfo),
|
||||||
|
request = requestInfo
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] =
|
||||||
|
scribeEventPublisher
|
||||||
|
|
||||||
|
override val alerts = Seq(
|
||||||
|
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
|
||||||
|
)
|
||||||
|
}
|
@ -1,212 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.side_effect
|
|
||||||
|
|
||||||
import com.twitter.finagle.tracing.Trace
|
|
||||||
import com.twitter.home_mixer.marshaller.timeline_logging.ConversationEntryMarshaller
|
|
||||||
import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetEntryMarshaller
|
|
||||||
import com.twitter.home_mixer.marshaller.timeline_logging.TweetEntryMarshaller
|
|
||||||
import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowEntryMarshaller
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
|
|
||||||
import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
|
|
||||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
|
||||||
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
|
||||||
import com.twitter.logpipeline.client.common.EventPublisher
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
|
|
||||||
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
|
|
||||||
import com.twitter.util.Time
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Side effect that logs home timeline served entries to Scribe.
|
|
||||||
*/
|
|
||||||
@Singleton
|
|
||||||
class HomeScribeServedEntriesSideEffect @Inject() (
|
|
||||||
scribeEventPublisher: EventPublisher[thrift.Timeline])
|
|
||||||
extends PipelineResultSideEffect[
|
|
||||||
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
||||||
Timeline
|
|
||||||
] {
|
|
||||||
|
|
||||||
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedEntries")
|
|
||||||
|
|
||||||
final override def apply(
|
|
||||||
inputs: PipelineResultSideEffect.Inputs[
|
|
||||||
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
||||||
Timeline
|
|
||||||
]
|
|
||||||
): Stitch[Unit] = {
|
|
||||||
val timelineThrift = buildTimeline(inputs)
|
|
||||||
Stitch.callFuture(scribeEventPublisher.publish(timelineThrift)).unit
|
|
||||||
}
|
|
||||||
|
|
||||||
def buildTimeline(
|
|
||||||
inputs: PipelineResultSideEffect.Inputs[
|
|
||||||
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
||||||
Timeline
|
|
||||||
]
|
|
||||||
): thrift.Timeline = {
|
|
||||||
val timelineType = inputs.query.product match {
|
|
||||||
case FollowingProduct => thrift.TimelineType.HomeLatest
|
|
||||||
case ForYouProduct => thrift.TimelineType.Home
|
|
||||||
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
|
||||||
}
|
|
||||||
val requestProvenance = inputs.query.deviceContext.map { deviceContext =>
|
|
||||||
deviceContext.requestContextValue match {
|
|
||||||
case RequestContext.Foreground => thrift.RequestProvenance.Foreground
|
|
||||||
case RequestContext.Launch => thrift.RequestProvenance.Launch
|
|
||||||
case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
|
|
||||||
case _ => thrift.RequestProvenance.Other
|
|
||||||
}
|
|
||||||
}
|
|
||||||
val queryType = inputs.query.features.map { featureMap =>
|
|
||||||
if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
|
|
||||||
else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
|
|
||||||
else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
|
|
||||||
else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
|
|
||||||
else thrift.QueryType.Other
|
|
||||||
}
|
|
||||||
|
|
||||||
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
|
||||||
inputs.selectedCandidates.flatMap {
|
|
||||||
case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
|
|
||||||
Seq((item.candidateIdLong, item))
|
|
||||||
case module: ModuleCandidateWithDetails
|
|
||||||
if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
|
|
||||||
module.candidates.map(item => (item.candidateIdLong, item))
|
|
||||||
case _ => Seq.empty
|
|
||||||
}.toMap
|
|
||||||
|
|
||||||
val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
|
||||||
inputs.selectedCandidates.flatMap {
|
|
||||||
case module: ModuleCandidateWithDetails
|
|
||||||
if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
|
|
||||||
module.candidates.map { item =>
|
|
||||||
(item.candidateIdLong, item)
|
|
||||||
}
|
|
||||||
case _ => Seq.empty
|
|
||||||
}.toMap
|
|
||||||
|
|
||||||
val timelineEntries = inputs.response.instructions.zipWithIndex.collect {
|
|
||||||
case (AddEntriesTimelineInstruction(entries), index) =>
|
|
||||||
entries.collect {
|
|
||||||
case entry: TweetItem if entry.promotedMetadata.isDefined =>
|
|
||||||
val promotedTweetEntry = PromotedTweetEntryMarshaller(entry, index)
|
|
||||||
Seq(
|
|
||||||
thrift.TimelineEntry(
|
|
||||||
content = thrift.Content.PromotedTweetEntry(promotedTweetEntry),
|
|
||||||
position = index.shortValue(),
|
|
||||||
entryId = entry.entryIdentifier,
|
|
||||||
entryType = thrift.EntryType.PromotedTweet,
|
|
||||||
sortIndex = entry.sortIndex,
|
|
||||||
verticalSize = Some(1)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case entry: TweetItem =>
|
|
||||||
val candidate = tweetIdToItemCandidateMap(entry.id)
|
|
||||||
val tweetEntry = TweetEntryMarshaller(entry, candidate)
|
|
||||||
Seq(
|
|
||||||
thrift.TimelineEntry(
|
|
||||||
content = thrift.Content.TweetEntry(tweetEntry),
|
|
||||||
position = index.shortValue(),
|
|
||||||
entryId = entry.entryIdentifier,
|
|
||||||
entryType = thrift.EntryType.Tweet,
|
|
||||||
sortIndex = entry.sortIndex,
|
|
||||||
verticalSize = Some(1)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case module: TimelineModule
|
|
||||||
if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
|
|
||||||
val whoToFollowEntries = module.items.collect {
|
|
||||||
case ModuleItem(entry: UserItem, _, _) =>
|
|
||||||
val candidate = userIdToItemCandidateMap(entry.id)
|
|
||||||
val whoToFollowEntry = WhoToFollowEntryMarshaller(entry, candidate)
|
|
||||||
thrift.AtomicEntry.WtfEntry(whoToFollowEntry)
|
|
||||||
}
|
|
||||||
Seq(
|
|
||||||
thrift.TimelineEntry(
|
|
||||||
content = thrift.Content.Entries(whoToFollowEntries),
|
|
||||||
position = index.shortValue(),
|
|
||||||
entryId = module.entryIdentifier,
|
|
||||||
entryType = thrift.EntryType.WhoToFollowModule,
|
|
||||||
sortIndex = module.sortIndex
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case module: TimelineModule
|
|
||||||
if module.sortIndex.isDefined && module.items.headOption.exists(
|
|
||||||
_.item.isInstanceOf[TweetItem]) =>
|
|
||||||
val conversationTweetEntries = module.items.collect {
|
|
||||||
case ModuleItem(entry: TweetItem, _, _) =>
|
|
||||||
val candidate = tweetIdToItemCandidateMap(entry.id)
|
|
||||||
val conversationEntry = ConversationEntryMarshaller(entry, candidate)
|
|
||||||
thrift.AtomicEntry.ConversationEntry(conversationEntry)
|
|
||||||
}
|
|
||||||
Seq(
|
|
||||||
thrift.TimelineEntry(
|
|
||||||
content = thrift.Content.Entries(conversationTweetEntries),
|
|
||||||
position = index.shortValue(),
|
|
||||||
entryId = module.entryIdentifier,
|
|
||||||
entryType = thrift.EntryType.ConversationModule,
|
|
||||||
sortIndex = module.sortIndex
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case _ => Seq.empty
|
|
||||||
}.flatten
|
|
||||||
// Other instructions
|
|
||||||
case _ => Seq.empty[thrift.TimelineEntry]
|
|
||||||
}.flatten
|
|
||||||
|
|
||||||
thrift.Timeline(
|
|
||||||
timelineEntries = timelineEntries,
|
|
||||||
requestTimeMs = inputs.query.queryTime.inMilliseconds,
|
|
||||||
traceId = Trace.id.traceId.toLong,
|
|
||||||
userId = inputs.query.getOptionalUserId,
|
|
||||||
clientAppId = inputs.query.clientContext.appId,
|
|
||||||
sourceJobInstance = None,
|
|
||||||
hasDarkRequest = inputs.query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
|
|
||||||
parentId = Some(Trace.id.parentId.toLong),
|
|
||||||
spanId = Some(Trace.id.spanId.toLong),
|
|
||||||
timelineType = Some(timelineType),
|
|
||||||
ipAddress = inputs.query.clientContext.ipAddress,
|
|
||||||
userAgent = inputs.query.clientContext.userAgent,
|
|
||||||
queryType = queryType,
|
|
||||||
requestProvenance = requestProvenance,
|
|
||||||
sessionId = None,
|
|
||||||
timeZone = None,
|
|
||||||
browserNotificationPermission = None,
|
|
||||||
lastNonePollingTimeMs = None,
|
|
||||||
languageCode = inputs.query.clientContext.languageCode,
|
|
||||||
countryCode = inputs.query.clientContext.countryCode,
|
|
||||||
requestEndTimeMs = Some(Time.now.inMilliseconds),
|
|
||||||
servedRequestId = inputs.query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
|
|
||||||
requestJoinId = inputs.query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)),
|
|
||||||
requestSeenTweetIds = inputs.query.seenTweetIds
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
override val alerts = Seq(
|
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
|
|
||||||
)
|
|
||||||
}
|
|
@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.side_effect
|
|||||||
import com.twitter.eventbus.client.EventBusPublisher
|
import com.twitter.eventbus.client.EventBusPublisher
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
@ -22,6 +23,7 @@ import javax.inject.Singleton
|
|||||||
object PublishClientSentImpressionsEventBusSideEffect {
|
object PublishClientSentImpressionsEventBusSideEffect {
|
||||||
val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline))
|
val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline))
|
||||||
val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline))
|
val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline))
|
||||||
|
val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed))
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -56,6 +58,7 @@ class PublishClientSentImpressionsEventBusSideEffect @Inject() (
|
|||||||
val surfaceArea = query.product match {
|
val surfaceArea = query.product match {
|
||||||
case ForYouProduct => HomeSurfaceArea
|
case ForYouProduct => HomeSurfaceArea
|
||||||
case FollowingProduct => HomeLatestSurfaceArea
|
case FollowingProduct => HomeLatestSurfaceArea
|
||||||
|
case SubscribedProduct => HomeSubscribedSurfaceArea
|
||||||
case _ => None
|
case _ => None
|
||||||
}
|
}
|
||||||
query.seenTweetIds.map { seenTweetIds =>
|
query.seenTweetIds.map { seenTweetIds =>
|
||||||
|
@ -1,39 +1,46 @@
|
|||||||
package com.twitter.home_mixer.functional_component.side_effect
|
package com.twitter.home_mixer.functional_component.side_effect
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
|
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
|
||||||
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
||||||
|
import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||||
import com.twitter.stitch.Stitch
|
import com.twitter.stitch.Stitch
|
||||||
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
|
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
|
||||||
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
|
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBloomFilter)
|
class PublishImpressionBloomFilterSideEffect @Inject() (
|
||||||
extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, Timeline]
|
bloomFilterClient: ManhattanStoreClient[
|
||||||
with PipelineResultSideEffect.Conditionally[PipelineQuery with HasSeenTweetIds, Timeline] {
|
blm.ImpressionBloomFilterKey,
|
||||||
|
blm.ImpressionBloomFilterSeq
|
||||||
private val SurfaceArea = t.SurfaceArea.HomeTimeline
|
]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling]
|
||||||
|
with PipelineResultSideEffect.Conditionally[
|
||||||
|
PipelineQuery with HasSeenTweetIds,
|
||||||
|
HasMarshalling
|
||||||
|
] {
|
||||||
|
|
||||||
override val identifier: SideEffectIdentifier =
|
override val identifier: SideEffectIdentifier =
|
||||||
SideEffectIdentifier("UpdateImpressionBloomFilter")
|
SideEffectIdentifier("PublishImpressionBloomFilter")
|
||||||
|
|
||||||
|
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
|
||||||
|
|
||||||
override def onlyIf(
|
override def onlyIf(
|
||||||
query: PipelineQuery with HasSeenTweetIds,
|
query: PipelineQuery with HasSeenTweetIds,
|
||||||
selectedCandidates: Seq[CandidateWithDetails],
|
selectedCandidates: Seq[CandidateWithDetails],
|
||||||
remainingCandidates: Seq[CandidateWithDetails],
|
remainingCandidates: Seq[CandidateWithDetails],
|
||||||
droppedCandidates: Seq[CandidateWithDetails],
|
droppedCandidates: Seq[CandidateWithDetails],
|
||||||
response: Timeline
|
response: HasMarshalling
|
||||||
): Boolean = query.seenTweetIds.exists(_.nonEmpty)
|
): Boolean =
|
||||||
|
query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty)
|
||||||
|
|
||||||
def buildEvents(query: PipelineQuery): Option[t.ImpressionBloomFilterSeq] = {
|
def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = {
|
||||||
query.features.flatMap { featureMap =>
|
query.features.flatMap { featureMap =>
|
||||||
val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature)
|
val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature)
|
||||||
if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq)
|
if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq)
|
||||||
@ -42,19 +49,17 @@ class UpdateImpressionBloomFilterSideEffect @Inject() (bloomFilter: ImpressionBl
|
|||||||
}
|
}
|
||||||
|
|
||||||
override def apply(
|
override def apply(
|
||||||
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, Timeline]
|
inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling]
|
||||||
): Stitch[Unit] = {
|
): Stitch[Unit] = {
|
||||||
buildEvents(inputs.query)
|
buildEvents(inputs.query)
|
||||||
.map { updatedBloomFilter =>
|
.map { updatedBloomFilterSeq =>
|
||||||
bloomFilter.writeBloomFilterSeq(
|
bloomFilterClient.write(
|
||||||
userId = inputs.query.getRequiredUserId,
|
blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea),
|
||||||
surfaceArea = SurfaceArea,
|
updatedBloomFilterSeq)
|
||||||
impressionBloomFilterSeq = updatedBloomFilter)
|
|
||||||
}.getOrElse(Stitch.Unit)
|
}.getOrElse(Stitch.Unit)
|
||||||
}
|
}
|
||||||
|
|
||||||
override val alerts = Seq(
|
override val alerts = Seq(
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8),
|
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8)
|
||||||
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(30.millis)
|
|
||||||
)
|
)
|
||||||
}
|
}
|
@ -1,80 +0,0 @@
|
|||||||
package com.twitter.home_mixer.functional_component.side_effect
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
|
||||||
import com.twitter.home_mixer.param.HomeGlobalParams.AuthorListForStatsParam
|
|
||||||
import com.twitter.home_mixer.util.CandidatesUtil
|
|
||||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
|
||||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import javax.inject.Inject
|
|
||||||
import javax.inject.Singleton
|
|
||||||
|
|
||||||
@Singleton
|
|
||||||
class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver)
|
|
||||||
extends PipelineResultSideEffect[PipelineQuery, Timeline] {
|
|
||||||
|
|
||||||
override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats")
|
|
||||||
|
|
||||||
private val baseStatsReceiver = statsReceiver.scope(identifier.toString)
|
|
||||||
private val authorStatsReceiver = baseStatsReceiver.scope("Author")
|
|
||||||
private val candidateSourceStatsReceiver = baseStatsReceiver.scope("CandidateSource")
|
|
||||||
private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance")
|
|
||||||
private val inNetworkStatsCounter = contentBalanceStatsReceiver.counter("InNetwork")
|
|
||||||
private val outOfNetworkStatsCounter = contentBalanceStatsReceiver.counter("OutOfNetwork")
|
|
||||||
|
|
||||||
override def apply(
|
|
||||||
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
|
|
||||||
): Stitch[Unit] = {
|
|
||||||
val tweetCandidates = CandidatesUtil
|
|
||||||
.getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]())
|
|
||||||
|
|
||||||
recordAuthorStats(tweetCandidates, inputs.query.params(AuthorListForStatsParam))
|
|
||||||
recordCandidateSourceStats(tweetCandidates)
|
|
||||||
recordContentBalanceStats(tweetCandidates)
|
|
||||||
Stitch.Unit
|
|
||||||
}
|
|
||||||
|
|
||||||
def recordAuthorStats(candidates: Seq[CandidateWithDetails], authors: Set[Long]): Unit = {
|
|
||||||
candidates
|
|
||||||
.filter { candidate =>
|
|
||||||
candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) &&
|
|
||||||
// Only include original tweets
|
|
||||||
(!candidate.features.getOrElse(IsRetweetFeature, false)) &&
|
|
||||||
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
|
|
||||||
}
|
|
||||||
.groupBy { candidate =>
|
|
||||||
(getCandidateSourceId(candidate), candidate.features.get(AuthorIdFeature).get)
|
|
||||||
}
|
|
||||||
.foreach {
|
|
||||||
case ((candidateSourceId, authorId), authorCandidates) =>
|
|
||||||
authorStatsReceiver
|
|
||||||
.scope(authorId.toString).counter(candidateSourceId).incr(authorCandidates.size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def recordCandidateSourceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
|
|
||||||
candidates.groupBy(getCandidateSourceId).foreach {
|
|
||||||
case (candidateSourceId, candidateSourceCandidates) =>
|
|
||||||
candidateSourceStatsReceiver.counter(candidateSourceId).incr(candidateSourceCandidates.size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def recordContentBalanceStats(candidates: Seq[ItemCandidateWithDetails]): Unit = {
|
|
||||||
val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true))
|
|
||||||
inNetworkStatsCounter.incr(in.size)
|
|
||||||
outOfNetworkStatsCounter.incr(oon.size)
|
|
||||||
}
|
|
||||||
|
|
||||||
private def getCandidateSourceId(candidate: CandidateWithDetails): String =
|
|
||||||
candidate.features.getOrElse(CandidateSourceIdFeature, None).map(_.name).getOrElse("None")
|
|
||||||
}
|
|
@ -3,8 +3,10 @@ package com.twitter.home_mixer.functional_component.side_effect
|
|||||||
import com.twitter.home_mixer.model.HomeFeatures._
|
import com.twitter.home_mixer.model.HomeFeatures._
|
||||||
import com.twitter.home_mixer.model.request.FollowingProduct
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
||||||
import com.twitter.home_mixer.model.request.ForYouProduct
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature
|
||||||
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
||||||
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
||||||
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
|
||||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||||
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
||||||
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
||||||
@ -97,12 +99,14 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
|
|||||||
val entries = inputs.response.instructions.collect {
|
val entries = inputs.response.instructions.collect {
|
||||||
case AddEntriesTimelineInstruction(entries) =>
|
case AddEntriesTimelineInstruction(entries) =>
|
||||||
entries.collect {
|
entries.collect {
|
||||||
// includes both tweets and promoted tweets
|
// includes tweets, tweet previews, and promoted tweets
|
||||||
case entry: TweetItem if entry.sortIndex.isDefined =>
|
case entry: TweetItem if entry.sortIndex.isDefined => {
|
||||||
Seq(
|
Seq(
|
||||||
buildTweetEntryWithItemIds(
|
buildTweetEntryWithItemIds(
|
||||||
tweetIdToItemCandidateMap(entry.id),
|
tweetIdToItemCandidateMap(entry.id),
|
||||||
entry.sortIndex.get))
|
entry.sortIndex.get
|
||||||
|
))
|
||||||
|
}
|
||||||
// tweet conversation modules are flattened to individual tweets in the persistence store
|
// tweet conversation modules are flattened to individual tweets in the persistence store
|
||||||
case module: TimelineModule
|
case module: TimelineModule
|
||||||
if module.sortIndex.isDefined && module.items.headOption.exists(
|
if module.sortIndex.isDefined && module.items.headOption.exists(
|
||||||
@ -125,6 +129,19 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
|
|||||||
size = module.items.size.toShort,
|
size = module.items.size.toShort,
|
||||||
itemIds = Some(userIds)
|
itemIds = Some(userIds)
|
||||||
))
|
))
|
||||||
|
case module: TimelineModule
|
||||||
|
if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
|
||||||
|
val userIds = module.items
|
||||||
|
.map(item =>
|
||||||
|
UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId =
|
||||||
|
Some(item.item.id.asInstanceOf[Long])))
|
||||||
|
Seq(
|
||||||
|
EntryWithItemIds(
|
||||||
|
entityIdType = EntityIdType.WhoToSubscribe,
|
||||||
|
sortIndex = module.sortIndex.get,
|
||||||
|
size = module.items.size.toShort,
|
||||||
|
itemIds = Some(userIds)
|
||||||
|
))
|
||||||
}.flatten
|
}.flatten
|
||||||
case ShowCoverInstruction(cover) =>
|
case ShowCoverInstruction(cover) =>
|
||||||
Seq(
|
Seq(
|
||||||
@ -216,8 +233,11 @@ class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
|
|||||||
userId = None
|
userId = None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false)
|
||||||
|
val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet
|
||||||
|
|
||||||
EntryWithItemIds(
|
EntryWithItemIds(
|
||||||
entityIdType = EntityIdType.Tweet,
|
entityIdType = entityType,
|
||||||
sortIndex = sortIndex,
|
sortIndex = sortIndex,
|
||||||
size = 1.toShort,
|
size = 1.toShort,
|
||||||
itemIds = Some(Seq(itemIds))
|
itemIds = Some(Seq(itemIds))
|
||||||
|
@ -4,14 +4,12 @@ scala_library(
|
|||||||
strict_deps = True,
|
strict_deps = True,
|
||||||
tags = ["bazel-compatible"],
|
tags = ["bazel-compatible"],
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||||
],
|
],
|
||||||
exports = [
|
exports = [
|
||||||
"dspbidder/thrift/src/main/thrift/com/twitter/dspbidder/commons:thrift-scala",
|
|
||||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
||||||
|
@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProductContext
|
|||||||
import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext
|
import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext
|
||||||
import com.twitter.home_mixer.model.request.ListTweetsProductContext
|
import com.twitter.home_mixer.model.request.ListTweetsProductContext
|
||||||
import com.twitter.home_mixer.model.request.ScoredTweetsProductContext
|
import com.twitter.home_mixer.model.request.ScoredTweetsProductContext
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProductContext
|
||||||
import com.twitter.home_mixer.{thriftscala => t}
|
import com.twitter.home_mixer.{thriftscala => t}
|
||||||
import com.twitter.product_mixer.core.model.marshalling.request.ProductContext
|
import com.twitter.product_mixer.core.model.marshalling.request.ProductContext
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
@ -26,15 +26,17 @@ class HomeMixerProductContextUnmarshaller @Inject() (
|
|||||||
ForYouProductContext(
|
ForYouProductContext(
|
||||||
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
||||||
seenTweetIds = p.seenTweetIds,
|
seenTweetIds = p.seenTweetIds,
|
||||||
dspClientContext = p.dspClientContext
|
dspClientContext = p.dspClientContext,
|
||||||
|
pushToHomeTweetId = p.pushToHomeTweetId
|
||||||
)
|
)
|
||||||
case t.ProductContext.Realtime(p) =>
|
case t.ProductContext.ListManagement(p) =>
|
||||||
throw new UnsupportedOperationException(s"This product is no longer used")
|
throw new UnsupportedOperationException(s"This product is no longer used")
|
||||||
case t.ProductContext.ScoredTweets(p) =>
|
case t.ProductContext.ScoredTweets(p) =>
|
||||||
ScoredTweetsProductContext(
|
ScoredTweetsProductContext(
|
||||||
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
||||||
seenTweetIds = p.seenTweetIds,
|
seenTweetIds = p.seenTweetIds,
|
||||||
servedTweetIds = p.servedTweetIds
|
servedTweetIds = p.servedTweetIds,
|
||||||
|
backfillTweetIds = p.backfillTweetIds
|
||||||
)
|
)
|
||||||
case t.ProductContext.ListTweets(p) =>
|
case t.ProductContext.ListTweets(p) =>
|
||||||
ListTweetsProductContext(
|
ListTweetsProductContext(
|
||||||
@ -46,7 +48,13 @@ class HomeMixerProductContextUnmarshaller @Inject() (
|
|||||||
ListRecommendedUsersProductContext(
|
ListRecommendedUsersProductContext(
|
||||||
listId = p.listId,
|
listId = p.listId,
|
||||||
selectedUserIds = p.selectedUserIds,
|
selectedUserIds = p.selectedUserIds,
|
||||||
excludedUserIds = p.excludedUserIds
|
excludedUserIds = p.excludedUserIds,
|
||||||
|
listName = p.listName
|
||||||
|
)
|
||||||
|
case t.ProductContext.Subscribed(p) =>
|
||||||
|
SubscribedProductContext(
|
||||||
|
deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)),
|
||||||
|
seenTweetIds = p.seenTweetIds,
|
||||||
)
|
)
|
||||||
case t.ProductContext.UnknownUnionField(field) =>
|
case t.ProductContext.UnknownUnionField(field) =>
|
||||||
throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}")
|
throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}")
|
||||||
|
@ -5,9 +5,9 @@ import com.twitter.home_mixer.model.request.ForYouProduct
|
|||||||
import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct
|
import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct
|
||||||
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
||||||
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
||||||
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
||||||
import com.twitter.home_mixer.{thriftscala => t}
|
import com.twitter.home_mixer.{thriftscala => t}
|
||||||
import com.twitter.product_mixer.core.model.marshalling.request.Product
|
import com.twitter.product_mixer.core.model.marshalling.request.Product
|
||||||
|
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import javax.inject.Singleton
|
import javax.inject.Singleton
|
||||||
|
|
||||||
@ -17,11 +17,12 @@ class HomeMixerProductUnmarshaller @Inject() () {
|
|||||||
def apply(product: t.Product): Product = product match {
|
def apply(product: t.Product): Product = product match {
|
||||||
case t.Product.Following => FollowingProduct
|
case t.Product.Following => FollowingProduct
|
||||||
case t.Product.ForYou => ForYouProduct
|
case t.Product.ForYou => ForYouProduct
|
||||||
case t.Product.Realtime =>
|
case t.Product.ListManagement =>
|
||||||
throw new UnsupportedOperationException(s"This product is no longer used")
|
throw new UnsupportedOperationException(s"This product is no longer used")
|
||||||
case t.Product.ScoredTweets => ScoredTweetsProduct
|
case t.Product.ScoredTweets => ScoredTweetsProduct
|
||||||
case t.Product.ListTweets => ListTweetsProduct
|
case t.Product.ListTweets => ListTweetsProduct
|
||||||
case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct
|
case t.Product.ListRecommendedUsers => ListRecommendedUsersProduct
|
||||||
|
case t.Product.Subscribed => SubscribedProduct
|
||||||
case t.Product.EnumUnknownProduct(value) =>
|
case t.Product.EnumUnknownProduct(value) =>
|
||||||
throw new UnsupportedOperationException(s"Unknown product: $value")
|
throw new UnsupportedOperationException(s"Unknown product: $value")
|
||||||
}
|
}
|
||||||
|
@ -1,16 +0,0 @@
|
|||||||
package com.twitter.home_mixer.marshaller.timeline_logging
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
|
||||||
|
|
||||||
object ConversationEntryMarshaller {
|
|
||||||
|
|
||||||
def apply(entry: TweetItem, candidate: ItemCandidateWithDetails): thriftlog.ConversationEntry =
|
|
||||||
thriftlog.ConversationEntry(
|
|
||||||
displayedTweetId = entry.id,
|
|
||||||
displayType = Some(entry.displayType.toString),
|
|
||||||
score = candidate.features.getOrElse(ScoreFeature, None)
|
|
||||||
)
|
|
||||||
}
|
|
@ -3,15 +3,13 @@ package com.twitter.home_mixer.marshaller.timeline_logging
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
||||||
|
|
||||||
object PromotedTweetEntryMarshaller {
|
object PromotedTweetDetailsMarshaller {
|
||||||
|
|
||||||
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetEntry = {
|
def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetDetails = {
|
||||||
thriftlog.PromotedTweetEntry(
|
thriftlog.PromotedTweetDetails(
|
||||||
id = entry.id,
|
advertiserId = Some(entry.promotedMetadata.map(_.advertiserId).getOrElse(0L)),
|
||||||
advertiserId = entry.promotedMetadata.map(_.advertiserId).getOrElse(0L),
|
insertPosition = Some(position),
|
||||||
insertPosition = position,
|
impressionId = entry.promotedMetadata.flatMap(_.impressionString)
|
||||||
impressionId = entry.promotedMetadata.flatMap(_.impressionString),
|
|
||||||
displayType = Some(entry.displayType.toString)
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -0,0 +1,47 @@
|
|||||||
|
package com.twitter.home_mixer.marshaller.timeline_logging
|
||||||
|
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||||
|
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||||
|
import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation
|
||||||
|
import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation
|
||||||
|
import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller
|
||||||
|
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ConversationGeneralContextType
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext
|
||||||
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext
|
||||||
|
import com.twitter.timelines.service.{thriftscala => tst}
|
||||||
|
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
||||||
|
|
||||||
|
object TweetDetailsMarshaller {
|
||||||
|
|
||||||
|
private val generalContextTypeMarshaller = new GeneralContextTypeMarshaller()
|
||||||
|
|
||||||
|
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetDetails = {
|
||||||
|
val socialContext = candidate.presentation.flatMap {
|
||||||
|
case _ @UrtItemPresentation(timelineItem: TweetItem, _) => timelineItem.socialContext
|
||||||
|
case _ @UrtModulePresentation(timelineModule) =>
|
||||||
|
timelineModule.items.head.item match {
|
||||||
|
case timelineItem: TweetItem => timelineItem.socialContext
|
||||||
|
case _ => Some(ConversationGeneralContextType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val socialContextType = socialContext match {
|
||||||
|
case Some(GeneralContext(contextType, _, _, _, _)) =>
|
||||||
|
Some(generalContextTypeMarshaller(contextType).value.toShort)
|
||||||
|
case Some(TopicContext(_, _)) => Some(tst.ContextType.Topic.value.toShort)
|
||||||
|
case _ => None
|
||||||
|
}
|
||||||
|
|
||||||
|
thriftlog.TweetDetails(
|
||||||
|
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
|
||||||
|
socialContextType = socialContextType,
|
||||||
|
suggestType = candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name),
|
||||||
|
authorId = candidate.features.getOrElse(AuthorIdFeature, None),
|
||||||
|
sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
@ -1,29 +0,0 @@
|
|||||||
package com.twitter.home_mixer.marshaller.timeline_logging
|
|
||||||
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SocialContextFeature
|
|
||||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
|
||||||
import com.twitter.timelines.service.{thriftscala => tst}
|
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
|
||||||
|
|
||||||
object TweetEntryMarshaller {
|
|
||||||
|
|
||||||
def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetEntry = {
|
|
||||||
val socialContextType = candidate.features.getOrElse(SocialContextFeature, None) match {
|
|
||||||
case Some(tst.SocialContext.GeneralContext(tst.GeneralContext(contextType, _, _, _, _))) =>
|
|
||||||
Some(contextType.value.toShort)
|
|
||||||
case Some(tst.SocialContext.TopicContext(_)) =>
|
|
||||||
Some(tst.ContextType.Topic.value.toShort)
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
thriftlog.TweetEntry(
|
|
||||||
id = candidate.candidateIdLong,
|
|
||||||
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
|
|
||||||
displayType = Some(entry.displayType.toString),
|
|
||||||
score = candidate.features.getOrElse(ScoreFeature, None),
|
|
||||||
socialContextType = socialContextType
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,17 +1,13 @@
|
|||||||
package com.twitter.home_mixer.marshaller.timeline_logging
|
package com.twitter.home_mixer.marshaller.timeline_logging
|
||||||
|
|
||||||
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.ScoreFeature
|
|
||||||
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
||||||
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
|
||||||
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog}
|
||||||
|
|
||||||
object WhoToFollowEntryMarshaller {
|
object WhoToFollowDetailsMarshaller {
|
||||||
|
|
||||||
def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowEntry =
|
def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowDetails =
|
||||||
thriftlog.WhoToFollowEntry(
|
thriftlog.WhoToFollowDetails(
|
||||||
userId = entry.id,
|
|
||||||
displayType = Some(entry.displayType.toString),
|
|
||||||
score = candidate.features.getOrElse(ScoreFeature, None),
|
|
||||||
enableReactiveBlending = entry.enableReactiveBlending,
|
enableReactiveBlending = entry.enableReactiveBlending,
|
||||||
impressionId = entry.promotedMetadata.flatMap(_.impressionString),
|
impressionId = entry.promotedMetadata.flatMap(_.impressionString),
|
||||||
advertiserId = entry.promotedMetadata.map(_.advertiserId)
|
advertiserId = entry.promotedMetadata.map(_.advertiserId)
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user