Compare commits

..

1 Commits

Author SHA1 Message Date
Yan Yan
7d8e377f55
Merge b1f865b393 into fb54d8b549 2023-05-22 17:38:32 -05:00
393 changed files with 3850 additions and 9334 deletions

View File

@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer) - ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
- Fetch Tweet Candidates - Fetch Tweet Candidates
- ScoredTweetsInNetworkCandidatePipelineConfig - ScoredTweetsInNetworkCandidatePipelineConfig
- ScoredTweetsTweetMixerCandidatePipelineConfig - ScoredTweetsCrMixerCandidatePipelineConfig
- ScoredTweetsUtegCandidatePipelineConfig - ScoredTweetsUtegCandidatePipelineConfig
- ScoredTweetsFrsCandidatePipelineConfig - ScoredTweetsFrsCandidatePipelineConfig
- Feature Hydration and Scoring - Feature Hydration and Scoring
@ -99,3 +99,4 @@ Timeline tabs powered by Home Mixer.
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service) - ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules) - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
- ListTweetsAdsCandidatePipelineConfig (fetch ads) - ListTweetsAdsCandidatePipelineConfig (fetch ads)

View File

@ -21,7 +21,6 @@ scala_library(
"finatra/inject/inject-utils/src/main/scala", "finatra/inject/inject-utils/src/main/scala",
"home-mixer/server/src/main/resources", "home-mixer/server/src/main/resources",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller", "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
@ -32,10 +31,6 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala", "src/thrift/com/twitter/timelines/render:thrift-scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"stringcenter/client", "stringcenter/client",
"stringcenter/client/src/main/java", "stringcenter/client/src/main/java",
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client", "stringcenter/client/src/main/scala/com/twitter/stringcenter/client",

View File

@ -2,7 +2,7 @@ package com.twitter.home_mixer
import com.twitter.finatra.http.routing.HttpWarmup import com.twitter.finatra.http.routing.HttpWarmup
import com.twitter.finatra.httpclient.RequestBuilder._ import com.twitter.finatra.httpclient.RequestBuilder._
import com.twitter.util.logging.Logging import com.twitter.inject.Logging
import com.twitter.inject.utils.Handler import com.twitter.inject.utils.Handler
import com.twitter.util.Try import com.twitter.util.Try
import javax.inject.Inject import javax.inject.Inject

View File

@ -12,63 +12,57 @@ import com.twitter.finatra.thrift.ThriftServer
import com.twitter.finatra.thrift.filters._ import com.twitter.finatra.thrift.filters._
import com.twitter.finatra.thrift.routing.ThriftRouter import com.twitter.finatra.thrift.routing.ThriftRouter
import com.twitter.home_mixer.controller.HomeThriftController import com.twitter.home_mixer.controller.HomeThriftController
import com.twitter.home_mixer.federated.HomeMixerColumn
import com.twitter.home_mixer.module._ import com.twitter.home_mixer.module._
import com.twitter.home_mixer.param.GlobalParamConfigModule import com.twitter.home_mixer.param.GlobalParamConfigModule
import com.twitter.home_mixer.product.HomeMixerProductModule import com.twitter.home_mixer.product.HomeMixerProductModule
import com.twitter.home_mixer.{thriftscala => st} import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
import com.twitter.product_mixer.component_library.module.CrMixerClientModule
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
import com.twitter.product_mixer.component_library.module.EarlybirdModule import com.twitter.product_mixer.component_library.module.EarlybirdModule
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
import com.twitter.product_mixer.core.controllers.ProductMixerController import com.twitter.product_mixer.core.controllers.ProductMixerController
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
import com.twitter.product_mixer.core.module.ProductMixerModule import com.twitter.product_mixer.core.module.ProductMixerModule
import com.twitter.product_mixer.core.module.StratoClientModule
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.fed.server.StratoFedServer
object HomeMixerServerMain extends HomeMixerServer object HomeMixerServerMain extends HomeMixerServer
class HomeMixerServer class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
extends StratoFedServer
with ThriftServer
with Mtls
with HttpServer
with HttpMtls {
override val name = "home-mixer-server" override val name = "home-mixer-server"
override val modules: Seq[Module] = Seq( override val modules: Seq[Module] = Seq(
AccountRecommendationsMixerModule, AccountRecommendationsMixerModule,
AdvertiserBrandSafetySettingsStoreModule, AdvertiserBrandSafetySettingsStoreModule,
BlenderClientModule,
ClientSentImpressionsPublisherModule, ClientSentImpressionsPublisherModule,
ConversationServiceModule, ConversationServiceModule,
CrMixerClientModule,
EarlybirdModule, EarlybirdModule,
ExploreRankerClientModule, ExploreRankerClientModule,
FeedbackHistoryClientModule,
GizmoduckClientModule, GizmoduckClientModule,
GlobalParamConfigModule, GlobalParamConfigModule,
HomeAdsCandidateSourceModule, HomeAdsCandidateSourceModule,
HomeMixerFlagsModule, HomeMixerFlagsModule,
HomeMixerProductModule, HomeMixerProductModule,
HomeMixerResourcesModule, HomeMixerResourcesModule,
HomeNaviModelClientModule,
ImpressionBloomFilterModule, ImpressionBloomFilterModule,
InjectionHistoryClientModule, InjectionHistoryClientModule,
FeedbackHistoryClientModule,
ManhattanClientsModule, ManhattanClientsModule,
ManhattanFeatureRepositoryModule, ManhattanFeatureRepositoryModule,
ManhattanTweetImpressionStoreModule, ManhattanTweetImpressionStoreModule,
MemcachedFeatureRepositoryModule, MemcachedFeatureRepositoryModule,
NaviModelClientModule,
OnboardingTaskServiceModule, OnboardingTaskServiceModule,
OptimizedStratoClientModule, OptimizedStratoClientModule,
PeopleDiscoveryServiceModule, PeopleDiscoveryServiceModule,
@ -80,23 +74,24 @@ class HomeMixerServer
SimClustersRecentEngagementsClientModule, SimClustersRecentEngagementsClientModule,
SocialGraphServiceModule, SocialGraphServiceModule,
StaleTweetsCacheModule, StaleTweetsCacheModule,
StratoClientModule,
ThriftFeatureRepositoryModule, ThriftFeatureRepositoryModule,
TimelineMixerClientModule,
TimelineRankerClientModule, TimelineRankerClientModule,
TimelineScorerClientModule, TimelineScorerClientModule,
TimelineServiceClientModule, TimelineServiceClientModule,
TimelinesPersistenceStoreClientModule, TimelinesPersistenceStoreClientModule,
TopicSocialProofClientModule,
TweetImpressionStoreModule, TweetImpressionStoreModule,
TweetMixerClientModule, TweetyPieClientModule,
TweetypieClientModule,
TweetypieStaticEntitiesCacheClientModule, TweetypieStaticEntitiesCacheClientModule,
UserMetadataStoreModule,
UserSessionStoreModule, UserSessionStoreModule,
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](), new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this), new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
new ProductScopeStringCenterModule() new ProductScopeStringCenterModule()
) )
override def configureThrift(router: ThriftRouter): Unit = { def configureThrift(router: ThriftRouter): Unit = {
router router
.filter[LoggingMDCFilter] .filter[LoggingMDCFilter]
.filter[TraceIdMDCFilter] .filter[TraceIdMDCFilter]
@ -116,11 +111,6 @@ class HomeMixerServer
this.injector, this.injector,
st.HomeMixer.ExecutePipeline)) st.HomeMixer.ExecutePipeline))
override val dest: String = "/s/home-mixer/home-mixer:strato"
override val columns: Seq[Class[_ <: StratoFed.Column]] =
Seq(classOf[HomeMixerColumn])
override protected def warmup(): Unit = { override protected def warmup(): Unit = {
handle[HomeMixerThriftServerWarmupHandler]() handle[HomeMixerThriftServerWarmupHandler]()
handle[HomeMixerHttpServerWarmupHandler]() handle[HomeMixerHttpServerWarmupHandler]()

View File

@ -3,7 +3,7 @@ package com.twitter.home_mixer
import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.thrift.ClientId
import com.twitter.finatra.thrift.routing.ThriftWarmup import com.twitter.finatra.thrift.routing.ThriftWarmup
import com.twitter.home_mixer.{thriftscala => st} import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.util.logging.Logging import com.twitter.inject.Logging
import com.twitter.inject.utils.Handler import com.twitter.inject.utils.Handler
import com.twitter.product_mixer.core.{thriftscala => pt} import com.twitter.product_mixer.core.{thriftscala => pt}
import com.twitter.scrooge.Request import com.twitter.scrooge.Request

View File

@ -5,13 +5,19 @@ scala_library(
tags = ["bazel-compatible"], tags = ["bazel-compatible"],
dependencies = [ dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
@ -19,6 +25,10 @@ scala_library(
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
], ],
exports = [ exports = [
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",

View File

@ -1,23 +1,18 @@
package com.twitter.home_mixer.candidate_pipeline package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.filter.FeatureFilter
import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
@ -38,8 +33,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
conversationServiceCandidateSource: ConversationServiceCandidateSource, conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator, tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
override val gates: Seq[BaseGate[Query]], override val gates: Seq[BaseGate[Query]],
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]]) override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
extends DependentCandidatePipelineConfig[ extends DependentCandidatePipelineConfig[
@ -67,10 +62,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
val tweetsWithConversationMetadata = candidates.map { candidate => val tweetsWithConversationMetadata = candidates.map { candidate =>
TweetWithConversationMetadata( TweetWithConversationMetadata(
tweetId = candidate.candidateIdLong, tweetId = candidate.candidateIdLong,
userId = candidate.features.getOrElse(AuthorIdFeature, None), userId = None,
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), sourceTweetId = None,
sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), sourceUserId = None,
inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), inReplyToTweetId = None,
conversationId = None, conversationId = None,
ancestors = Seq.empty ancestors = Seq.empty
) )
@ -89,10 +84,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
override val preFilterFeatureHydrationPhase1: Seq[ override val preFilterFeatureHydrationPhase1: Seq[
BaseCandidateFeatureHydrator[Query, TweetCandidate, _] BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
] = Seq( ] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
tweetypieFeatureHydrator,
InNetworkFeatureHydrator,
)
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq( override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
RetweetDeduplicationFilter, RetweetDeduplicationFilter,
@ -101,7 +93,6 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
FilterIdentifier(QuotedTweetDroppedFilterId), FilterIdentifier(QuotedTweetDroppedFilterId),
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
), ),
invalidSubscriptionTweetFilter,
InvalidConversationModuleFilter InvalidConversationModuleFilter
) )

View File

@ -1,9 +1,9 @@
package com.twitter.home_mixer.candidate_pipeline package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.gate.BaseGate
@ -15,7 +15,7 @@ import javax.inject.Singleton
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() ( class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
conversationServiceCandidateSource: ConversationServiceCandidateSource, conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator, tweetypieFeatureHydrator: TweetypieFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator) { namesFeatureHydrator: NamesFeatureHydrator) {
def build( def build(
@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
new ConversationServiceCandidatePipelineConfig( new ConversationServiceCandidatePipelineConfig(
conversationServiceCandidateSource, conversationServiceCandidateSource,
tweetypieFeatureHydrator, tweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator,
namesFeatureHydrator, namesFeatureHydrator,
invalidSubscriptionTweetFilter,
gates, gates,
decorator decorator
) )

View File

@ -1,7 +1,7 @@
package com.twitter.home_mixer.candidate_pipeline package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.service.HomeMixerAlertConfig

View File

@ -13,5 +13,8 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
"snowflake/src/main/scala/com/twitter/snowflake/id", "snowflake/src/main/scala/com/twitter/snowflake/id",
"src/thrift/com/twitter/context:twitter-context-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"twitter-context/src/main/scala",
], ],
) )

View File

@ -7,7 +7,6 @@ import com.twitter.home_mixer.service.ScoredTweetsService
import com.twitter.home_mixer.{thriftscala => t} import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.controllers.DebugTwitterContext import com.twitter.product_mixer.core.controllers.DebugTwitterContext
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
import com.twitter.product_mixer.core.service.urt.UrtService import com.twitter.product_mixer.core.service.urt.UrtService
import com.twitter.snowflake.id.SnowflakeId import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch

View File

@ -1,24 +0,0 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"stitch/stitch-repo/src/main/scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
"strato/src/main/scala/com/twitter/strato/callcontext",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
],
)

View File

@ -1,217 +0,0 @@
package com.twitter.home_mixer.federated
import com.twitter.gizmoduck.{thriftscala => gd}
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
import com.twitter.home_mixer.model.request.HomeMixerRequest
import com.twitter.home_mixer.{thriftscala => hm}
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
import com.twitter.product_mixer.core.{thriftscala => pm}
import com.twitter.stitch.Arrow
import com.twitter.stitch.Stitch
import com.twitter.strato.callcontext.CallContext
import com.twitter.strato.catalog.OpMetadata
import com.twitter.strato.config._
import com.twitter.strato.data._
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
import com.twitter.strato.graphql.timelines.{thriftscala => gql}
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.timelines.render.{thriftscala => tr}
import com.twitter.util.Try
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class HomeMixerColumn @Inject() (
homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn,
paramsBuilder: ParamsBuilder,
productPipelineRegistry: ProductPipelineRegistry)
extends StratoFed.Column(HomeMixerColumn.Path)
with StratoFed.Fetch.Arrow {
override val contactInfo: ContactInfo = ContactInfo(
contactEmail = "",
ldapGroup = "",
slackRoomId = ""
)
override val metadata: OpMetadata =
OpMetadata(
lifecycle = Some(Lifecycle.Production),
description =
Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
)
private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
ServiceIdentifierPattern(
role = "",
service = "",
env = "",
zone = Seq(""))
)
override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
override type Key = gql.TimelineKey
override type View = gql.HomeTimelineView
override type Value = tr.Timeline
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
private def createHomeMixerRequestArrow(
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn
): Arrow[(Key, View), hm.HomeMixerRequest] = {
val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
(gd.LookupContext(), Set(gd.QueryFields.Roles))
val populateUserRoles = Arrow
.flatMap[(Key, View), Option[Set[String]]] { _ =>
Stitch.collect {
CallContext.twitterUserId.map { userId =>
compositeOnUserClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch(userId, gizmoduckView).map(_.v)
.map {
_.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
}
}
}
}
val populateIpAddress = Arrow
.flatMap[(Key, View), Option[String]](_ =>
auditIpClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch((), ()).map(_.v))
Arrow.join(
populateUserRoles,
populateIpAddress
)
}
Arrow.zipWithArg(populateUserRolesAndIp).map {
case ((key, view), (roles, ipAddress)) =>
val deviceContextOpt = Some(
hm.DeviceContext(
isPolling = CallContext.isPolling,
requestContext = view.requestContext,
latestControlAvailable = view.latestControlAvailable,
autoplayEnabled = view.autoplayEnabled
))
val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
val (product, productContext) = key match {
case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
(
hm.Product.ForYou,
hm.ProductContext.ForYou(
hm.ForYou(
deviceContextOpt,
seenTweetIds,
view.dspClientContext,
view.pushToHomeTweetId
)
))
case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
(
hm.Product.Following,
hm.ProductContext.Following(
hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
(
hm.Product.Subscribed,
hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
}
val clientContext = pm.ClientContext(
userId = CallContext.twitterUserId,
guestId = CallContext.guestId,
guestIdAds = CallContext.guestIdAds,
guestIdMarketing = CallContext.guestIdMarketing,
appId = CallContext.clientApplicationId,
ipAddress = ipAddress,
userAgent = CallContext.userAgent,
countryCode = CallContext.requestCountryCode,
languageCode = CallContext.requestLanguageCode,
isTwoffice = CallContext.isInternalOrTwoffice,
userRoles = roles,
deviceId = CallContext.deviceId,
mobileDeviceId = CallContext.mobileDeviceId,
mobileDeviceAdId = CallContext.adId,
limitAdTracking = CallContext.limitAdTracking
)
hm.HomeMixerRequest(
clientContext = clientContext,
product = product,
productContext = Some(productContext),
maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
cursor = view.cursor.filter(_.nonEmpty)
)
}
}
override val fetch: Arrow[(Key, View), Result[Value]] = {
val transformThriftIntoPipelineRequest: Arrow[
(Key, View),
ProductPipelineRequest[HomeMixerRequest]
] = {
Arrow
.identity[(Key, View)]
.andThen {
createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
}
.map {
case thriftRequest =>
val request = homeMixerRequestUnmarshaller(thriftRequest)
val params = paramsBuilder.build(
clientContext = request.clientContext,
product = request.product,
featureOverrides =
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
)
ProductPipelineRequest(request, params)
}
}
val underlyingProduct: Arrow[
ProductPipelineRequest[HomeMixerRequest],
ProductPipelineResult[tr.TimelineResponse]
] = Arrow
.identity[ProductPipelineRequest[HomeMixerRequest]]
.map { pipelineRequest =>
val pipelineArrow = productPipelineRegistry
.getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
pipelineRequest.request.product)
.arrow
(pipelineArrow, pipelineRequest)
}.applyArrow
transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
_.result match {
case Some(result) => found(result.timeline)
case _ => missing
}
}
}
}
object HomeMixerColumn {
val Path = "home-mixer/homeMixer.Timeline"
private val FetchCallstack = s"$Path:fetch"
private val MaxCount: Option[Int] = Some(100)
}

View File

@ -10,8 +10,11 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
"src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search:earlybird-scala",
"stitch/stitch-timelineservice/src/main/scala", "stitch/stitch-timelineservice/src/main/scala",
"strato/config/columns/recommendations/similarity:similarity-strato-client",
"strato/src/main/scala/com/twitter/strato/client",
], ],
exports = [ exports = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.list_recommended_users.candidate_source package com.twitter.home_mixer.functional_component.candidate_source
import com.twitter.hermit.candidate.{thriftscala => t} import com.twitter.hermit.candidate.{thriftscala => t}
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
@ -21,18 +21,12 @@ class SimilarityBasedUsersCandidateSource @Inject() (
private val fetcher: Fetcher[Long, Unit, t.Candidates] = private val fetcher: Fetcher[Long, Unit, t.Candidates] =
similarUsersBySimsOnUserClientColumn.fetcher similarUsersBySimsOnUserClientColumn.fetcher
private val MaxCandidatesToKeep = 4000
override def apply(request: Seq[Long]): Stitch[Seq[t.Candidate]] = { override def apply(request: Seq[Long]): Stitch[Seq[t.Candidate]] = {
Stitch Stitch
.collect { .collect {
request.map { userId => request.map { userId =>
fetcher fetcher.fetch(userId, Unit).map { result =>
.fetch(userId, Unit).map { result =>
result.v.map(_.candidates).getOrElse(Seq.empty) result.v.map(_.candidates).getOrElse(Seq.empty)
}.map { candidates =>
val sortedCandidates = candidates.sortBy(-_.score)
sortedCandidates.take(MaxCandidatesToKeep)
} }
} }
}.map(_.flatten) }.map(_.flatten)

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -9,6 +9,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t} import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -6,11 +6,13 @@ scala_library(
dependencies = [ dependencies = [
"finagle/finagle-core/src/main", "finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
@ -23,6 +25,8 @@ scala_library(
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"stringcenter/client", "stringcenter/client",
"stringcenter/client/src/main/java", "stringcenter/client/src/main/java",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe", "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
], ],
) )

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -13,6 +13,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -17,6 +17,7 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tls} import com.twitter.timelineservice.{thriftscala => tls}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.finagle.tracing.Trace import com.twitter.finagle.tracing.Trace
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.bijection.Base64String import com.twitter.bijection.Base64String
import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.bijection.scrooge.BinaryScalaCodec

View File

@ -1,8 +1,6 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.FollowingProduct
@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => t} import com.twitter.timelines.service.{thriftscala => t}
import com.twitter.timelines.util.FeedbackMetadataSerializer import com.twitter.timelines.util.FeedbackMetadataSerializer
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
@ -14,13 +14,10 @@ import javax.inject.Singleton
@Singleton @Singleton
case class HomeTweetSocialContextBuilder @Inject() ( case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder: LikedBySocialContextBuilder, likedBySocialContextBuilder: LikedBySocialContextBuilder,
listsSocialContextBuilder: ListsSocialContextBuilder,
followedBySocialContextBuilder: FollowedBySocialContextBuilder, followedBySocialContextBuilder: FollowedBySocialContextBuilder,
topicSocialContextBuilder: TopicSocialContextBuilder, topicSocialContextBuilder: TopicSocialContextBuilder,
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder, extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder, receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
def apply( def apply(
@ -34,9 +31,6 @@ case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder(query, candidate, features) likedBySocialContextBuilder(query, candidate, features)
.orElse(followedBySocialContextBuilder(query, candidate, features)) .orElse(followedBySocialContextBuilder(query, candidate, features))
.orElse(topicSocialContextBuilder(query, candidate, features)) .orElse(topicSocialContextBuilder(query, candidate, features))
.orElse(popularVideoSocialContextBuilder(query, candidate, features))
.orElse(listsSocialContextBuilder(query, candidate, features))
.orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
case Some(_) => case Some(_) =>
val conversationId = features.getOrElse(ConversationModuleIdFeature, None) val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
// Only hydrate the social context into the root tweet in a conversation module // Only hydrate the social context into the root tweet in a conversation module

View File

@ -1,10 +1,8 @@
package com.twitter.home_mixer.functional_component.decorator.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
import com.twitter.tweetypie.{thriftscala => tpt} import com.twitter.tweetypie.{thriftscala => tpt}
@ -27,76 +25,74 @@ object HomeTweetTypePredicates {
( (
"has_exclusive_conversation_author_id", "has_exclusive_conversation_author_id",
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty), _.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
("is_eligible_for_connect_boost", _ => false), ("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)),
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)), ("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)), ("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)), ("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)), ("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("get_initial", _.getOrElse(GetInitialFeature, false)), ("get_initial", _.getOrElse(GetInitialFeature, false)),
("get_newer", _.getOrElse(GetNewerFeature, false)), ("get_newer", _.getOrElse(GetNewerFeature, false)),
("get_middle", _.getOrElse(GetMiddleFeature, false)), ("get_middle", _.getOrElse(GetMiddleFeature, false)),
("get_older", _.getOrElse(GetOlderFeature, false)), ("get_older", _.getOrElse(GetOlderFeature, false)),
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)), ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
("polling", _.getOrElse(PollingFeature, false)), ("polling", _.getOrElse(PollingFeature, false)),
("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)), ("tls_size_20_plus", _ => false),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)), ("near_empty", _ => false),
("ranked_request", _ => false),
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)), ("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
(
"less_than_10_mins_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)), ("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)), ("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
( (
"conversation_module_has_2_displayed_tweets", "is_signup_request",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)), candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)), ("empty_request", _ => false),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)), ("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
( (
"served_size_between_50_and_100", "served_size_between_50_and_100",
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)), _.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)), ("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty), ("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)), ("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
( (
"account_age_less_than_30_minutes", "account_age_less_than_30_minutes",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)), _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
( (
"account_age_less_than_1_day", "account_age_less_than_1_day",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)), _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
( (
"account_age_less_than_7_days", "account_age_less_than_7_days",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)), _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
("root_user_is_in_first_degree", _ => false),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
( (
"part_of_utt", "part_of_utt",
_.getOrElse(EarlybirdFeature, None) _.getOrElse(EarlybirdFeature, None)
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation => .exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))), annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
(
"has_home_latest_request_past_week",
_.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)), ("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)), ("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)), ("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("viewer_is_employee", _ => false),
("viewer_is_timelines_employee", _ => false),
("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
( (
"has_ancestor_authored_by_viewer", "has_ancestor_authored_by_viewer",
candidate => candidate =>
@ -104,6 +100,11 @@ object HomeTweetTypePredicates {
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor => .getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)), candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)), ("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
( (
"deep_reply", "deep_reply",
candidate => candidate =>
@ -118,22 +119,23 @@ object HomeTweetTypePredicates {
"tweet_age_less_than_15_seconds", "tweet_age_less_than_15_seconds",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 15.seconds)), .exists(_.untilNow <= 15.seconds)),
( ("is_followed_topic_tweet", _ => false),
"less_than_1_hour_since_lnpt", ("is_recommended_topic_tweet", _ => false),
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)), ("is_topic_tweet", _ => false),
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))), ("preferred_language_matches_tweet_language", _ => false),
( (
"device_language_matches_tweet_language", "device_language_matches_tweet_language",
candidate => candidate =>
candidate.getOrElse(TweetLanguageFeature, None) == candidate.getOrElse(TweetLanguageFeature, None) ==
candidate.getOrElse(DeviceLanguageFeature, None)), candidate.getOrElse(DeviceLanguageFeature, None)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))), ("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
("in_network", _.getOrElse(InNetworkFeature, true)), ("in_network", _.getOrElse(FromInNetworkSourceFeature, true)),
("viewer_follows_original_author", _ => false),
("has_account_follow_prompt", _ => false),
("has_relevance_prompt", _ => false),
("has_topic_annotation_haug_prompt", _ => false),
("has_topic_annotation_random_precision_prompt", _ => false),
("has_topic_annotation_prompt", _ => false),
( (
"has_political_annotation", "has_political_annotation",
_.getOrElse(EarlybirdFeature, None).exists( _.getOrElse(EarlybirdFeature, None).exists(
@ -151,14 +153,9 @@ object HomeTweetTypePredicates {
_.getOrElse(EarlybirdFeature, None) _.getOrElse(EarlybirdFeature, None)
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))), .exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)), ("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
( ("is_viewer_not_invited_to_reply", _ => false),
"is_followed_topic_tweet", ("is_viewer_invited_to_reply", _ => false),
_.getOrElse(TopicContextFunctionalityTypeFeature, None) ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
.exists(_ == BasicTopicContextFunctionalityType)),
(
"is_recommended_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == RecommendationTopicContextFunctionalityType)),
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))), ("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))), ("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
( (
@ -167,6 +164,8 @@ object HomeTweetTypePredicates {
( (
"has_gte_100k_favs", "has_gte_100k_favs",
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))), _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
("above_neighbor_is_topic_tweet", _ => false),
("is_topic_tweet_with_neighbor_below", _ => false),
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)), ("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)), ("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
( (
@ -188,7 +187,6 @@ object HomeTweetTypePredicates {
( (
"has_toxicity_score_above_threshold", "has_toxicity_score_above_threshold",
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))), _.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
( (
"text_only", "text_only",
candidate => candidate =>
@ -206,50 +204,23 @@ object HomeTweetTypePredicates {
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)), ("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)), ("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)), ("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
("3_or_more_consecutive_not_in_network", _ => false),
("2_or_more_consecutive_not_in_network", _ => false),
("5_out_of_7_not_in_network", _ => false),
("7_out_of_7_not_in_network", _ => false),
("5_out_of_5_not_in_network", _ => false),
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)), ("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
("has_liked_by_social_context", _ => false),
("has_followed_by_social_context", _ => false),
("has_topic_social_context", _ => false),
("timeline_entry_has_banner", _ => false),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
( (
"has_liked_by_social_context", "conversation_module_has_2_displayed_tweets",
candidateFeatures => _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
candidateFeatures ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) ("served_in_recap_tweet_candidate_module_injection", _ => false),
.exists(candidateFeatures ("served_in_threaded_conversation_module", _ => false)
.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
(
"has_followed_by_social_context",
_.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
(
"has_topic_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(TopicIdSocialContextFeature, None)
.isDefined &&
candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
(
"video_bt_10_60_sec",
_.getOrElse(VideoDurationMsFeature, None).exists(duration =>
duration > 10000 && duration <= 60000)),
("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
(
"tweet_age_lte_30_minutes",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 30.minutes)),
(
"tweet_age_lte_1_hour",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 1.hour)),
(
"tweet_age_lte_6_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 6.hours)),
(
"tweet_age_lte_12_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 12.hours)),
(
"tweet_age_gte_24_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow >= 24.hours)),
) )
val PredicateMap = CandidatePredicates.toMap val PredicateMap = CandidatePredicates.toMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.list_tweets.decorator package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.ListClientEventDetailsBuilder import com.twitter.home_mixer.functional_component.decorator.builder.ListClientEventDetailsBuilder
@ -23,10 +23,8 @@ object ListConversationServiceCandidateDecorator {
def apply(): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = { def apply(): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = {
val suggestType = st.SuggestType.OrganicListTweet val suggestType = st.SuggestType.OrganicListTweet
val component = InjectionScribeUtil.scribeComponent(suggestType).get val component = InjectionScribeUtil.scribeComponent(suggestType).get
val clientEventInfoBuilder = ClientEventInfoBuilder( val clientEventInfoBuilder =
component = component, ClientEventInfoBuilder(component, Some(ListClientEventDetailsBuilder))
detailsBuilder = Some(ListClientEventDetailsBuilder(st.SuggestType.OrganicListTweet))
)
val tweetItemBuilder = TweetCandidateUrtItemBuilder( val tweetItemBuilder = TweetCandidateUrtItemBuilder(
clientEventInfoBuilder = clientEventInfoBuilder clientEventInfoBuilder = clientEventInfoBuilder
) )

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
@ -15,6 +15,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
@ -12,6 +12,7 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tlst} import com.twitter.timelineservice.{thriftscala => tlst}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -8,6 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t} import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
@ -11,6 +11,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,48 +1,50 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Provider import javax.inject.Provider
import javax.inject.Singleton import javax.inject.Singleton
/**
* Renders a fixed 'You Might Like' string above all OON Tweets.
*/
@Singleton @Singleton
case class PopularVideoSocialContextBuilder @Inject() ( case class YouMightLikeSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings, externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter]) @ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get() private val stringCenter = stringCenterProvider.get()
private val popularVideoString = externalStrings.socialContextPopularVideoString private val youMightLikeString = externalStrings.socialContextYouMightLikeString
def apply( def apply(
query: PipelineQuery, query: PipelineQuery,
candidate: TweetCandidate, candidate: TweetCandidate,
candidateFeatures: FeatureMap candidateFeatures: FeatureMap
): Option[SocialContext] = { ): Option[SocialContext] = {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None) val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true)
if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) { val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false)
if (!isInNetwork && !isRetweet) {
Some( Some(
GeneralContext( GeneralContext(
contextType = SparkleGeneralContextType, contextType = SparkleGeneralContextType,
text = stringCenter.prepare(popularVideoString), text = stringCenter.prepare(youMightLikeString),
url = None, url = None,
contextImageUrls = None, contextImageUrls = None,
landingUrl = Some( landingUrl = None
Url(
urlType = DeepLink,
url = ""
)
)
)) ))
} else None } else {
None
}
} }
} }

View File

@ -8,9 +8,7 @@ scala_library(
"finagle/finagle-core/src/main", "finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
@ -20,7 +18,6 @@ scala_library(
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe", "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
], ],
) )

View File

@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelineservice.suggests.{thriftscala => st} import com.twitter.timelineservice.suggests.{thriftscala => st}
case class ListClientEventDetailsBuilder(suggestType: st.SuggestType) object ListClientEventDetailsBuilder
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] { extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
override def apply( override def apply(
@ -20,7 +20,7 @@ case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
conversationDetails = None, conversationDetails = None,
timelinesDetails = Some( timelinesDetails = Some(
TimelinesDetails( TimelinesDetails(
injectionType = Some(suggestType.name), injectionType = Some(st.SuggestType.OrganicListTweet.name),
controllerData = None, controllerData = None,
sourceData = None)), sourceData = None)),
articleDetails = None, articleDetails = None,

View File

@ -4,16 +4,9 @@ scala_library(
strict_deps = True, strict_deps = True,
tags = ["bazel-compatible"], tags = ["bazel-compatible"],
dependencies = [ dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
"src/thrift/com/twitter/timelines/service:thrift-scala", "src/thrift/com/twitter/timelines/service:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
], ],
) )

View File

@ -1,18 +0,0 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
class FeedbackStrings @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
private val externalStringRegistry = externalStringRegistryProvider.get()
val seeLessOftenFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOften")
val seeLessOftenConfirmationFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
}

View File

@ -5,6 +5,7 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Provider import javax.inject.Provider
@ -31,13 +32,12 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
@Singleton @Singleton
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() ( case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
feedbackStrings: FeedbackStrings, @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
@ProductScoped stringCenterProvider: Provider[StringCenter]) @ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, externalStringRegistry = externalStringRegistryProvider.get(),
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(), stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest) encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
) )

View File

@ -1,52 +0,0 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => tl}
import com.twitter.timelines.util.FeedbackRequestSerializer
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.timelineservice.thriftscala.FeedbackType
object HomeWhoToSubscribeFeedbackActionInfoBuilder {
private val FeedbackMetadata = tl.FeedbackMetadata(
injectionType = Some(SuggestType.WhoToSubscribe),
engagementType = None,
entityIds = Seq.empty,
ttlMs = None
)
private val FeedbackRequest =
tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
private val EncodedFeedbackRequest =
FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
}
@Singleton
case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest =
Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)
override def apply(
query: PipelineQuery,
candidate: UserCandidate,
candidateFeatures: FeatureMap
): Option[FeedbackActionInfo] =
whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
}

View File

@ -1,50 +0,0 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* "Your Lists" will be rendered for the context and a url link for your lists.
*/
@Singleton
case class ListsSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
candidateFeatures.get(SuggestTypeFeature) match {
case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
Some(
GeneralContext(
contextType = ListGeneralContextType,
text = stringCenter.prepare(listString),
url = userName.map(name => ""),
contextImageUrls = None,
landingUrl = None
))
case _ => None
}
}
}

View File

@ -1,43 +0,0 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
case class PopularInYourAreaSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
Some(
GeneralContext(
contextType = LocationGeneralContextType,
text = stringCenter.prepare(popularInYourAreaString),
url = None,
contextImageUrls = None,
landingUrl = None
))
} else None
}
}

View File

@ -1,7 +1,6 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -9,11 +8,9 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta}
import com.twitter.tweetconvosvc.{thriftscala => tcs} import com.twitter.tweetconvosvc.{thriftscala => tcs}
import com.twitter.util.Future
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@ -26,16 +23,15 @@ class AncestorFeatureHydrator @Inject() (
override val features: Set[Feature[_, _]] = Set(AncestorsFeature) override val features: Set[Feature[_, _]] = Set(AncestorsFeature)
private val DefaultFeatureMap = FeatureMapBuilder().add(AncestorsFeature, Seq.empty).build()
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidate: TweetCandidate, candidate: TweetCandidate,
existingFeatures: FeatureMap existingFeatures: FeatureMap
): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { ): Stitch[FeatureMap] = {
if (existingFeatures.getOrElse(InReplyToTweetIdFeature, None).isDefined) {
val ancestorsRequest = tcs.GetAncestorsRequest(Seq(candidate.id)) val ancestorsRequest = tcs.GetAncestorsRequest(Seq(candidate.id))
conversationServiceClient.getAncestors(ancestorsRequest).map { getAncestorsResponse =>
Stitch.callFuture(conversationServiceClient.getAncestors(ancestorsRequest)).map {
getAncestorsResponse =>
val ancestors = getAncestorsResponse.ancestors.headOption val ancestors = getAncestorsResponse.ancestors.headOption
.collect { .collect {
case tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult) case tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult)
@ -45,7 +41,6 @@ class AncestorFeatureHydrator @Inject() (
FeatureMapBuilder().add(AncestorsFeature, ancestors).build() FeatureMapBuilder().add(AncestorsFeature, ancestors).build()
} }
} else Future.value(DefaultFeatureMap)
} }
private def getTruncatedRootTweet( private def getTruncatedRootTweet(

View File

@ -1,28 +1,28 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.AuthorFeatureRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.AuthorFeatureRepository
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter
import com.twitter.home_mixer.util.CandidatesUtil
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.servo.repository.KeyValueResult import com.twitter.servo.repository.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.author_features.v1.{thriftjava => af} import com.twitter.timelines.author_features.v1.{thriftjava => af}
import com.twitter.util.Future import com.twitter.util.Future
import com.twitter.util.Try import com.twitter.util.Try
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Named import javax.inject.Named
import javax.inject.Singleton import javax.inject.Singleton
@ -51,28 +51,34 @@ class AuthorFeatureHydrator @Inject() (
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { ): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val possiblyAuthorIds = extractKeys(candidates) val possiblyAuthorIds = extractKeys(candidates)
val authorIds = possiblyAuthorIds.flatten val authorIds = possiblyAuthorIds.flatten
val response: Future[KeyValueResult[Long, af.AuthorFeatures]] = val response: Future[KeyValueResult[Long, af.AuthorFeatures]] =
if (authorIds.nonEmpty) client(authorIds) if (authorIds.isEmpty) {
else Future.value(KeyValueResult.empty) Future.value(KeyValueResult.empty)
} else {
client(authorIds)
}
response.map { result => response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId => possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result) val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value) val transformedValue = postTransformer(value)
FeatureMapBuilder().add(AuthorFeature, transformedValue).build() FeatureMapBuilder()
.add(AuthorFeature, transformedValue)
.build()
}
} }
} }
} }
private def postTransformer(authorFeatures: Try[Option[af.AuthorFeatures]]): Try[DataRecord] = { private def postTransformer(authorFeatures: Try[Option[af.AuthorFeatures]]): Try[DataRecord] = {
authorFeatures.map { authorFeatures.map { features =>
_.map { features => AuthorFeaturesAdapter.adaptToDataRecords(features).asScala.head } AuthorFeaturesAdapter.adaptToDataRecords(features).asScala.head
.getOrElse(new DataRecord())
} }
} }
@ -80,7 +86,10 @@ class AuthorFeatureHydrator @Inject() (
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]] = { ): Seq[Option[Long]] = {
candidates.map { candidate => candidates.map { candidate =>
CandidatesUtil.getOriginalAuthorId(candidate.features) candidate.features
.getTry(AuthorIdFeature)
.toOption
.flatten
} }
} }
} }

View File

@ -4,56 +4,95 @@ scala_library(
strict_deps = True, strict_deps = True,
tags = ["bazel-compatible"], tags = ["bazel-compatible"],
dependencies = [ dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider", "configapi/configapi-decider",
"finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-core/src/main/scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
"joinkey/src/main/scala/com/twitter/joinkey/context", "joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
"representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
"representation-scorer/server/src/main/thrift:thrift-scala",
"servo/repo/src/main/scala",
"snowflake/src/main/scala/com/twitter/snowflake/id", "snowflake/src/main/scala/com/twitter/snowflake/id",
"src/java/com/twitter/ml/api/constant",
"src/java/com/twitter/search/common/util/lang", "src/java/com/twitter/search/common/util/lang",
"src/scala/com/twitter/timelines/prediction/adapters/request_context", "src/scala/com/twitter/ml/api/util",
"src/scala/com/twitter/timelines/prediction/adapters/real_graph",
"src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/adapters/twistly",
"src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
"src/scala/com/twitter/timelines/prediction/common/util",
"src/scala/com/twitter/timelines/prediction/features/common",
"src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/features/recap",
"src/scala/com/twitter/timelines/prediction/features/time_features",
"src/thrift/com/twitter/gizmoduck:thrift-scala", "src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/ml/api:data-java",
"src/thrift/com/twitter/ml/api:embedding-java",
"src/thrift/com/twitter/onboarding/relevance/features:features-java",
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
"src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search:earlybird-scala",
"src/thrift/com/twitter/search/common:constants-java", "src/thrift/com/twitter/search/common:constants-java",
"src/thrift/com/twitter/socialgraph:thrift-scala", "src/thrift/com/twitter/socialgraph:thrift-scala",
"src/thrift/com/twitter/spam/rtf:safety-result-scala", "src/thrift/com/twitter/spam/rtf:safety-result-scala",
"src/thrift/com/twitter/timelineranker:thrift-scala", "src/thrift/com/twitter/timelineranker:thrift-scala",
"src/thrift/com/twitter/timelines/author_features:thrift-java",
"src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
"src/thrift/com/twitter/timelines/impression:thrift-scala", "src/thrift/com/twitter/timelines/impression:thrift-scala",
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala", "src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
"src/thrift/com/twitter/tweetypie:service-scala", "src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala", "src/thrift/com/twitter/tweetypie:tweet-scala",
"src/thrift/com/twitter/user_session_store:thrift-java", "src/thrift/com/twitter/user_session_store:thrift-java",
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
"src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
"stitch/stitch-core", "stitch/stitch-core",
"stitch/stitch-gizmoduck", "stitch/stitch-gizmoduck",
"stitch/stitch-socialgraph", "stitch/stitch-socialgraph",
"stitch/stitch-timelineservice", "stitch/stitch-timelineservice",
"stitch/stitch-tweetypie", "stitch/stitch-tweetypie",
"strato/config/columns/topic-signals/tsp",
"strato/config/columns/topic-signals/tsp:tsp-strato-client",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store", "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly",
"timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store", "timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"topic-social-proof/server/src/main/thrift:thrift-scala",
"topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
"tweetconvosvc/thrift/src/main/thrift:thrift-scala",
"twitter-config/yaml",
"user_session_store/src/main/scala/com/twitter/user_session_store", "user_session_store/src/main/scala/com/twitter/user_session_store",
"util/util-core", "util/util-core",
], ],

View File

@ -1,10 +1,9 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.EarlybirdAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.earlybird.EarlybirdAdapter
import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
@ -12,7 +11,6 @@ import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.home_mixer.util.earlybird.EarlybirdResponseUtil import com.twitter.home_mixer.util.earlybird.EarlybirdResponseUtil
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
@ -23,7 +21,6 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.search.earlybird.{thriftscala => eb}
import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.keyvalue.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueRepository
@ -53,12 +50,8 @@ class EarlybirdFeatureHydrator @Inject() (
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Earlybird") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Earlybird")
override val features: Set[Feature[_, _]] = Set( override val features: Set[Feature[_, _]] =
EarlybirdDataRecordFeature, Set(EarlybirdDataRecordFeature, EarlybirdFeature, TweetUrlsFeature)
EarlybirdFeature,
EarlybirdSearchResultFeature,
TweetUrlsFeature
)
override val statScope: String = identifier.toString override val statScope: String = identifier.toString
@ -66,66 +59,52 @@ class EarlybirdFeatureHydrator @Inject() (
private val originalKeyFoundCounter = scopedStatsReceiver.counter("originalKey/found") private val originalKeyFoundCounter = scopedStatsReceiver.counter("originalKey/found")
private val originalKeyLossCounter = scopedStatsReceiver.counter("originalKey/loss") private val originalKeyLossCounter = scopedStatsReceiver.counter("originalKey/loss")
private val ebSearchResultNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean =
candidate => candidate.features.getOrElse(EarlybirdSearchResultFeature, None).isEmpty
private val ebFeaturesNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = private val ebFeaturesNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean =
candidate => candidate.features.getOrElse(EarlybirdFeature, None).isEmpty candidate => candidate.features.getOrElse(EarlybirdFeature, None).isEmpty
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { ): Stitch[Seq[FeatureMap]] = {
val candidatesToHydrate = candidates.filter { candidate => val candidatesToHydrate = candidates.filter { candidate =>
val isEmpty = val isEmpty = ebFeaturesNotExistPredicate(candidate)
ebFeaturesNotExistPredicate(candidate) && ebSearchResultNotExistPredicate(candidate)
if (isEmpty) originalKeyLossCounter.incr() else originalKeyFoundCounter.incr() if (isEmpty) originalKeyLossCounter.incr() else originalKeyFoundCounter.incr()
isEmpty isEmpty
} }
Stitch
client((candidatesToHydrate.map(_.candidate.id), query.getRequiredUserId)) .callFuture(client((candidatesToHydrate.map(_.candidate.id), query.getRequiredUserId)))
.map(handleResponse(query, candidates, _, candidatesToHydrate)) .map(handleResponse(query, candidates, _))
} }
private def handleResponse( private def handleResponse(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]], candidates: Seq[CandidateWithFeatures[TweetCandidate]],
results: KeyValueResult[Long, eb.ThriftSearchResult], results: KeyValueResult[Long, eb.ThriftSearchResult]
candidatesToHydrate: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[FeatureMap] = { ): Seq[FeatureMap] = {
val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) val queryFeatureMap = query.features.getOrElse(FeatureMap.empty)
val userLanguages = queryFeatureMap.getOrElse(UserLanguagesFeature, Seq.empty) val userLanguages = queryFeatureMap.getOrElse(UserLanguagesFeature, Seq.empty)
val uiLanguageCode = queryFeatureMap.getOrElse(DeviceLanguageFeature, None) val uiLanguageCode = queryFeatureMap.getOrElse(DeviceLanguageFeature, None)
val screenName = queryFeatureMap.getOrElse(UserScreenNameFeature, None) val screenName = queryFeatureMap.getOrElse(UserScreenNameFeature, None)
val followedUserIds = queryFeatureMap.getOrElse(SGSFollowedUsersFeature, Seq.empty).toSet
val searchResults = candidatesToHydrate val searchResults = candidates
.map { candidate => .filter(ebFeaturesNotExistPredicate).map { candidate =>
observedGet(Some(candidate.candidate.id), results) observedGet(Some(candidate.candidate.id), results)
}.collect { }.collect {
case Return(Some(value)) => value case Return(Some(value)) => value
} }
val allSearchResults = searchResults ++ val tweetIdToEbFeatures = EarlybirdResponseUtil.getOONTweetThriftFeaturesByTweetId(
candidates.filter(!ebSearchResultNotExistPredicate(_)).flatMap { candidate =>
candidate.features
.getOrElse(EarlybirdSearchResultFeature, None)
}
val idToSearchResults = allSearchResults.map(sr => sr.id -> sr).toMap
val tweetIdToEbFeatures = EarlybirdResponseUtil.getTweetThriftFeaturesByTweetId(
searcherUserId = query.getRequiredUserId, searcherUserId = query.getRequiredUserId,
screenName = screenName, screenName = screenName,
userLanguages = userLanguages, userLanguages = userLanguages,
uiLanguageCode = uiLanguageCode, uiLanguageCode = uiLanguageCode,
followedUserIds = followedUserIds, searchResults = searchResults
mutuallyFollowingUserIds = Set.empty,
searchResults = allSearchResults,
sourceTweetSearchResults = Seq.empty,
) )
candidates.map { candidate => candidates.map { candidate =>
val transformedEbFeatures = tweetIdToEbFeatures.get(candidate.candidate.id) val hydratedEbFeatures = tweetIdToEbFeatures.get(candidate.candidate.id)
val earlybirdFeatures = val earlybirdFeatures =
if (transformedEbFeatures.nonEmpty) transformedEbFeatures if (hydratedEbFeatures.nonEmpty) hydratedEbFeatures
else candidate.features.getOrElse(EarlybirdFeature, None) else candidate.features.getOrElse(EarlybirdFeature, None)
val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false)
@ -143,7 +122,6 @@ class EarlybirdFeatureHydrator @Inject() (
FeatureMapBuilder() FeatureMapBuilder()
.add(EarlybirdFeature, earlybirdFeatures) .add(EarlybirdFeature, earlybirdFeatures)
.add(EarlybirdDataRecordFeature, earlybirdDataRecord) .add(EarlybirdDataRecordFeature, earlybirdDataRecord)
.add(EarlybirdSearchResultFeature, idToSearchResults.get(candidate.candidate.id))
.add(TweetUrlsFeature, earlybirdFeatures.flatMap(_.urlsList).getOrElse(Seq.empty)) .add(TweetUrlsFeature, earlybirdFeatures.flatMap(_.urlsList).getOrElse(Seq.empty))
.build() .build()
} }

View File

@ -1,10 +1,12 @@
package com.twitter.home_mixer.functional_component.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
@ -15,12 +17,16 @@ import javax.inject.Singleton
@Singleton @Singleton
case class FeedbackHistoryQueryFeatureHydrator @Inject() ( case class FeedbackHistoryQueryFeatureHydrator @Inject() (
feedbackHistoryClient: FeedbackHistoryManhattanClient) feedbackHistoryClient: FeedbackHistoryManhattanClient)
extends QueryFeatureHydrator[PipelineQuery] { extends QueryFeatureHydrator[PipelineQuery]
with Conditionally[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature) override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableFeedbackFatigueParam)
override def hydrate( override def hydrate(
query: PipelineQuery query: PipelineQuery
): Stitch[FeatureMap] = ): Stitch[FeatureMap] =

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.for_you.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature

View File

@ -0,0 +1,41 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class FollowedTopicsQueryFeatureHydrator @Inject() (
followedTopicsCandidateSource: FollowedTopicsCandidateSource)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
followedTopicsCandidateSource(request)
.map { topics =>
FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
}.handle {
case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
}
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
)
}

View File

@ -0,0 +1,58 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.gizmoduck.{thriftscala => gt}
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.stitch.gizmoduck.Gizmoduck
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("GizmoduckAuthorSafety")
override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
val blueVerifiedStitch = authorIdOption
.map { authorId =>
gizmoduck
.getUserById(
userId = authorId,
queryFields = queryFields
)
.map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
}.getOrElse(Stitch.False)
blueVerifiedStitch.map { isBlueVerified =>
FeatureMapBuilder()
.add(AuthorIsBlueVerifiedFeature, isBlueVerified)
.build()
}
}
}

View File

@ -1,13 +1,15 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.graph_feature_service.{thriftscala => gfs} import com.twitter.graph_feature_service.{thriftscala => gfs}
import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.IsExtendedReplyFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository
import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.CandidatesUtil
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.home_mixer.util.ReplyRetweetUtil
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
@ -19,7 +21,6 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.two_hop_features.TwoHopFeaturesAdapter import com.twitter.timelines.prediction.adapters.two_hop_features.TwoHopFeaturesAdapter
@ -57,19 +58,28 @@ class GraphTwoHopFeatureHydrator @Inject() (
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { ): Stitch[Seq[FeatureMap]] = {
// Apply filters to in network candidates for retweets only. // Apply filters to in network candidates for ExtendedReplyAncestors and retweets.
// ExtendedReplyAncestors should also be in candidates. No filter for oon.
val (inNetworkCandidates, oonCandidates) = candidates.partition { candidate => val (inNetworkCandidates, oonCandidates) = candidates.partition { candidate =>
candidate.features.getOrElse(FromInNetworkSourceFeature, false) candidate.features.getOrElse(InNetworkFeature, false)
} }
val inNetworkCandidatesToHydrate = val inNetworkReplyToAncestorTweet =
ReplyRetweetUtil.replyToAncestorTweetCandidatesMap(inNetworkCandidates)
val inNetworkExtendedReplyAncestors = inNetworkCandidates
.filter(_.features.getOrElse(IsExtendedReplyFeature, false)).flatMap { inNetworkCandidate =>
inNetworkReplyToAncestorTweet.get(inNetworkCandidate.candidate.id)
}.flatten
val inNetworkCandidatesToHydrate = inNetworkExtendedReplyAncestors ++
inNetworkCandidates.filter(_.features.getOrElse(IsRetweetFeature, false)) inNetworkCandidates.filter(_.features.getOrElse(IsRetweetFeature, false))
val candidatesToHydrate = (inNetworkCandidatesToHydrate ++ oonCandidates) val candidatesToHydrate = (inNetworkCandidatesToHydrate ++ oonCandidates)
.flatMap(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)).distinct .flatMap(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)).distinct
val response = client((candidatesToHydrate, query.getRequiredUserId)) val response = Stitch.callFuture(client((candidatesToHydrate, query.getRequiredUserId)))
response.map { result => response.map { result =>
candidates.map { candidate => candidates.map { candidate =>
@ -77,9 +87,7 @@ class GraphTwoHopFeatureHydrator @Inject() (
val value = observedGet(key = originalAuthorId, keyValueResult = result) val value = observedGet(key = originalAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value) val transformedValue = postTransformer(value)
val followedByUserIds = value.toOption val followedByUserIds = value.toOption.flatMap(getFollowedByUserIds(_)).getOrElse(Seq.empty)
.flatMap(getFollowedByUserIds(_))
.getOrElse(Seq.empty)
FeatureMapBuilder() FeatureMapBuilder()
.add(GraphTwoHopFeature, transformedValue) .add(GraphTwoHopFeature, transformedValue)

View File

@ -3,7 +3,6 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -12,8 +11,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@ -21,35 +19,32 @@ import javax.inject.Singleton
@Singleton @Singleton
case class ImpressionBloomFilterQueryFeatureHydrator[ case class ImpressionBloomFilterQueryFeatureHydrator[
Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
bloomFilterClient: ManhattanStoreClient[ bloomFilter: ImpressionBloomFilter)
blm.ImpressionBloomFilterKey, extends QueryFeatureHydrator[Query] {
blm.ImpressionBloomFilterSeq
]) extends QueryFeatureHydrator[Query] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
"ImpressionBloomFilter") "ImpressionBloomFilter")
private val ImpressionBloomFilterTTL = 7.day private val ImpressionBloomFilterTTL = 7.day
private val ImpressionBloomFilterFalsePositiveRate = 0.002
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature) override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
private val SurfaceArea = blm.SurfaceArea.HomeTimeline private val SurfaceArea = t.SurfaceArea.HomeTimeline
override def hydrate(query: Query): Stitch[FeatureMap] = { override def hydrate(query: Query): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId val userId = query.getRequiredUserId
bloomFilterClient bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq =>
.get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
.map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
.map { bloomFilterSeq =>
val updatedBloomFilterSeq = val updatedBloomFilterSeq =
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
else { else {
ImpressionBloomFilter.addSeenTweetIds( bloomFilter.addElements(
userId = userId,
surfaceArea = SurfaceArea, surfaceArea = SurfaceArea,
tweetIds = query.seenTweetIds.get, tweetIds = query.seenTweetIds.get,
bloomFilterSeq = bloomFilterSeq, bloomFilterEntrySeq = bloomFilterSeq,
timeToLive = ImpressionBloomFilterTTL, timeToLive = ImpressionBloomFilterTTL,
falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam) falsePositiveRate = ImpressionBloomFilterFalsePositiveRate
) )
} }
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build() FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()

View File

@ -1,41 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object InNetworkFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
val featureMaps = candidates.map { candidate =>
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
isSelfTweet || followedUserIds.contains(authorId)
}
FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
}
Stitch.value(featureMaps)
}
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.request.HasListId import com.twitter.home_mixer.model.request.HasListId
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -15,18 +15,17 @@ import com.twitter.stitch.socialgraph.SocialGraph
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
case object RecentListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { case object ListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] {
override val defaultValue: Seq[Long] = Seq.empty override val defaultValue: Seq[Long] = Seq.empty
} }
@Singleton @Singleton
class RecentListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) class ListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph)
extends QueryFeatureHydrator[PipelineQuery with HasListId] { extends QueryFeatureHydrator[PipelineQuery with HasListId] {
override val identifier: FeatureHydratorIdentifier = override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListMembers")
FeatureHydratorIdentifier("RecentListMembers")
override val features: Set[Feature[_, _]] = Set(RecentListMembersFeature) override val features: Set[Feature[_, _]] = Set(ListMembersFeature)
private val MaxRecentMembers = 10 private val MaxRecentMembers = 10
@ -37,7 +36,7 @@ class RecentListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph)
pageRequest = Some(sg.PageRequest(selectAll = Some(true), count = Some(MaxRecentMembers))) pageRequest = Some(sg.PageRequest(selectAll = Some(true), count = Some(MaxRecentMembers)))
) )
socialGraph.ids(request).map(_.ids).map { listMembers => socialGraph.ids(request).map(_.ids).map { listMembers =>
FeatureMapBuilder().add(RecentListMembersFeature, listMembers).build() FeatureMapBuilder().add(ListMembersFeature, listMembers).build()
} }
} }
} }

View File

@ -1,4 +1,5 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer
package functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
@ -13,11 +14,11 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.keyvalue.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.util.Future import com.twitter.util.Future
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Named import javax.inject.Named
import javax.inject.Singleton import javax.inject.Singleton
@ -44,17 +45,25 @@ class MetricCenterUserCountingFeatureHydrator @Inject() (
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { ): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val possiblyAuthorIds = extractKeys(candidates) val possiblyAuthorIds = extractKeys(candidates)
val userIds = possiblyAuthorIds.flatten val userIds = possiblyAuthorIds.flatten
val response: Future[KeyValueResult[Long, rf.MCUserCountingFeatures]] = val response: Future[KeyValueResult[Long, rf.MCUserCountingFeatures]] = if (userIds.isEmpty) {
if (userIds.isEmpty) Future.value(KeyValueResult.empty) else client(userIds) Future.value(KeyValueResult.empty)
} else {
client(userIds)
}
response.map { result => response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId => possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result) val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
FeatureMapBuilder().add(MetricCenterUserCountingFeature, value).build()
FeatureMapBuilder()
.add(MetricCenterUserCountingFeature, value)
.build()
}
} }
} }
} }

View File

@ -2,10 +2,8 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ForYouProduct
@ -29,27 +27,17 @@ import javax.inject.Singleton
@Singleton @Singleton
case class PersistenceStoreQueryFeatureHydrator @Inject() ( case class PersistenceStoreQueryFeatureHydrator @Inject() (
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3], timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery] { extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
private val WhoToFollowExcludedUserIdsLimit = 1000 private val WhoToFollowExcludedUserIdsLimit = 1000
private val ServedTweetIdsDuration = 10.minutes private val ServedTweetIdsDuration = 1.hour
private val ServedTweetIdsLimit = 100 private val ServedTweetIdsLimit = 100
private val ServedTweetPreviewIdsDuration = 10.hours
private val ServedTweetPreviewIdsLimit = 10
override val features: Set[Feature[_, _]] = override val features: Set[Feature[_, _]] =
Set( Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature)
ServedTweetIdsFeature,
ServedTweetPreviewIdsFeature,
PersistenceEntriesFeature,
WhoToFollowExcludedUserIdsFeature)
private val supportedClients = Seq( private val supportedClients = Seq(
ClientPlatform.IPhone, ClientPlatform.IPhone,
@ -92,19 +80,8 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
.flatMap( .flatMap(
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit)) _.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
servedTweetIdsSizeStat.add(servedTweetIds.size)
val servedTweetPreviewIds = timelineResponses
.filter(_.clientPlatform == clientPlatform)
.filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
.sortBy(-_.servedTime.inMilliseconds)
.flatMap(_.entries
.filter(_.entityIdType == EntityIdType.TweetPreview)
.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
FeatureMapBuilder() FeatureMapBuilder()
.add(ServedTweetIdsFeature, servedTweetIds) .add(ServedTweetIdsFeature, servedTweetIds)
.add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
.add(PersistenceEntriesFeature, timelineResponses) .add(PersistenceEntriesFeature, timelineResponses)
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds) .add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
.build() .build()

View File

@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.stitch.timelineservice.TimelineService import com.twitter.stitch.timelineservice.TimelineService
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
@ -38,10 +37,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { ): Stitch[Seq[FeatureMap]] = {
val engagingUserIdtoTweetId = candidates.flatMap { candidate => val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
candidate.features candidate.features
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) .get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.map(favoritedBy => favoritedBy -> candidate.candidate.id) .map(favoritedBy => favoritedBy -> candidate.candidate.id)
} }
@ -60,7 +59,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
candidates.map { candidate => candidates.map { candidate =>
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) .get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) } .filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
FeatureMapBuilder() FeatureMapBuilder()

View File

@ -32,15 +32,11 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
val realGraphScoresFeatures = realGraphFollowedUsers val realGraphScoresFeatures = realGraphFollowedUsers
.getOrElse(Seq.empty) .getOrElse(Seq.empty)
.sortBy(-_.score) .sortBy(-_.score)
.map(candidate => candidate.userId -> scaleScore(candidate.score)) .map(candidate => candidate.userId -> candidate.score)
.take(RealGraphCandidateCount) .take(RealGraphCandidateCount)
.toMap .toMap
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build() FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
} }
} }
// Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
private def scaleScore(score: Double): Double =
if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
} }

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphFeatureRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphFeatureRepository
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature

View File

@ -1,8 +1,8 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures
import com.twitter.home_mixer.util.MissingKeyException import com.twitter.home_mixer.util.MissingKeyException
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -14,7 +14,6 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter
import com.twitter.timelines.prediction.adapters.real_graph.RealGraphFeaturesAdapter import com.twitter.timelines.prediction.adapters.real_graph.RealGraphFeaturesAdapter
@ -60,13 +59,13 @@ class RealGraphViewerAuthorFeatureHydrator @Inject() ()
query: PipelineQuery, query: PipelineQuery,
candidate: TweetCandidate, candidate: TweetCandidate,
existingFeatures: FeatureMap existingFeatures: FeatureMap
): Stitch[FeatureMap] = OffloadFuturePools.offload { ): Stitch[FeatureMap] = {
val viewerId = query.getRequiredUserId val viewerId = query.getRequiredUserId
val realGraphFeatures = query.features val realGraphFeatures = query.features
.flatMap(_.getOrElse(RealGraphFeatures, None)) .flatMap(_.getOrElse(RealGraphFeatures, None))
.getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures])
existingFeatures.getOrElse(AuthorIdFeature, None) match { val result: FeatureMap = existingFeatures.getOrElse(AuthorIdFeature, None) match {
case Some(authorId) => case Some(authorId) =>
val realGraphAuthorFeatures = val realGraphAuthorFeatures =
getRealGraphViewerAuthorFeatures(viewerId, authorId, realGraphFeatures) getRealGraphViewerAuthorFeatures(viewerId, authorId, realGraphFeatures)
@ -91,6 +90,7 @@ class RealGraphViewerAuthorFeatureHydrator @Inject() ()
.build() .build()
case _ => MissingKeyFeatureMap case _ => MissingKeyFeatureMap
} }
Stitch(result)
} }
private def getRealGraphViewerAuthorFeatures( private def getRealGraphViewerAuthorFeatures(

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter
import com.twitter.timelines.real_graph.v1.{thriftscala => v1} import com.twitter.timelines.real_graph.v1.{thriftscala => v1}
@ -44,24 +43,26 @@ class RealGraphViewerRelatedUsersFeatureHydrator @Inject() ()
query: PipelineQuery, query: PipelineQuery,
candidate: TweetCandidate, candidate: TweetCandidate,
existingFeatures: FeatureMap existingFeatures: FeatureMap
): Stitch[FeatureMap] = OffloadFuturePools.offload { ): Stitch[FeatureMap] = {
val realGraphQueryFeatures = query.features val realGraphQueryFeatures = query.features
.flatMap(_.getOrElse(RealGraphFeatures, None)) .flatMap(_.getOrElse(RealGraphFeatures, None))
.getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures])
val allRelatedUserIds = getRelatedUserIds(existingFeatures) val allRelatedUserIds = getRelatedUserIds(existingFeatures)
val realGraphFeatures = RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures( val realGraphFeatures =
RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures(
allRelatedUserIds, allRelatedUserIds,
realGraphQueryFeatures realGraphQueryFeatures)
)
val realGraphFeaturesDataRecord = RealGraphEdgeFeaturesCombineAdapter val realGraphFeaturesDataRecord = RealGraphEdgeFeaturesCombineAdapter
.adaptToDataRecords(Some(realGraphFeatures)).asScala.headOption .adaptToDataRecords(Some(realGraphFeatures)).asScala.headOption
.getOrElse(new DataRecord) .getOrElse(new DataRecord)
Stitch.value {
FeatureMapBuilder() FeatureMapBuilder()
.add(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord) .add(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord)
.build() .build()
} }
}
private def getRelatedUserIds(features: FeatureMap): Seq[Long] = { private def getRelatedUserIds(features: FeatureMap): Seq[Long] = {
(CandidatesUtil.getEngagerUserIds(features) ++ (CandidatesUtil.getEngagerUserIds(features) ++

View File

@ -1,22 +1,22 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.realtime_interaction_graph.RealTimeInteractionGraphFeaturesAdapter import com.twitter.timelines.prediction.adapters.realtime_interaction_graph.RealTimeInteractionGraphFeaturesAdapter
import com.twitter.timelines.prediction.features.realtime_interaction_graph.RealTimeInteractionGraphEdgeFeatures import com.twitter.timelines.prediction.features.realtime_interaction_graph.RealTimeInteractionGraphEdgeFeatures
import com.twitter.util.Time import com.twitter.util.Time
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
import scala.collection.JavaConverters._ import scala.collection.JavaConverters._
@ -31,8 +31,8 @@ object RealTimeInteractionGraphEdgeFeature
class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() () class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() ()
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
FeatureHydratorIdentifier("RealTimeInteractionGraphEdge") "RealTimeInteractionGraphEdge")
override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphEdgeFeature) override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphEdgeFeature)
@ -41,21 +41,24 @@ class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() ()
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { ): Stitch[Seq[FeatureMap]] = {
val userVertex = val userVertex =
query.features.flatMap(_.getOrElse(RealTimeInteractionGraphUserVertexQueryFeature, None)) query.features.flatMap(_.getOrElse(RealTimeInteractionGraphUserVertexQueryFeature, None))
val realTimeInteractionGraphFeaturesMap = val realTimeInteractionGraphFeaturesMap =
userVertex.map(RealTimeInteractionGraphEdgeFeatures(_, Time.now)) userVertex.map(RealTimeInteractionGraphEdgeFeatures(_, Time.now))
Stitch.value {
candidates.map { candidate => candidates.map { candidate =>
val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId => val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId =>
realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId)) realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId))
} }
val dataRecordFeature = FeatureMapBuilder()
realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head .add(
RealTimeInteractionGraphEdgeFeature,
FeatureMapBuilder().add(RealTimeInteractionGraphEdgeFeature, dataRecordFeature).build() realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head)
.build()
}
} }
} }
} }

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.google.inject.name.Named import com.google.inject.name.Named
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
@ -13,6 +13,7 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.servo.cache.ReadCache import com.twitter.servo.cache.ReadCache
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,56 +1,28 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.ContentFeatures
import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.InReplyToContentFeatureAdapter
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.InReplyToEarlybirdAdapter
import com.twitter.home_mixer.util.ReplyRetweetUtil import com.twitter.home_mixer.util.ReplyRetweetUtil
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures
import com.twitter.snowflake.id.SnowflakeId import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures
import com.twitter.timelines.conversation_features.{thriftscala => cf}
import com.twitter.timelines.prediction.adapters.conversation_features.ConversationFeaturesAdapter
import com.twitter.util.Duration import com.twitter.util.Duration
import com.twitter.util.Time import com.twitter.util.Time
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
import scala.collection.JavaConverters._
object InReplyToTweetHydratedEarlybirdFeature object InReplyToTweetHydratedEarlybirdFeature
extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] extends Feature[TweetCandidate, Option[ThriftTweetFeatures]]
object ConversationDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object InReplyToEarlybirdDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object InReplyToTweetypieContentDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
/** /**
* The purpose of this hydrator is to * The purpose of this hydrator is to
* 1) hydrate simple features into replies and their ancestor tweets * 1) hydrate simple features into replies and their ancestor tweets
@ -64,19 +36,13 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver)
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ReplyTweet") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ReplyTweet")
override val features: Set[Feature[_, _]] = Set( override val features: Set[Feature[_, _]] = Set(
ConversationDataRecordFeature, ConversationFeature,
InReplyToTweetHydratedEarlybirdFeature, InReplyToTweetHydratedEarlybirdFeature
InReplyToEarlybirdDataRecordFeature,
InReplyToTweetypieContentDataRecordFeature
) )
private val defaulDataRecord: DataRecord = new DataRecord()
private val DefaultFeatureMap = FeatureMapBuilder() private val DefaultFeatureMap = FeatureMapBuilder()
.add(ConversationDataRecordFeature, defaulDataRecord) .add(ConversationFeature, None)
.add(InReplyToTweetHydratedEarlybirdFeature, None) .add(InReplyToTweetHydratedEarlybirdFeature, None)
.add(InReplyToEarlybirdDataRecordFeature, defaulDataRecord)
.add(InReplyToTweetypieContentDataRecordFeature, defaulDataRecord)
.build() .build()
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
@ -86,7 +52,7 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver)
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { ): Stitch[Seq[FeatureMap]] = {
val replyToInReplyToTweetMap = val replyToInReplyToTweetMap =
ReplyRetweetUtil.replyTweetIdToInReplyToTweetMap(candidates) ReplyRetweetUtil.replyTweetIdToInReplyToTweetMap(candidates)
val candidatesWithRepliesHydrated = candidates.map { candidate => val candidatesWithRepliesHydrated = candidates.map { candidate =>
@ -128,28 +94,16 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver)
inReplyToTweetEarlyBirdFeature, inReplyToTweetEarlyBirdFeature,
updatedReplyConversationFeatures)) updatedReplyConversationFeatures))
} }
Stitch.value(
candidatesWithRepliesAndAncestorTweetsHydrated.map { candidatesWithRepliesAndAncestorTweetsHydrated.map {
case (candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) => case (candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) =>
val conversationDataRecordFeature = updatedConversationFeatures
.map(f => ConversationFeaturesAdapter.adaptToDataRecord(cf.ConversationFeatures.V1(f)))
.getOrElse(defaulDataRecord)
val inReplyToEarlybirdDataRecord =
InReplyToEarlybirdAdapter
.adaptToDataRecords(inReplyToTweetEarlyBirdFeature).asScala.head
val inReplyToContentDataRecord = InReplyToContentFeatureAdapter
.adaptToDataRecords(
inReplyToTweetEarlyBirdFeature.map(ContentFeatures.fromThrift)).asScala.head
FeatureMapBuilder() FeatureMapBuilder()
.add(ConversationDataRecordFeature, conversationDataRecordFeature) .add(ConversationFeature, updatedConversationFeatures)
.add(InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature) .add(InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature)
.add(InReplyToEarlybirdDataRecordFeature, inReplyToEarlybirdDataRecord)
.add(InReplyToTweetypieContentDataRecordFeature, inReplyToContentDataRecord)
.build() .build()
case _ => DefaultFeatureMap case _ => DefaultFeatureMap
} }
)
} }
private def hydratedReplyCandidate( private def hydratedReplyCandidate(

View File

@ -17,13 +17,9 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
import com.twitter.search.common.util.lang.ThriftLanguageUtil import com.twitter.search.common.util.lang.ThriftLanguageUtil
import com.twitter.snowflake.id.SnowflakeId import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
import java.util.UUID import java.util.UUID
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@ -49,9 +45,6 @@ class RequestQueryFeatureHydrator[
PullToRefreshFeature, PullToRefreshFeature,
RequestJoinIdFeature, RequestJoinIdFeature,
ServedRequestIdFeature, ServedRequestIdFeature,
TimestampFeature,
TimestampGMTDowFeature,
TimestampGMTHourFeature,
ViewerIdFeature ViewerIdFeature
) )
@ -74,7 +67,6 @@ class RequestQueryFeatureHydrator[
override def hydrate(query: Query): Stitch[FeatureMap] = { override def hydrate(query: Query): Stitch[FeatureMap] = {
val requestContext = query.deviceContext.flatMap(_.requestContextValue) val requestContext = query.deviceContext.flatMap(_.requestContextValue)
val servedRequestId = UUID.randomUUID.getMostSignificantBits val servedRequestId = UUID.randomUUID.getMostSignificantBits
val timestamp = query.queryTime.inMilliseconds
val featureMap = FeatureMapBuilder() val featureMap = FeatureMapBuilder()
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt)) .add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
@ -105,15 +97,8 @@ class RequestQueryFeatureHydrator[
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh)) .add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
.add(ServedRequestIdFeature, Some(servedRequestId)) .add(ServedRequestIdFeature, Some(servedRequestId))
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId)) .add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
.add(TimestampFeature, timestamp)
.add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
.add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
.add(HasDarkRequestFeature, hasDarkRequest) .add(HasDarkRequestFeature, hasDarkRequest)
.add( .add(ViewerIdFeature, query.getRequiredUserId)
ViewerIdFeature,
query.getOptionalUserId
.orElse(query.getGuestId).getOrElse(
throw PipelineFailure(BadRequest, "Missing viewer id")))
.build() .build()
Stitch.value(featureMap) Stitch.value(featureMap)

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature
@ -22,8 +22,8 @@ object SourceTweetEarlybirdFeature extends Feature[TweetCandidate, Option[Thrift
object RetweetSourceTweetFeatureHydrator object RetweetSourceTweetFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
FeatureHydratorIdentifier("RetweetSourceTweet") "RetweetSourceTweet")
override val features: Set[Feature[_, _]] = Set( override val features: Set[Feature[_, _]] = Set(
SourceTweetEarlybirdFeature, SourceTweetEarlybirdFeature,

View File

@ -0,0 +1,46 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton
case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
socialGraphStitchClient: SocialGraphStitchClient)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SGSFollowedUsers")
override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
private val SocialGraphLimit = 14999
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
val request = sg.IdsRequest(
relationships = Seq(
sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
),
pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
)
socialGraphStitchClient
.ids(request).map(_.ids)
.map { followedUsers =>
FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
}
}
}

View File

@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph import com.twitter.stitch.socialgraph.SocialGraph
@ -42,7 +41,8 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { ): Stitch[Seq[FeatureMap]] = {
val allSocialContextUserIds = val allSocialContextUserIds =
candidates.flatMap { candidate => candidates.flatMap { candidate =>
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++ candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++

View File

@ -1,5 +1,6 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -13,7 +14,6 @@ import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient
import com.twitter.timelines.configapi.decider.BooleanDeciderParam import com.twitter.timelines.configapi.decider.BooleanDeciderParam
@ -29,7 +29,8 @@ object SimClustersEngagementSimilarityFeature
@Singleton @Singleton
class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( class SimClustersEngagementSimilarityFeatureHydrator @Inject() (
simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient) simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient,
statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery] { with Conditionally[PipelineQuery] {
@ -38,6 +39,10 @@ class SimClustersEngagementSimilarityFeatureHydrator @Inject() (
override val features: Set[Feature[_, _]] = Set(SimClustersEngagementSimilarityFeature) override val features: Set[Feature[_, _]] = Set(SimClustersEngagementSimilarityFeature)
private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
private val hydratedCandidatesSizeStat = scopedStatsReceiver.stat("hydratedCandidatesSize")
private val simClustersRecentEngagementSimilarityFeaturesAdapter = private val simClustersRecentEngagementSimilarityFeaturesAdapter =
new SimClustersRecentEngagementSimilarityFeaturesAdapter new SimClustersRecentEngagementSimilarityFeaturesAdapter
@ -50,14 +55,15 @@ class SimClustersEngagementSimilarityFeatureHydrator @Inject() (
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { ): Stitch[Seq[FeatureMap]] = {
val tweetToCandidates = candidates.map(candidate => candidate.candidate.id -> candidate).toMap val tweetToCandidates = candidates.map(candidate => candidate.candidate.id -> candidate).toMap
val tweetIds = tweetToCandidates.keySet.toSeq val tweetIds = tweetToCandidates.keySet.toSeq
val userId = query.getRequiredUserId val userId = query.getRequiredUserId
val userTweetEdges = tweetIds.map(tweetId => (userId, tweetId)) val userTweetEdges = tweetIds.map(tweetId => (userId, tweetId))
simClustersEngagementSimilarityClient val resultFuture = simClustersEngagementSimilarityClient
.getSimClustersRecentEngagementSimilarityScores(userTweetEdges).map { .getSimClustersRecentEngagementSimilarityScores(userTweetEdges).map {
simClustersRecentEngagementSimilarityScoresMap => simClustersRecentEngagementSimilarityScoresMap =>
hydratedCandidatesSizeStat.add(simClustersRecentEngagementSimilarityScoresMap.size)
candidates.map { candidate => candidates.map { candidate =>
val similarityFeatureOpt = simClustersRecentEngagementSimilarityScoresMap val similarityFeatureOpt = simClustersRecentEngagementSimilarityScoresMap
.get(userId -> candidate.candidate.id).flatten .get(userId -> candidate.candidate.id).flatten
@ -71,5 +77,7 @@ class SimClustersEngagementSimilarityFeatureHydrator @Inject() (
.build() .build()
} }
} }
Stitch.callFuture(resultFuture)
} }
} }

View File

@ -0,0 +1,67 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SocialGraphService")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
val idsRequest = createIdsRequest(
userId = viewerId,
relationshipTypes = Set(sg.RelationshipType.Following),
targetIds = Some(distinctNonSelfAuthorIds)
)
socialGraphStitchClient
.ids(request = idsRequest, requestContext = None)
.map { idResult =>
authorIds.map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
val inNetworkAuthorIds = idResult.ids.toSet
val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
}
}
}
private def createIdsRequest(
userId: Long,
relationshipTypes: Set[sg.RelationshipType],
targetIds: Option[Seq[Long]] = None
): sg.IdsRequest = sg.IdsRequest(
relationshipTypes.map { relationshipType =>
sg.SrcRelationship(userId, relationshipType, targets = targetIds)
}.toSeq,
Some(sg.PageRequest(selectAll = Some(true)))
)
}

View File

@ -0,0 +1,162 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.contentrecommender.{thriftscala => cr}
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
import com.twitter.topiclisting.TopicListingViewerContext
import com.twitter.tsp.{thriftscala => tsp}
import javax.inject.Inject
import javax.inject.Singleton
import scala.collection.JavaConverters._
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
object TSPInferredTopicDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TSPInferredTopicFeatureHydrator @Inject() (
topicSocialProofClientColumn: TopicSocialProofClientColumn,
statsReceiver: StatsReceiver,
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
override val features: Set[Feature[_, _]] =
Set(
TSPInferredTopicFeature,
TSPInferredTopicDataRecordFeature,
TopicIdSocialContextFeature,
TopicContextFunctionalityTypeFeature)
private val topK = 3
private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
sid.CandidateTweetSourceId.Simcluster,
sid.CandidateTweetSourceId.CroonTweet
)
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
private val DefaultFeatureMap = FeatureMapBuilder()
.add(TSPInferredTopicFeature, Map.empty[Long, Double])
.add(TSPInferredTopicDataRecordFeature, new DataRecord())
.add(TopicIdSocialContextFeature, None)
.add(TopicContextFunctionalityTypeFeature, None)
.build()
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val tags = candidates.collect {
case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
candidate.candidate.id -> candidate.features
.getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
}.toMap
val topicSocialProofRequest =
tsp.TopicSocialProofRequest(
userId = query.getRequiredUserId,
tweetIds = candidates.map(_.candidate.id).toSet,
displayLocation = cr.DisplayLocation.HomeTimeline,
topicListingSetting = tsp.TopicListingSetting.Followable,
context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
bypassModes = None,
// Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
tags = if (tags.isEmpty) None else Some(tags)
)
topicSocialProofClientColumn.fetcher
.fetch(topicSocialProofRequest)
.map(_.v)
.map {
case Some(response) =>
candidates.map { candidate =>
val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
if (topicWithScores.nonEmpty) {
keyFoundCounter.incr()
val (socialProofId, socialProofFunctionalityType) =
if (candidate.features
.getOrElse(CandidateSourceIdFeature, None)
.exists(sourcesToSetSocialProof.contains)) {
getSocialProof(topicWithScores)
} else {
(None, None)
}
val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
val inferredTopicDataRecord =
InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
FeatureMapBuilder()
.add(TSPInferredTopicFeature, inferredTopicFeatures)
.add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
.add(TopicIdSocialContextFeature, socialProofId)
.add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
.build()
} else {
keyLossCounter.incr()
DefaultFeatureMap
}
}
case _ =>
requestFailCounter.incr()
candidates.map { _ =>
DefaultFeatureMap
}
}
}
private def getSocialProof(
topicWithScores: Seq[tsp.TopicWithScore]
): (Option[Long], Option[TopicContextFunctionalityType]) = {
val followingTopicId = topicWithScores
.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
topicId
}
if (followingTopicId.nonEmpty) {
return (followingTopicId, Some(BasicTopicContextFunctionalityType))
}
val implicitFollowingId = topicWithScores.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
topicId
}
if (implicitFollowingId.nonEmpty) {
return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
}
(None, None)
}
private def convertTopicWithScores(
topicWithScores: Seq[tsp.TopicWithScore],
): Map[Long, Double] = {
topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
}
}

View File

@ -0,0 +1,251 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.FDsl._
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.search.common.features.{thriftscala => sc}
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
import com.twitter.timelines.prediction.features.time_features.TimeFeatures
import com.twitter.util.Duration
import scala.collection.Searching._
object TimeFeaturesDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
Stitch.value {
val richDataRecord = new RichDataRecord()
setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
FeatureMapBuilder()
.add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
.build()
}
}
private def setTimeFeatures(
richDataRecord: RichDataRecord,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
query: PipelineQuery,
): Unit = {
val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
}
private[feature_hydrator] def getTimeFeatures(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
): Option[TimeFeatures] = {
for {
requestTimestampMs <- Some(query.queryTime.inMilliseconds)
tweetId <- Some(candidate.id)
viewerId <- query.getOptionalUserId
tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
}
timeSinceSourceTweetCreation =
existingFeatures
.getOrElse(SourceTweetIdFeature, None)
.flatMap { sourceTweetId =>
timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
requestTimestampMs - sourceTweetCreationTimeMs
}
}
.getOrElse(timeSinceTweetCreation)
if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
} yield {
val timeFeatures = TimeFeatures(
timeSinceTweetCreation = timeSinceTweetCreation,
timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
)
val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
existingFeatures.getOrElse(EarlybirdFeature, None),
timeFeatures,
timeSinceSourceTweetCreation
).getOrElse(timeFeatures)
val nonPollingTimestampsMs =
query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
timeFeaturesWithLastEngagement,
requestTimestampMs,
tweetCreationTimeMs,
nonPollingTimestampsMs
)
timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
}
}
private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
if (SnowflakeId.isSnowflakeId(tweetOrUserId))
Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
else None
}
private def addLastEngagementTimeFeatures(
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
timeFeatures: TimeFeatures,
timeSinceSourceTweetCreation: Long
): Option[TimeFeatures] = {
tweetFeaturesOpt.map { tweetFeatures =>
val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
timeFeatures.copy(
lastFavSinceCreationHrs = lastFavSinceCreationHrs,
lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
lastReplySinceCreationHrs = lastReplySinceCreationHrs,
lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
lastFavSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
lastRetweetSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
lastReplySinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
lastQuoteSinceCreationHrs,
timeSinceSourceTweetCreation
)
)
}
}
private def addNonPollingTimeFeatures(
timeFeatures: TimeFeatures,
requestTimestampMs: Long,
creationTimeMs: Long,
nonPollingTimestampsMs: Option[Seq[Long]]
): Option[TimeFeatures] = {
for {
nonPollingTimestampsMs <- nonPollingTimestampsMs
lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
} yield {
val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
1.0,
timeFeatures.timeSinceTweetCreation
)
/*
* Non-polling timestamps are stored in chronological order.
* The latest timestamps occur first, therefore we need to explicitly search in reverse order.
*/
val nonPollingRequestsSinceTweetCreation =
if (nonPollingTimestampsMs.nonEmpty) {
nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
} else {
0
}
/*
* Calculate the average time between non-polling requests; include
* request time in this calculation as latest timestamp.
*/
val timeBetweenNonPollingRequestsAvg =
(requestTimestampMs - earliestNonPollingTimestampMs) / math
.max(1.0, nonPollingTimestampsMs.size)
val timeFeaturesWithNonPolling = timeFeatures.copy(
timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
tweetAgeRatio = Some(tweetAgeRatio)
)
timeFeaturesWithNonPolling
}
}
private[this] def getTimeSinceLastEngagementHrs(
lastEngagementTimeSinceCreationHrsOpt: Option[Double],
timeSinceTweetCreation: Long
): Option[Double] = {
lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
}
}
private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
val record = richDataRecord.getRecord
.setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
.setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
.setFeatureValueFromOption(
TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
features.timeSinceViewerAccountCreationSecs)
.setFeatureValue(
USER_ID_IS_SNOWFLAKE_ID,
features.timeSinceViewerAccountCreationSecs.isDefined
)
.setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
.setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
.setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
.setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
.setFeatureValueFromOption(
LAST_RETWEET_SINCE_CREATION_HRS,
features.lastRetweetSinceCreationHrs
)
.setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
.setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
/*
* set features whose values are optional as some users do not have non-polling timestamps
*/
features.timeBetweenNonPollingRequestsAvg.foreach(
record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
)
features.timeSinceLastNonPollingRequest.foreach(
record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
)
features.nonPollingRequestsSinceTweetCreation.foreach(
record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
)
features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
}
}

View File

@ -1,7 +1,6 @@
package com.twitter.home_mixer.product.for_you.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller
import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature
import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.DeviceContext
import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext
import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.service.HomeMixerAlertConfig
@ -17,6 +16,8 @@ import com.twitter.timelineservice.{thriftscala => t}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton @Singleton
case class TimelineServiceTweetsQueryFeatureHydrator @Inject() ( case class TimelineServiceTweetsQueryFeatureHydrator @Inject() (
timelineService: TimelineService, timelineService: TimelineService,

View File

@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
extends QueryFeatureHydrator[Query] { extends QueryFeatureHydrator[Query] {
private val TweetImpressionTTL = 2.days private val TweetImpressionTTL = 1.day
private val TweetImpressionCap = 5000 private val TweetImpressionCap = 3000
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.CandidatesUtil
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures
import java.lang.{Long => JLong} import java.lang.{Long => JLong}
@ -37,10 +36,16 @@ object TweetMetaDataFeatureHydrator
query: PipelineQuery, query: PipelineQuery,
candidate: TweetCandidate, candidate: TweetCandidate,
existingFeatures: FeatureMap existingFeatures: FeatureMap
): Stitch[FeatureMap] = OffloadFuturePools.offload { ): Stitch[FeatureMap] = {
val richDataRecord = new RichDataRecord() val richDataRecord = new RichDataRecord()
setFeatures(richDataRecord, candidate, existingFeatures) setFeatures(richDataRecord, candidate, existingFeatures)
FeatureMapBuilder().add(TweetMetaDataDataRecord, richDataRecord.getRecord).build()
Stitch.value {
FeatureMapBuilder()
.add(TweetMetaDataDataRecord, richDataRecord.getRecord)
.build()
}
} }
private def setFeatures( private def setFeatures(

View File

@ -1,12 +1,12 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.escherbird.{thriftscala => esb} import com.twitter.escherbird.{thriftscala => esb}
import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.Stat
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ContentFeatureAdapter
import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.ContentFeatureAdapter
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.home_mixer.util.tweetypie.content.FeatureExtractionHelper import com.twitter.home_mixer.util.tweetypie.content.FeatureExtractionHelper
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
@ -64,18 +64,21 @@ class TweetypieContentFeatureHydrator @Inject() (
private val bulkPostTransformerLatencyStat = private val bulkPostTransformerLatencyStat =
statsReceiver.scope(statScope).scope("bulkPostTransformer").stat("latency_ms") statsReceiver.scope(statScope).scope("bulkPostTransformer").stat("latency_ms")
private val DefaultDataRecord: DataRecord = new DataRecord()
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { ): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val tweetIdsToHydrate = candidates.map(getCandidateOriginalTweetId).distinct val tweetIdsToHydrate = candidates.map(getCandidateOriginalTweetId).distinct
val response: Future[KeyValueResult[Long, tp.Tweet]] = Stat.timeFuture(bulkRequestLatencyStat) { val response: Future[KeyValueResult[Long, tp.Tweet]] =
if (tweetIdsToHydrate.isEmpty) Future.value(KeyValueResult.empty) Stat.timeFuture(bulkRequestLatencyStat)(
else client(tweetIdsToHydrate) if (tweetIdsToHydrate.isEmpty) {
Future.value(KeyValueResult.empty)
} else {
client(tweetIdsToHydrate)
} }
)
response.flatMap { result => response.flatMap { result =>
Stat.timeFuture(bulkPostTransformerLatencyStat) { Stat.timeFuture(bulkPostTransformerLatencyStat) {
@ -84,7 +87,7 @@ class TweetypieContentFeatureHydrator @Inject() (
candidates, candidates,
parTransformer(result, _), parTransformer(result, _),
parallelism = 32, parallelism = 32,
default = Return((Seq.empty, DefaultDataRecord)) default = Return((Seq.empty, new DataRecord))
).map { ).map {
_.map { _.map {
case Return(result) => case Return(result) =>
@ -102,12 +105,14 @@ class TweetypieContentFeatureHydrator @Inject() (
} }
} }
} }
}
private def parTransformer( private def parTransformer(
result: KeyValueResult[Long, tp.Tweet], result: KeyValueResult[Long, tp.Tweet],
candidate: CandidateWithFeatures[TweetCandidate] candidate: CandidateWithFeatures[TweetCandidate]
): Try[(Seq[Long], DataRecord)] = { ): Try[(Seq[Long], DataRecord)] = {
val originalTweetId = Some(getCandidateOriginalTweetId(candidate)) val originalTweetId = Some(getCandidateOriginalTweetId(candidate))
val value = observedGet(key = originalTweetId, keyValueResult = result) val value = observedGet(key = originalTweetId, keyValueResult = result)
Stat.time(postTransformerLatencyStat)(postTransformer(value)) Stat.time(postTransformerLatencyStat)(postTransformer(value))
} }

View File

@ -1,9 +1,6 @@
package com.twitter.home_mixer.functional_component.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
@ -13,13 +10,11 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.util.tweetypie.RequestFields import com.twitter.home_mixer.util.tweetypie.RequestFields
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
@ -34,22 +29,17 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
import com.twitter.tweetypie.{thriftscala => tp} import com.twitter.tweetypie.{thriftscala => tp}
import com.twitter.util.logging.Logging
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@Singleton @Singleton
class TweetypieFeatureHydrator @Inject() ( class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
tweetypieStitchClient: TweetypieStitchClient, extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
statsReceiver: StatsReceiver)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Logging {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
override val features: Set[Feature[_, _]] = Set( override val features: Set[Feature[_, _]] = Set(
AuthorIdFeature, AuthorIdFeature,
ExclusiveConversationAuthorIdFeature,
InReplyToTweetIdFeature, InReplyToTweetIdFeature,
IsHydratedFeature, IsHydratedFeature,
IsNsfw, IsNsfw,
@ -61,7 +51,6 @@ class TweetypieFeatureHydrator @Inject() (
SourceTweetIdFeature, SourceTweetIdFeature,
SourceUserIdFeature, SourceUserIdFeature,
TweetTextFeature, TweetTextFeature,
TweetLanguageFeature,
VisibilityReason VisibilityReason
) )
@ -81,12 +70,9 @@ class TweetypieFeatureHydrator @Inject() (
): Stitch[FeatureMap] = { ): Stitch[FeatureMap] = {
val safetyLevel = query.product match { val safetyLevel = query.product match {
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
case ForYouProduct => case ForYouProduct => rtf.SafetyLevel.TimelineHome
val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown") case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
} }
@ -96,12 +82,9 @@ class TweetypieFeatureHydrator @Inject() (
includeQuotedTweet = true, includeQuotedTweet = true,
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
safetyLevel = Some(safetyLevel), safetyLevel = Some(safetyLevel),
forUserId = query.getOptionalUserId forUserId = Some(query.getRequiredUserId)
) )
val exclusiveAuthorIdOpt =
existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map { tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) => case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
val coreData = found.tweet.coreData val coreData = found.tweet.coreData
@ -123,7 +106,6 @@ class TweetypieFeatureHydrator @Inject() (
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)) found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
val tweetText = coreData.map(_.text) val tweetText = coreData.map(_.text)
val tweetLanguage = found.tweet.language.map(_.language)
val tweetAuthorId = coreData.map(_.userId) val tweetAuthorId = coreData.map(_.userId)
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)) val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
@ -145,7 +127,6 @@ class TweetypieFeatureHydrator @Inject() (
FeatureMapBuilder() FeatureMapBuilder()
.add(AuthorIdFeature, tweetAuthorId) .add(AuthorIdFeature, tweetAuthorId)
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, inReplyToTweetId) .add(InReplyToTweetIdFeature, inReplyToTweetId)
.add(IsHydratedFeature, true) .add(IsHydratedFeature, true)
.add(IsNsfw, Some(isNsfw)) .add(IsNsfw, Some(isNsfw))
@ -156,24 +137,20 @@ class TweetypieFeatureHydrator @Inject() (
.add(QuotedUserIdFeature, quotedTweetUserId) .add(QuotedUserIdFeature, quotedTweetUserId)
.add(SourceTweetIdFeature, retweetedTweetId) .add(SourceTweetIdFeature, retweetedTweetId)
.add(SourceUserIdFeature, retweetedTweetUserId) .add(SourceUserIdFeature, retweetedTweetUserId)
.add(TweetLanguageFeature, tweetLanguage)
.add(TweetTextFeature, tweetText) .add(TweetTextFeature, tweetText)
.add(VisibilityReason, found.suppressReason) .add(VisibilityReason, found.suppressReason)
.build() .build()
// If no tweet result found, return default and pre-existing features // If no tweet result found, return default and pre-existing features
case _ => case _ =>
DefaultFeatureMap ++ FeatureMapBuilder() DefaultFeatureMap +
.add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) (AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) +
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) (InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) +
.add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) (IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) +
.add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) (QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) +
.add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) (QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) +
.add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) (SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) +
.add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) (SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
.add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
.add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
.build()
} }
} }
} }

View File

@ -1,10 +1,9 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.google.inject.name.Named import com.google.inject.name.Named
import com.twitter.conversions.DurationOps.RichDuration import com.twitter.conversions.DurationOps.RichDuration
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature
import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
@ -14,6 +13,7 @@ import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature
import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SemanticAnnotationFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache
@ -47,7 +47,6 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
override val features: Set[Feature[_, _]] = Set( override val features: Set[Feature[_, _]] = Set(
AuthorIdFeature, AuthorIdFeature,
DirectedAtUserIdFeature, DirectedAtUserIdFeature,
ExclusiveConversationAuthorIdFeature,
HasImageFeature, HasImageFeature,
HasVideoFeature, HasVideoFeature,
InReplyToTweetIdFeature, InReplyToTweetIdFeature,
@ -57,6 +56,7 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
MentionUserIdFeature, MentionUserIdFeature,
QuotedTweetIdFeature, QuotedTweetIdFeature,
QuotedUserIdFeature, QuotedUserIdFeature,
SemanticAnnotationFeature,
SourceTweetIdFeature, SourceTweetIdFeature,
SourceUserIdFeature SourceUserIdFeature
) )
@ -66,7 +66,6 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
private val DefaultFeatureMap = FeatureMapBuilder() private val DefaultFeatureMap = FeatureMapBuilder()
.add(AuthorIdFeature, None) .add(AuthorIdFeature, None)
.add(DirectedAtUserIdFeature, None) .add(DirectedAtUserIdFeature, None)
.add(ExclusiveConversationAuthorIdFeature, None)
.add(HasImageFeature, false) .add(HasImageFeature, false)
.add(HasVideoFeature, false) .add(HasVideoFeature, false)
.add(InReplyToTweetIdFeature, None) .add(InReplyToTweetIdFeature, None)
@ -76,6 +75,7 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
.add(MentionUserIdFeature, Seq.empty) .add(MentionUserIdFeature, Seq.empty)
.add(QuotedTweetIdFeature, None) .add(QuotedTweetIdFeature, None)
.add(QuotedUserIdFeature, None) .add(QuotedUserIdFeature, None)
.add(SemanticAnnotationFeature, Seq.empty)
.add(SourceTweetIdFeature, None) .add(SourceTweetIdFeature, None)
.add(SourceUserIdFeature, None) .add(SourceUserIdFeature, None)
.build() .build()
@ -114,13 +114,12 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
val mentions = tweet.mentions.getOrElse(Seq.empty) val mentions = tweet.mentions.getOrElse(Seq.empty)
val share = coreData.flatMap(_.share) val share = coreData.flatMap(_.share)
val reply = coreData.flatMap(_.reply) val reply = coreData.flatMap(_.reply)
val semanticAnnotations =
tweet.escherbirdEntityAnnotations.map(_.entityAnnotations).getOrElse(Seq.empty)
FeatureMapBuilder() FeatureMapBuilder()
.add(AuthorIdFeature, coreData.map(_.userId)) .add(AuthorIdFeature, coreData.map(_.userId))
.add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId)))
.add(
ExclusiveConversationAuthorIdFeature,
tweet.exclusiveTweetControl.map(_.conversationAuthorId))
.add(HasImageFeature, TweetMediaFeaturesExtractor.hasImage(tweet)) .add(HasImageFeature, TweetMediaFeaturesExtractor.hasImage(tweet))
.add(HasVideoFeature, TweetMediaFeaturesExtractor.hasVideo(tweet)) .add(HasVideoFeature, TweetMediaFeaturesExtractor.hasVideo(tweet))
.add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId))
@ -130,6 +129,7 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
.add(MentionUserIdFeature, mentions.flatMap(_.userId)) .add(MentionUserIdFeature, mentions.flatMap(_.userId))
.add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId))
.add(QuotedUserIdFeature, quotedTweet.map(_.userId)) .add(QuotedUserIdFeature, quotedTweet.map(_.userId))
.add(SemanticAnnotationFeature, semanticAnnotations)
.add(SourceTweetIdFeature, share.map(_.sourceStatusId)) .add(SourceTweetIdFeature, share.map(_.sourceStatusId))
.add(SourceUserIdFeature, share.map(_.sourceUserId)) .add(SourceUserIdFeature, share.map(_.sourceUserId))
.build() .build()

View File

@ -1,9 +1,9 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureRepository import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollow20200101FeatureRepository
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.{thriftscala => ml} import com.twitter.ml.api.{thriftscala => ml}
@ -17,61 +17,69 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueRepository
import com.twitter.servo.repository.KeyValueResult import com.twitter.servo.repository.KeyValueResult
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.util.Future import com.twitter.util.Future
import com.twitter.util.Try import com.twitter.util.Try
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Named import javax.inject.Named
import javax.inject.Singleton import javax.inject.Singleton
import scala.collection.JavaConverters._ import scala.collection.JavaConverters._
object TwhinAuthorFollowFeature object TwhinAuthorFollow20220101Feature
extends DataRecordInAFeature[TweetCandidate] extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord() override def defaultValue: DataRecord = new DataRecord()
} }
@Singleton @Singleton
class TwhinAuthorFollowFeatureHydrator @Inject() ( class TwhinAuthorFollow20220101FeatureHydrator @Inject() (
@Named(TwhinAuthorFollowFeatureRepository) @Named(TwhinAuthorFollow20200101FeatureRepository)
client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], client: KeyValueRepository[Seq[Long], Long, ml.Embedding],
override val statsReceiver: StatsReceiver) override val statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with ObservedKeyValueResultHandler { with ObservedKeyValueResultHandler {
override val identifier: FeatureHydratorIdentifier = override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("TwhinAuthorFollow") FeatureHydratorIdentifier("TwhinAuthorFollow20220101")
override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollowFeature) override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollow20220101Feature)
override val statScope: String = identifier.toString override val statScope: String = identifier.toString
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { ): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val possiblyAuthorIds = extractKeys(candidates) val possiblyAuthorIds = extractKeys(candidates)
val authorIds = possiblyAuthorIds.flatten val authorIds = possiblyAuthorIds.flatten
val response: Future[KeyValueResult[Long, ml.FloatTensor]] = val response: Future[KeyValueResult[Long, ml.Embedding]] =
if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) if (authorIds.isEmpty) {
Future.value(KeyValueResult.empty)
} else {
client(authorIds)
}
response.map { result => response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId => possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result) val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value) val transformedValue = postTransformer(value)
FeatureMapBuilder().add(TwhinAuthorFollowFeature, transformedValue).build() FeatureMapBuilder()
.add(TwhinAuthorFollow20220101Feature, transformedValue)
.build()
}
} }
} }
} }
private def postTransformer(embedding: Try[Option[ml.FloatTensor]]): Try[DataRecord] = { private def postTransformer(embedding: Try[Option[ml.Embedding]]): Try[DataRecord] = {
embedding.map { floatTensor => embedding.map { e =>
TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(e).asScala.head
} }
} }
@ -79,7 +87,10 @@ class TwhinAuthorFollowFeatureHydrator @Inject() (
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]] = { ): Seq[Option[Long]] = {
candidates.map { candidate => candidates.map { candidate =>
CandidatesUtil.getOriginalAuthorId(candidate.features) candidate.features
.getTry(AuthorIdFeature)
.toOption
.flatten
} }
} }
} }

View File

@ -1,15 +1,17 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserEngagementFeatureRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserEngagementFeatureRepository
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions
import com.twitter.ml.api.{thriftscala => ml} import com.twitter.ml.api.{thriftscala => ml}
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
@ -20,7 +22,6 @@ import com.twitter.util.Throw
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Named import javax.inject.Named
import javax.inject.Singleton import javax.inject.Singleton
import scala.collection.JavaConverters._
object TwhinUserEngagementFeature object TwhinUserEngagementFeature
extends DataRecordInAFeature[PipelineQuery] extends DataRecordInAFeature[PipelineQuery]
@ -47,7 +48,8 @@ class TwhinUserEngagementQueryFeatureHydrator @Inject() (
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId val userId = query.getRequiredUserId
Stitch.callFuture(client(Seq(userId))).map { results => Stitch.callFuture {
client(Seq(userId)).map { results =>
val embedding: Option[ml.FloatTensor] = results(userId) match { val embedding: Option[ml.FloatTensor] = results(userId) match {
case Return(value) => case Return(value) =>
if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr()
@ -59,14 +61,20 @@ class TwhinUserEngagementQueryFeatureHydrator @Inject() (
case _ => case _ =>
None None
} }
val dataRecord = val dataRecord =
TwhinUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head new RichDataRecord(new DataRecord, TwhinUserEngagementEmbeddingsAdapter.getFeatureContext)
embedding.foreach { floatTensor =>
dataRecord.setFeatureValue(
TwhinUserEngagementEmbeddingsAdapter.twhinEmbeddingsFeature,
ScalaToJavaDataRecordConversions.scalaTensor2Java(
ml.GeneralTensor.FloatTensor(floatTensor))
)
}
FeatureMapBuilder() FeatureMapBuilder()
.add(TwhinUserEngagementFeature, dataRecord) .add(TwhinUserEngagementFeature, dataRecord.getRecord)
.build() .build()
} }
} }
}
} }

View File

@ -1,15 +1,18 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserFollowFeatureRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserFollowFeatureRepository
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter
import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions
import com.twitter.ml.api.{thriftscala => ml} import com.twitter.ml.api.{thriftscala => ml}
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
@ -20,7 +23,6 @@ import com.twitter.util.Throw
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Named import javax.inject.Named
import javax.inject.Singleton import javax.inject.Singleton
import scala.collection.JavaConverters._
object TwhinUserFollowFeature object TwhinUserFollowFeature
extends DataRecordInAFeature[PipelineQuery] extends DataRecordInAFeature[PipelineQuery]
@ -47,7 +49,8 @@ class TwhinUserFollowQueryFeatureHydrator @Inject() (
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId val userId = query.getRequiredUserId
Stitch.callFuture(client(Seq(userId))).map { results => Stitch.callFuture(
client(Seq(userId)).map { results =>
val embedding: Option[ml.FloatTensor] = results(userId) match { val embedding: Option[ml.FloatTensor] = results(userId) match {
case Return(value) => case Return(value) =>
if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr()
@ -59,12 +62,19 @@ class TwhinUserFollowQueryFeatureHydrator @Inject() (
case _ => case _ =>
None None
} }
val dataRecord =
val dataRecord = TwhinUserFollowEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head new RichDataRecord(new DataRecord, TwhinUserFollowEmbeddingsAdapter.getFeatureContext)
embedding.foreach { floatTensor =>
dataRecord.setFeatureValue(
TwhinUserFollowEmbeddingsAdapter.twhinEmbeddingsFeature,
ScalaToJavaDataRecordConversions.scalaTensor2Java(
ml.GeneralTensor
.FloatTensor(floatTensor)))
}
FeatureMapBuilder() FeatureMapBuilder()
.add(TwhinUserFollowFeature, dataRecord) .add(TwhinUserFollowFeature, dataRecord.getRecord)
.build() .build()
} }
)
} }
} }

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
@ -12,12 +12,12 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.keyvalue.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.util.Future import com.twitter.util.Future
import com.twitter.util.Try import com.twitter.util.Try
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Named import javax.inject.Named
import javax.inject.Singleton import javax.inject.Singleton
@ -42,19 +42,27 @@ class UserFollowedTopicIdsFeatureHydrator @Inject() (
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { ): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val possiblyAuthorIds = extractKeys(candidates) val possiblyAuthorIds = extractKeys(candidates)
val authorIds = possiblyAuthorIds.flatten val authorIds = possiblyAuthorIds.flatten
val response: Future[KeyValueResult[Long, Seq[Long]]] = val response: Future[KeyValueResult[Long, Seq[Long]]] =
if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) if (authorIds.isEmpty) {
Future.value(KeyValueResult.empty)
} else {
client(authorIds)
}
response.map { result => response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId => possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result) val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value) val transformedValue = postTransformer(value)
FeatureMapBuilder().add(UserFollowedTopicIdsFeature, transformedValue).build() FeatureMapBuilder()
.add(UserFollowedTopicIdsFeature, transformedValue)
.build()
}
} }
} }
} }

View File

@ -1,8 +1,6 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesStore
import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesRepository
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
@ -10,8 +8,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.search.common.constants.{thriftscala => scc} import com.twitter.search.common.constants.{thriftscala => scc}
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.storehaus.ReadableStore
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Named import javax.inject.Named
import javax.inject.Singleton import javax.inject.Singleton
@ -20,26 +18,17 @@ object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguag
@Singleton @Singleton
case class UserLanguagesFeatureHydrator @Inject() ( case class UserLanguagesFeatureHydrator @Inject() (
@Named(UserLanguagesRepository) client: KeyValueRepository[Seq[Long], Long, Seq[ @Named(UserLanguagesStore) store: ReadableStore[Long, Seq[scc.ThriftLanguage]])
scc.ThriftLanguage extends QueryFeatureHydrator[PipelineQuery] {
]],
statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery]
with ObservedKeyValueResultHandler {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLanguages") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLanguages")
override val features: Set[Feature[_, _]] = Set(UserLanguagesFeature) override val features: Set[Feature[_, _]] = Set(UserLanguagesFeature)
override val statScope: String = identifier.toString
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val key = query.getRequiredUserId Stitch.callFuture(store.get(query.getRequiredUserId)).map { languages =>
Stitch.callFuture(client(Seq(key))).map { result =>
val feature =
observedGet(key = Some(key), keyValueResult = result).map(_.getOrElse(Seq.empty))
FeatureMapBuilder() FeatureMapBuilder()
.add(UserLanguagesFeature, feature) .add(UserLanguagesFeature, languages.getOrElse(Seq.empty))
.build() .build()
} }
} }

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.service.HomeMixerAlertConfig
@ -9,12 +9,13 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.user_health.v1.{thriftscala => uhv1}
import com.twitter.timelines.user_health.{thriftscala => uh} import com.twitter.timelines.user_health.{thriftscala => uh}
import com.twitter.timelines.user_health.v1.{thriftscala => uhv1}
import com.twitter.user_session_store.ReadOnlyUserSessionStore import com.twitter.user_session_store.ReadOnlyUserSessionStore
import com.twitter.user_session_store.ReadRequest import com.twitter.user_session_store.ReadRequest
import com.twitter.user_session_store.UserSessionDataset import com.twitter.user_session_store.UserSessionDataset
import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

View File

@ -1,13 +1,9 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.FavoritedByCountFeature
import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature
import com.twitter.home_mixer.model.HomeFeatures.RepliedByCountFeature
import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.RetweetedByCountFeature
import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository
@ -20,12 +16,12 @@ import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.recos.recos_common.{thriftscala => rc} import com.twitter.recos.recos_common.{thriftscala => rc}
import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg}
import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.keyvalue.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Named import javax.inject.Named
import javax.inject.Singleton import javax.inject.Singleton
@ -44,10 +40,7 @@ class UtegFeatureHydrator @Inject() (
override val features: Set[Feature[_, _]] = Set( override val features: Set[Feature[_, _]] = Set(
FavoritedByUserIdsFeature, FavoritedByUserIdsFeature,
RetweetedByEngagerIdsFeature, RetweetedByEngagerIdsFeature,
RepliedByEngagerIdsFeature, RepliedByEngagerIdsFeature
FavoritedByCountFeature,
RetweetedByCountFeature,
RepliedByCountFeature
) )
override def onlyIf(query: PipelineQuery): Boolean = query.features override def onlyIf(query: PipelineQuery): Boolean = query.features
@ -56,7 +49,7 @@ class UtegFeatureHydrator @Inject() (
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { ): Stitch[Seq[FeatureMap]] = {
val seedUserWeights = query.features.map(_.get(RealGraphInNetworkScoresFeature)).get val seedUserWeights = query.features.map(_.get(RealGraphInNetworkScoresFeature)).get
val sourceTweetIds = candidates.flatMap(_.features.getOrElse(SourceTweetIdFeature, None)) val sourceTweetIds = candidates.flatMap(_.features.getOrElse(SourceTweetIdFeature, None))
@ -66,7 +59,9 @@ class UtegFeatureHydrator @Inject() (
val utegQuery = (tweetIdsToSend, (query.getRequiredUserId, seedUserWeights)) val utegQuery = (tweetIdsToSend, (query.getRequiredUserId, seedUserWeights))
client(utegQuery).map(handleResponse(candidates, _)) Stitch
.callFuture(client(utegQuery))
.map(handleResponse(candidates, _))
} }
private def handleResponse( private def handleResponse(
@ -74,7 +69,6 @@ class UtegFeatureHydrator @Inject() (
results: KeyValueResult[Long, uteg.TweetRecommendation], results: KeyValueResult[Long, uteg.TweetRecommendation],
): Seq[FeatureMap] = { ): Seq[FeatureMap] = {
candidates.map { candidate => candidates.map { candidate =>
val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false)
val candidateProof = results(candidate.candidate.id).toOption.flatten val candidateProof = results(candidate.candidate.id).toOption.flatten
val sourceProof = candidate.features val sourceProof = candidate.features
.getOrElse(SourceTweetIdFeature, None).flatMap(results(_).toOption.flatten) .getOrElse(SourceTweetIdFeature, None).flatMap(results(_).toOption.flatten)
@ -84,18 +78,10 @@ class UtegFeatureHydrator @Inject() (
val retweetedBy = proofs.flatMap(_.get(rc.SocialProofType.Retweet)).flatten val retweetedBy = proofs.flatMap(_.get(rc.SocialProofType.Retweet)).flatten
val repliedBy = proofs.flatMap(_.get(rc.SocialProofType.Reply)).flatten val repliedBy = proofs.flatMap(_.get(rc.SocialProofType.Reply)).flatten
val (favoritedByCount, retweetedByCount, repliedByCount) =
if (!inNetwork) {
(favoritedBy.size.toDouble, retweetedBy.size.toDouble, repliedBy.size.toDouble)
} else { (0.0, 0.0, 0.0) }
FeatureMapBuilder() FeatureMapBuilder()
.add(FavoritedByUserIdsFeature, favoritedBy) .add(FavoritedByUserIdsFeature, favoritedBy)
.add(RetweetedByEngagerIdsFeature, retweetedBy) .add(RetweetedByEngagerIdsFeature, retweetedBy)
.add(RepliedByEngagerIdsFeature, repliedBy) .add(RepliedByEngagerIdsFeature, repliedBy)
.add(FavoritedByCountFeature, favoritedByCount)
.add(RetweetedByCountFeature, retweetedByCount)
.add(RepliedByCountFeature, repliedByCount)
.build() .build()
} }
} }

View File

@ -0,0 +1,59 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
import com.twitter.ml.api.DataRecordMerger
import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.CompactDataRecordConverter
import com.twitter.ml.api.util.FDsl._
import com.twitter.timelines.author_features.v1.{thriftjava => af}
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
private val originalAuthorAggregatesFeatures =
TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
.buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
private val authorFeatures = originalAuthorAggregatesFeatures ++
Seq(
UserHealthFeatures.AuthorState,
UserHealthFeatures.NumAuthorFollowers,
UserHealthFeatures.NumAuthorConnectDays,
UserHealthFeatures.NumAuthorConnect)
private val featureContext = new FeatureContext(authorFeatures: _*)
override def getFeatureContext: FeatureContext = featureContext
override val commonFeatures: Set[Feature[_]] = Set.empty
private val compactDataRecordConverter = new CompactDataRecordConverter()
private val drMerger = new DataRecordMerger()
override def setFeatures(
authorFeaturesOpt: Option[af.AuthorFeatures],
richDataRecord: RichDataRecord
): Unit = {
authorFeaturesOpt.foreach { authorFeatures =>
val dataRecord = richDataRecord.getRecord
dataRecord.setFeatureValue(
UserHealthFeatures.AuthorState,
authorFeatures.user_health.user_state.getValue.toLong)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorFollowers,
authorFeatures.user_health.num_followers.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnectDays,
authorFeatures.user_health.num_connect_days.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnect,
authorFeatures.user_health.num_connect.toDouble)
val originalAuthorAggregatesDataRecord =
compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
}
}
}

View File

@ -4,7 +4,6 @@ scala_library(
strict_deps = True, strict_deps = True,
tags = ["bazel-compatible"], tags = ["bazel-compatible"],
dependencies = [ dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api:api-base",
"src/java/com/twitter/ml/api/util", "src/java/com/twitter/ml/api/util",
"src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/ml/api/util",

View File

@ -11,7 +11,6 @@ scala_library(
"src/scala/com/twitter/timelines/prediction/common/adapters", "src/scala/com/twitter/timelines/prediction/common/adapters",
"src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/common/adapters:base",
"src/scala/com/twitter/timelines/prediction/features/common", "src/scala/com/twitter/timelines/prediction/features/common",
"src/scala/com/twitter/timelines/prediction/features/conversation_features",
"src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/ml/api:data-java",
"src/thrift/com/twitter/ml/api:data-scala", "src/thrift/com/twitter/ml/api:data-scala",
], ],

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content
import com.twitter.home_mixer.model.ContentFeatures import com.twitter.home_mixer.model.ContentFeatures
import com.twitter.ml.api.Feature import com.twitter.ml.api.Feature
@ -8,14 +8,11 @@ import com.twitter.ml.api.util.DataRecordConverters._
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
import com.twitter.timelines.prediction.common.adapters.TweetLengthType import com.twitter.timelines.prediction.common.adapters.TweetLengthType
import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures
import com.twitter.timelines.prediction.features.conversation_features.ConversationFeatures
import scala.collection.JavaConverters._ import scala.collection.JavaConverters._
object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] {
override val getFeatureContext: FeatureContext = new FeatureContext( override val getFeatureContext: FeatureContext = new FeatureContext(
ConversationFeatures.IS_SELF_THREAD_TWEET,
ConversationFeatures.IS_LEAF_IN_SELF_THREAD,
TimelinesSharedFeatures.ASPECT_RATIO_DEN, TimelinesSharedFeatures.ASPECT_RATIO_DEN,
TimelinesSharedFeatures.ASPECT_RATIO_NUM, TimelinesSharedFeatures.ASPECT_RATIO_NUM,
TimelinesSharedFeatures.BIT_RATE, TimelinesSharedFeatures.BIT_RATE,
@ -82,16 +79,6 @@ object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[Content
): Unit = { ): Unit = {
if (contentFeatures.nonEmpty) { if (contentFeatures.nonEmpty) {
val features = contentFeatures.get val features = contentFeatures.get
// Conversation Features
richDataRecord.setFeatureValueFromOption(
ConversationFeatures.IS_SELF_THREAD_TWEET,
Some(features.selfThreadMetadata.nonEmpty)
)
richDataRecord.setFeatureValueFromOption(
ConversationFeatures.IS_LEAF_IN_SELF_THREAD,
features.selfThreadMetadata.map(_.isLeaf)
)
// Media Features // Media Features
richDataRecord.setFeatureValueFromOption( richDataRecord.setFeatureValueFromOption(
TimelinesSharedFeatures.ASPECT_RATIO_DEN, TimelinesSharedFeatures.ASPECT_RATIO_DEN,

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.earlybird
import com.twitter.ml.api.Feature import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.FeatureContext
@ -305,10 +305,6 @@ object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTwe
richDataRecord.setFeatureValueFromOption( richDataRecord.setFeatureValueFromOption(
RecapFeatures.REPLY_COUNT_V2, RecapFeatures.REPLY_COUNT_V2,
features.replyCountV2.map(_.toDouble)) features.replyCountV2.map(_.toDouble))
richDataRecord.setFeatureValueFromOption(
RecapFeatures.MENTIONED_SCREEN_NAMES,
features.mentionsList.map(_.toSet.asJava)
)
val urls = features.urlsList.getOrElse(Seq.empty) val urls = features.urlsList.getOrElse(Seq.empty)
richDataRecord.setFeatureValue( richDataRecord.setFeatureValue(
RecapFeatures.URL_DOMAINS, RecapFeatures.URL_DOMAINS,

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic
import com.twitter.ml.api.Feature import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.FeatureContext

Some files were not shown because too many files have changed in this diff Show More