Compare commits

..

1 Commits

Author SHA1 Message Date
Leon Banik
17b7850b6c
Merge ca2cc4c12f into fb54d8b549 2023-05-22 17:38:49 -05:00
393 changed files with 3850 additions and 9334 deletions

View File

@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
- Fetch Tweet Candidates
- ScoredTweetsInNetworkCandidatePipelineConfig
- ScoredTweetsTweetMixerCandidatePipelineConfig
- ScoredTweetsCrMixerCandidatePipelineConfig
- ScoredTweetsUtegCandidatePipelineConfig
- ScoredTweetsFrsCandidatePipelineConfig
- Feature Hydration and Scoring
@ -99,3 +99,4 @@ Timeline tabs powered by Home Mixer.
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
- ListTweetsAdsCandidatePipelineConfig (fetch ads)

View File

@ -21,7 +21,6 @@ scala_library(
"finatra/inject/inject-utils/src/main/scala",
"home-mixer/server/src/main/resources",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
@ -32,10 +31,6 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"stringcenter/client",
"stringcenter/client/src/main/java",
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",

View File

@ -2,7 +2,7 @@ package com.twitter.home_mixer
import com.twitter.finatra.http.routing.HttpWarmup
import com.twitter.finatra.httpclient.RequestBuilder._
import com.twitter.util.logging.Logging
import com.twitter.inject.Logging
import com.twitter.inject.utils.Handler
import com.twitter.util.Try
import javax.inject.Inject

View File

@ -12,63 +12,57 @@ import com.twitter.finatra.thrift.ThriftServer
import com.twitter.finatra.thrift.filters._
import com.twitter.finatra.thrift.routing.ThriftRouter
import com.twitter.home_mixer.controller.HomeThriftController
import com.twitter.home_mixer.federated.HomeMixerColumn
import com.twitter.home_mixer.module._
import com.twitter.home_mixer.param.GlobalParamConfigModule
import com.twitter.home_mixer.product.HomeMixerProductModule
import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
import com.twitter.product_mixer.component_library.module.CrMixerClientModule
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
import com.twitter.product_mixer.component_library.module.EarlybirdModule
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
import com.twitter.product_mixer.core.controllers.ProductMixerController
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
import com.twitter.product_mixer.core.module.ProductMixerModule
import com.twitter.product_mixer.core.module.StratoClientModule
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.fed.server.StratoFedServer
object HomeMixerServerMain extends HomeMixerServer
class HomeMixerServer
extends StratoFedServer
with ThriftServer
with Mtls
with HttpServer
with HttpMtls {
class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
override val name = "home-mixer-server"
override val modules: Seq[Module] = Seq(
AccountRecommendationsMixerModule,
AdvertiserBrandSafetySettingsStoreModule,
BlenderClientModule,
ClientSentImpressionsPublisherModule,
ConversationServiceModule,
CrMixerClientModule,
EarlybirdModule,
ExploreRankerClientModule,
FeedbackHistoryClientModule,
GizmoduckClientModule,
GlobalParamConfigModule,
HomeAdsCandidateSourceModule,
HomeMixerFlagsModule,
HomeMixerProductModule,
HomeMixerResourcesModule,
HomeNaviModelClientModule,
ImpressionBloomFilterModule,
InjectionHistoryClientModule,
FeedbackHistoryClientModule,
ManhattanClientsModule,
ManhattanFeatureRepositoryModule,
ManhattanTweetImpressionStoreModule,
MemcachedFeatureRepositoryModule,
NaviModelClientModule,
OnboardingTaskServiceModule,
OptimizedStratoClientModule,
PeopleDiscoveryServiceModule,
@ -80,23 +74,24 @@ class HomeMixerServer
SimClustersRecentEngagementsClientModule,
SocialGraphServiceModule,
StaleTweetsCacheModule,
StratoClientModule,
ThriftFeatureRepositoryModule,
TimelineMixerClientModule,
TimelineRankerClientModule,
TimelineScorerClientModule,
TimelineServiceClientModule,
TimelinesPersistenceStoreClientModule,
TopicSocialProofClientModule,
TweetImpressionStoreModule,
TweetMixerClientModule,
TweetypieClientModule,
TweetyPieClientModule,
TweetypieStaticEntitiesCacheClientModule,
UserMetadataStoreModule,
UserSessionStoreModule,
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
new ProductScopeStringCenterModule()
)
override def configureThrift(router: ThriftRouter): Unit = {
def configureThrift(router: ThriftRouter): Unit = {
router
.filter[LoggingMDCFilter]
.filter[TraceIdMDCFilter]
@ -116,11 +111,6 @@ class HomeMixerServer
this.injector,
st.HomeMixer.ExecutePipeline))
override val dest: String = "/s/home-mixer/home-mixer:strato"
override val columns: Seq[Class[_ <: StratoFed.Column]] =
Seq(classOf[HomeMixerColumn])
override protected def warmup(): Unit = {
handle[HomeMixerThriftServerWarmupHandler]()
handle[HomeMixerHttpServerWarmupHandler]()

View File

@ -3,7 +3,7 @@ package com.twitter.home_mixer
import com.twitter.finagle.thrift.ClientId
import com.twitter.finatra.thrift.routing.ThriftWarmup
import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.util.logging.Logging
import com.twitter.inject.Logging
import com.twitter.inject.utils.Handler
import com.twitter.product_mixer.core.{thriftscala => pt}
import com.twitter.scrooge.Request

View File

@ -5,13 +5,19 @@ scala_library(
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
@ -19,6 +25,10 @@ scala_library(
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
],
exports = [
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",

View File

@ -1,23 +1,18 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
import com.twitter.product_mixer.component_library.filter.FeatureFilter
import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
@ -38,8 +33,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
override val gates: Seq[BaseGate[Query]],
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
extends DependentCandidatePipelineConfig[
@ -67,10 +62,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
val tweetsWithConversationMetadata = candidates.map { candidate =>
TweetWithConversationMetadata(
tweetId = candidate.candidateIdLong,
userId = candidate.features.getOrElse(AuthorIdFeature, None),
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
userId = None,
sourceTweetId = None,
sourceUserId = None,
inReplyToTweetId = None,
conversationId = None,
ancestors = Seq.empty
)
@ -89,10 +84,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
override val preFilterFeatureHydrationPhase1: Seq[
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
] = Seq(
tweetypieFeatureHydrator,
InNetworkFeatureHydrator,
)
] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
RetweetDeduplicationFilter,
@ -101,7 +93,6 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
FilterIdentifier(QuotedTweetDroppedFilterId),
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
),
invalidSubscriptionTweetFilter,
InvalidConversationModuleFilter
)

View File

@ -1,9 +1,9 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
@ -15,7 +15,7 @@ import javax.inject.Singleton
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator) {
def build(
@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
new ConversationServiceCandidatePipelineConfig(
conversationServiceCandidateSource,
tweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator,
namesFeatureHydrator,
invalidSubscriptionTweetFilter,
gates,
decorator
)

View File

@ -1,7 +1,7 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
import com.twitter.home_mixer.service.HomeMixerAlertConfig

View File

@ -13,5 +13,8 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
"snowflake/src/main/scala/com/twitter/snowflake/id",
"src/thrift/com/twitter/context:twitter-context-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"twitter-context/src/main/scala",
],
)

View File

@ -7,7 +7,6 @@ import com.twitter.home_mixer.service.ScoredTweetsService
import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.controllers.DebugTwitterContext
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
import com.twitter.product_mixer.core.service.urt.UrtService
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch

View File

@ -1,24 +0,0 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"stitch/stitch-repo/src/main/scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
"strato/src/main/scala/com/twitter/strato/callcontext",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
],
)

View File

@ -1,217 +0,0 @@
package com.twitter.home_mixer.federated
import com.twitter.gizmoduck.{thriftscala => gd}
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
import com.twitter.home_mixer.model.request.HomeMixerRequest
import com.twitter.home_mixer.{thriftscala => hm}
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
import com.twitter.product_mixer.core.{thriftscala => pm}
import com.twitter.stitch.Arrow
import com.twitter.stitch.Stitch
import com.twitter.strato.callcontext.CallContext
import com.twitter.strato.catalog.OpMetadata
import com.twitter.strato.config._
import com.twitter.strato.data._
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
import com.twitter.strato.graphql.timelines.{thriftscala => gql}
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.timelines.render.{thriftscala => tr}
import com.twitter.util.Try
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class HomeMixerColumn @Inject() (
homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn,
paramsBuilder: ParamsBuilder,
productPipelineRegistry: ProductPipelineRegistry)
extends StratoFed.Column(HomeMixerColumn.Path)
with StratoFed.Fetch.Arrow {
override val contactInfo: ContactInfo = ContactInfo(
contactEmail = "",
ldapGroup = "",
slackRoomId = ""
)
override val metadata: OpMetadata =
OpMetadata(
lifecycle = Some(Lifecycle.Production),
description =
Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
)
private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
ServiceIdentifierPattern(
role = "",
service = "",
env = "",
zone = Seq(""))
)
override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
override type Key = gql.TimelineKey
override type View = gql.HomeTimelineView
override type Value = tr.Timeline
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
private def createHomeMixerRequestArrow(
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn
): Arrow[(Key, View), hm.HomeMixerRequest] = {
val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
(gd.LookupContext(), Set(gd.QueryFields.Roles))
val populateUserRoles = Arrow
.flatMap[(Key, View), Option[Set[String]]] { _ =>
Stitch.collect {
CallContext.twitterUserId.map { userId =>
compositeOnUserClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch(userId, gizmoduckView).map(_.v)
.map {
_.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
}
}
}
}
val populateIpAddress = Arrow
.flatMap[(Key, View), Option[String]](_ =>
auditIpClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch((), ()).map(_.v))
Arrow.join(
populateUserRoles,
populateIpAddress
)
}
Arrow.zipWithArg(populateUserRolesAndIp).map {
case ((key, view), (roles, ipAddress)) =>
val deviceContextOpt = Some(
hm.DeviceContext(
isPolling = CallContext.isPolling,
requestContext = view.requestContext,
latestControlAvailable = view.latestControlAvailable,
autoplayEnabled = view.autoplayEnabled
))
val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
val (product, productContext) = key match {
case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
(
hm.Product.ForYou,
hm.ProductContext.ForYou(
hm.ForYou(
deviceContextOpt,
seenTweetIds,
view.dspClientContext,
view.pushToHomeTweetId
)
))
case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
(
hm.Product.Following,
hm.ProductContext.Following(
hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
(
hm.Product.Subscribed,
hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
}
val clientContext = pm.ClientContext(
userId = CallContext.twitterUserId,
guestId = CallContext.guestId,
guestIdAds = CallContext.guestIdAds,
guestIdMarketing = CallContext.guestIdMarketing,
appId = CallContext.clientApplicationId,
ipAddress = ipAddress,
userAgent = CallContext.userAgent,
countryCode = CallContext.requestCountryCode,
languageCode = CallContext.requestLanguageCode,
isTwoffice = CallContext.isInternalOrTwoffice,
userRoles = roles,
deviceId = CallContext.deviceId,
mobileDeviceId = CallContext.mobileDeviceId,
mobileDeviceAdId = CallContext.adId,
limitAdTracking = CallContext.limitAdTracking
)
hm.HomeMixerRequest(
clientContext = clientContext,
product = product,
productContext = Some(productContext),
maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
cursor = view.cursor.filter(_.nonEmpty)
)
}
}
override val fetch: Arrow[(Key, View), Result[Value]] = {
val transformThriftIntoPipelineRequest: Arrow[
(Key, View),
ProductPipelineRequest[HomeMixerRequest]
] = {
Arrow
.identity[(Key, View)]
.andThen {
createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
}
.map {
case thriftRequest =>
val request = homeMixerRequestUnmarshaller(thriftRequest)
val params = paramsBuilder.build(
clientContext = request.clientContext,
product = request.product,
featureOverrides =
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
)
ProductPipelineRequest(request, params)
}
}
val underlyingProduct: Arrow[
ProductPipelineRequest[HomeMixerRequest],
ProductPipelineResult[tr.TimelineResponse]
] = Arrow
.identity[ProductPipelineRequest[HomeMixerRequest]]
.map { pipelineRequest =>
val pipelineArrow = productPipelineRegistry
.getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
pipelineRequest.request.product)
.arrow
(pipelineArrow, pipelineRequest)
}.applyArrow
transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
_.result match {
case Some(result) => found(result.timeline)
case _ => missing
}
}
}
}
object HomeMixerColumn {
val Path = "home-mixer/homeMixer.Timeline"
private val FetchCallstack = s"$Path:fetch"
private val MaxCount: Option[Int] = Some(100)
}

View File

@ -10,8 +10,11 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"stitch/stitch-timelineservice/src/main/scala",
"strato/config/columns/recommendations/similarity:similarity-strato-client",
"strato/src/main/scala/com/twitter/strato/client",
],
exports = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.list_recommended_users.candidate_source
package com.twitter.home_mixer.functional_component.candidate_source
import com.twitter.hermit.candidate.{thriftscala => t}
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
@ -21,19 +21,13 @@ class SimilarityBasedUsersCandidateSource @Inject() (
private val fetcher: Fetcher[Long, Unit, t.Candidates] =
similarUsersBySimsOnUserClientColumn.fetcher
private val MaxCandidatesToKeep = 4000
override def apply(request: Seq[Long]): Stitch[Seq[t.Candidate]] = {
Stitch
.collect {
request.map { userId =>
fetcher
.fetch(userId, Unit).map { result =>
result.v.map(_.candidates).getOrElse(Seq.empty)
}.map { candidates =>
val sortedCandidates = candidates.sortBy(-_.score)
sortedCandidates.take(MaxCandidatesToKeep)
}
fetcher.fetch(userId, Unit).map { result =>
result.v.map(_.candidates).getOrElse(Seq.empty)
}
}
}.map(_.flatten)
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -9,6 +9,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -6,11 +6,13 @@ scala_library(
dependencies = [
"finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
@ -23,6 +25,8 @@ scala_library(
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"stringcenter/client",
"stringcenter/client/src/main/java",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
],
)

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -13,6 +13,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -17,6 +17,7 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tls}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.finagle.tracing.Trace
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.bijection.Base64String
import com.twitter.bijection.scrooge.BinaryScalaCodec

View File

@ -1,8 +1,6 @@
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.request.FollowingProduct
@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => t}
import com.twitter.timelines.util.FeedbackMetadataSerializer
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
@ -14,13 +14,10 @@ import javax.inject.Singleton
@Singleton
case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder: LikedBySocialContextBuilder,
listsSocialContextBuilder: ListsSocialContextBuilder,
followedBySocialContextBuilder: FollowedBySocialContextBuilder,
topicSocialContextBuilder: TopicSocialContextBuilder,
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder,
popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
def apply(
@ -34,9 +31,6 @@ case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder(query, candidate, features)
.orElse(followedBySocialContextBuilder(query, candidate, features))
.orElse(topicSocialContextBuilder(query, candidate, features))
.orElse(popularVideoSocialContextBuilder(query, candidate, features))
.orElse(listsSocialContextBuilder(query, candidate, features))
.orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
case Some(_) =>
val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
// Only hydrate the social context into the root tweet in a conversation module

View File

@ -1,10 +1,8 @@
package com.twitter.home_mixer.functional_component.decorator.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
import com.twitter.tweetypie.{thriftscala => tpt}
@ -27,76 +25,74 @@ object HomeTweetTypePredicates {
(
"has_exclusive_conversation_author_id",
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
("is_eligible_for_connect_boost", _ => false),
("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)),
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("get_initial", _.getOrElse(GetInitialFeature, false)),
("get_newer", _.getOrElse(GetNewerFeature, false)),
("get_middle", _.getOrElse(GetMiddleFeature, false)),
("get_older", _.getOrElse(GetOlderFeature, false)),
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
("polling", _.getOrElse(PollingFeature, false)),
("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("tls_size_20_plus", _ => false),
("near_empty", _ => false),
("ranked_request", _ => false),
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
(
"less_than_10_mins_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
(
"conversation_module_has_2_displayed_tweets",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)),
"is_signup_request",
candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("empty_request", _ => false),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
(
"served_size_between_50_and_100",
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
(
"account_age_less_than_30_minutes",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
(
"account_age_less_than_1_day",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
(
"account_age_less_than_7_days",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
("root_user_is_in_first_degree", _ => false),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
(
"part_of_utt",
_.getOrElse(EarlybirdFeature, None)
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
(
"has_home_latest_request_past_week",
_.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("viewer_is_employee", _ => false),
("viewer_is_timelines_employee", _ => false),
("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
(
"has_ancestor_authored_by_viewer",
candidate =>
@ -104,6 +100,11 @@ object HomeTweetTypePredicates {
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
(
"deep_reply",
candidate =>
@ -118,22 +119,23 @@ object HomeTweetTypePredicates {
"tweet_age_less_than_15_seconds",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 15.seconds)),
(
"less_than_1_hour_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)),
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
("is_followed_topic_tweet", _ => false),
("is_recommended_topic_tweet", _ => false),
("is_topic_tweet", _ => false),
("preferred_language_matches_tweet_language", _ => false),
(
"device_language_matches_tweet_language",
candidate =>
candidate.getOrElse(TweetLanguageFeature, None) ==
candidate.getOrElse(DeviceLanguageFeature, None)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
("in_network", _.getOrElse(InNetworkFeature, true)),
("in_network", _.getOrElse(FromInNetworkSourceFeature, true)),
("viewer_follows_original_author", _ => false),
("has_account_follow_prompt", _ => false),
("has_relevance_prompt", _ => false),
("has_topic_annotation_haug_prompt", _ => false),
("has_topic_annotation_random_precision_prompt", _ => false),
("has_topic_annotation_prompt", _ => false),
(
"has_political_annotation",
_.getOrElse(EarlybirdFeature, None).exists(
@ -151,14 +153,9 @@ object HomeTweetTypePredicates {
_.getOrElse(EarlybirdFeature, None)
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
(
"is_followed_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == BasicTopicContextFunctionalityType)),
(
"is_recommended_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == RecommendationTopicContextFunctionalityType)),
("is_viewer_not_invited_to_reply", _ => false),
("is_viewer_invited_to_reply", _ => false),
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
(
@ -167,6 +164,8 @@ object HomeTweetTypePredicates {
(
"has_gte_100k_favs",
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
("above_neighbor_is_topic_tweet", _ => false),
("is_topic_tweet_with_neighbor_below", _ => false),
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
(
@ -188,7 +187,6 @@ object HomeTweetTypePredicates {
(
"has_toxicity_score_above_threshold",
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
(
"text_only",
candidate =>
@ -206,50 +204,23 @@ object HomeTweetTypePredicates {
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
("3_or_more_consecutive_not_in_network", _ => false),
("2_or_more_consecutive_not_in_network", _ => false),
("5_out_of_7_not_in_network", _ => false),
("7_out_of_7_not_in_network", _ => false),
("5_out_of_5_not_in_network", _ => false),
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
("has_liked_by_social_context", _ => false),
("has_followed_by_social_context", _ => false),
("has_topic_social_context", _ => false),
("timeline_entry_has_banner", _ => false),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
(
"has_liked_by_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
.exists(candidateFeatures
.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
(
"has_followed_by_social_context",
_.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
(
"has_topic_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(TopicIdSocialContextFeature, None)
.isDefined &&
candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
(
"video_bt_10_60_sec",
_.getOrElse(VideoDurationMsFeature, None).exists(duration =>
duration > 10000 && duration <= 60000)),
("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
(
"tweet_age_lte_30_minutes",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 30.minutes)),
(
"tweet_age_lte_1_hour",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 1.hour)),
(
"tweet_age_lte_6_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 6.hours)),
(
"tweet_age_lte_12_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 12.hours)),
(
"tweet_age_gte_24_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow >= 24.hours)),
"conversation_module_has_2_displayed_tweets",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
("served_in_recap_tweet_candidate_module_injection", _ => false),
("served_in_threaded_conversation_module", _ => false)
)
val PredicateMap = CandidatePredicates.toMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.list_tweets.decorator
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.ListClientEventDetailsBuilder
@ -23,10 +23,8 @@ object ListConversationServiceCandidateDecorator {
def apply(): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = {
val suggestType = st.SuggestType.OrganicListTweet
val component = InjectionScribeUtil.scribeComponent(suggestType).get
val clientEventInfoBuilder = ClientEventInfoBuilder(
component = component,
detailsBuilder = Some(ListClientEventDetailsBuilder(st.SuggestType.OrganicListTweet))
)
val clientEventInfoBuilder =
ClientEventInfoBuilder(component, Some(ListClientEventDetailsBuilder))
val tweetItemBuilder = TweetCandidateUrtItemBuilder(
clientEventInfoBuilder = clientEventInfoBuilder
)

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
@ -15,6 +15,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
@ -12,6 +12,7 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tlst}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -8,6 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
@ -11,6 +11,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,48 +1,50 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* Renders a fixed 'You Might Like' string above all OON Tweets.
*/
@Singleton
case class PopularVideoSocialContextBuilder @Inject() (
case class YouMightLikeSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val popularVideoString = externalStrings.socialContextPopularVideoString
private val youMightLikeString = externalStrings.socialContextYouMightLikeString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) {
val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true)
val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false)
if (!isInNetwork && !isRetweet) {
Some(
GeneralContext(
contextType = SparkleGeneralContextType,
text = stringCenter.prepare(popularVideoString),
text = stringCenter.prepare(youMightLikeString),
url = None,
contextImageUrls = None,
landingUrl = Some(
Url(
urlType = DeepLink,
url = ""
)
)
landingUrl = None
))
} else None
} else {
None
}
}
}

View File

@ -8,9 +8,7 @@ scala_library(
"finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
@ -20,7 +18,6 @@ scala_library(
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
],
)

View File

@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelineservice.suggests.{thriftscala => st}
case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
object ListClientEventDetailsBuilder
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
override def apply(
@ -20,7 +20,7 @@ case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
conversationDetails = None,
timelinesDetails = Some(
TimelinesDetails(
injectionType = Some(suggestType.name),
injectionType = Some(st.SuggestType.OrganicListTweet.name),
controllerData = None,
sourceData = None)),
articleDetails = None,

View File

@ -4,16 +4,9 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
"src/thrift/com/twitter/timelines/service:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
],
)

View File

@ -1,18 +0,0 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
class FeedbackStrings @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
private val externalStringRegistry = externalStringRegistryProvider.get()
val seeLessOftenFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOften")
val seeLessOftenConfirmationFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
}

View File

@ -5,6 +5,7 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
@ -31,13 +32,12 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
@Singleton
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
feedbackStrings: FeedbackStrings,
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
externalStringRegistry = externalStringRegistryProvider.get(),
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)

View File

@ -1,52 +0,0 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => tl}
import com.twitter.timelines.util.FeedbackRequestSerializer
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.timelineservice.thriftscala.FeedbackType
object HomeWhoToSubscribeFeedbackActionInfoBuilder {
private val FeedbackMetadata = tl.FeedbackMetadata(
injectionType = Some(SuggestType.WhoToSubscribe),
engagementType = None,
entityIds = Seq.empty,
ttlMs = None
)
private val FeedbackRequest =
tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
private val EncodedFeedbackRequest =
FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
}
@Singleton
case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest =
Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)
override def apply(
query: PipelineQuery,
candidate: UserCandidate,
candidateFeatures: FeatureMap
): Option[FeedbackActionInfo] =
whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
}

View File

@ -1,50 +0,0 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* "Your Lists" will be rendered for the context and a url link for your lists.
*/
@Singleton
case class ListsSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
candidateFeatures.get(SuggestTypeFeature) match {
case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
Some(
GeneralContext(
contextType = ListGeneralContextType,
text = stringCenter.prepare(listString),
url = userName.map(name => ""),
contextImageUrls = None,
landingUrl = None
))
case _ => None
}
}
}

View File

@ -1,43 +0,0 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
case class PopularInYourAreaSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
Some(
GeneralContext(
contextType = LocationGeneralContextType,
text = stringCenter.prepare(popularInYourAreaString),
url = None,
contextImageUrls = None,
landingUrl = None
))
} else None
}
}

View File

@ -1,7 +1,6 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -9,11 +8,9 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta}
import com.twitter.tweetconvosvc.{thriftscala => tcs}
import com.twitter.util.Future
import javax.inject.Inject
import javax.inject.Singleton
@ -26,16 +23,15 @@ class AncestorFeatureHydrator @Inject() (
override val features: Set[Feature[_, _]] = Set(AncestorsFeature)
private val DefaultFeatureMap = FeatureMapBuilder().add(AncestorsFeature, Seq.empty).build()
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture {
if (existingFeatures.getOrElse(InReplyToTweetIdFeature, None).isDefined) {
val ancestorsRequest = tcs.GetAncestorsRequest(Seq(candidate.id))
conversationServiceClient.getAncestors(ancestorsRequest).map { getAncestorsResponse =>
): Stitch[FeatureMap] = {
val ancestorsRequest = tcs.GetAncestorsRequest(Seq(candidate.id))
Stitch.callFuture(conversationServiceClient.getAncestors(ancestorsRequest)).map {
getAncestorsResponse =>
val ancestors = getAncestorsResponse.ancestors.headOption
.collect {
case tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult)
@ -44,8 +40,7 @@ class AncestorFeatureHydrator @Inject() (
}.getOrElse(Seq.empty)
FeatureMapBuilder().add(AncestorsFeature, ancestors).build()
}
} else Future.value(DefaultFeatureMap)
}
}
private def getTruncatedRootTweet(

View File

@ -1,28 +1,28 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.AuthorFeatureRepository
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter
import com.twitter.home_mixer.util.CandidatesUtil
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.servo.repository.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch
import com.twitter.timelines.author_features.v1.{thriftjava => af}
import com.twitter.util.Future
import com.twitter.util.Try
import javax.inject.Inject
import javax.inject.Named
import javax.inject.Singleton
@ -51,28 +51,34 @@ class AuthorFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture {
val possiblyAuthorIds = extractKeys(candidates)
val authorIds = possiblyAuthorIds.flatten
): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val possiblyAuthorIds = extractKeys(candidates)
val authorIds = possiblyAuthorIds.flatten
val response: Future[KeyValueResult[Long, af.AuthorFeatures]] =
if (authorIds.nonEmpty) client(authorIds)
else Future.value(KeyValueResult.empty)
val response: Future[KeyValueResult[Long, af.AuthorFeatures]] =
if (authorIds.isEmpty) {
Future.value(KeyValueResult.empty)
} else {
client(authorIds)
}
response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value)
response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value)
FeatureMapBuilder().add(AuthorFeature, transformedValue).build()
FeatureMapBuilder()
.add(AuthorFeature, transformedValue)
.build()
}
}
}
}
private def postTransformer(authorFeatures: Try[Option[af.AuthorFeatures]]): Try[DataRecord] = {
authorFeatures.map {
_.map { features => AuthorFeaturesAdapter.adaptToDataRecords(features).asScala.head }
.getOrElse(new DataRecord())
authorFeatures.map { features =>
AuthorFeaturesAdapter.adaptToDataRecords(features).asScala.head
}
}
@ -80,7 +86,10 @@ class AuthorFeatureHydrator @Inject() (
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]] = {
candidates.map { candidate =>
CandidatesUtil.getOriginalAuthorId(candidate.features)
candidate.features
.getTry(AuthorIdFeature)
.toOption
.flatten
}
}
}

View File

@ -4,56 +4,95 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider",
"finatra/inject/inject-core/src/main/scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
"representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
"representation-scorer/server/src/main/thrift:thrift-scala",
"servo/repo/src/main/scala",
"snowflake/src/main/scala/com/twitter/snowflake/id",
"src/java/com/twitter/ml/api/constant",
"src/java/com/twitter/search/common/util/lang",
"src/scala/com/twitter/timelines/prediction/adapters/request_context",
"src/scala/com/twitter/ml/api/util",
"src/scala/com/twitter/timelines/prediction/adapters/real_graph",
"src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/adapters/twistly",
"src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
"src/scala/com/twitter/timelines/prediction/common/util",
"src/scala/com/twitter/timelines/prediction/features/common",
"src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/features/recap",
"src/scala/com/twitter/timelines/prediction/features/time_features",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/ml/api:data-java",
"src/thrift/com/twitter/ml/api:embedding-java",
"src/thrift/com/twitter/onboarding/relevance/features:features-java",
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"src/thrift/com/twitter/search/common:constants-java",
"src/thrift/com/twitter/socialgraph:thrift-scala",
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
"src/thrift/com/twitter/timelineranker:thrift-scala",
"src/thrift/com/twitter/timelines/author_features:thrift-java",
"src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
"src/thrift/com/twitter/timelines/impression:thrift-scala",
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
"src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala",
"src/thrift/com/twitter/user_session_store:thrift-java",
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
"src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
"stitch/stitch-core",
"stitch/stitch-gizmoduck",
"stitch/stitch-socialgraph",
"stitch/stitch-timelineservice",
"stitch/stitch-tweetypie",
"strato/config/columns/topic-signals/tsp",
"strato/config/columns/topic-signals/tsp:tsp-strato-client",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
"timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly",
"timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"topic-social-proof/server/src/main/thrift:thrift-scala",
"topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
"tweetconvosvc/thrift/src/main/thrift:thrift-scala",
"twitter-config/yaml",
"user_session_store/src/main/scala/com/twitter/user_session_store",
"util/util-core",
],

View File

@ -1,10 +1,9 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.EarlybirdAdapter
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.earlybird.EarlybirdAdapter
import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
@ -12,7 +11,6 @@ import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.home_mixer.util.earlybird.EarlybirdResponseUtil
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
@ -23,7 +21,6 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.search.earlybird.{thriftscala => eb}
import com.twitter.servo.keyvalue.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository
@ -53,12 +50,8 @@ class EarlybirdFeatureHydrator @Inject() (
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Earlybird")
override val features: Set[Feature[_, _]] = Set(
EarlybirdDataRecordFeature,
EarlybirdFeature,
EarlybirdSearchResultFeature,
TweetUrlsFeature
)
override val features: Set[Feature[_, _]] =
Set(EarlybirdDataRecordFeature, EarlybirdFeature, TweetUrlsFeature)
override val statScope: String = identifier.toString
@ -66,66 +59,52 @@ class EarlybirdFeatureHydrator @Inject() (
private val originalKeyFoundCounter = scopedStatsReceiver.counter("originalKey/found")
private val originalKeyLossCounter = scopedStatsReceiver.counter("originalKey/loss")
private val ebSearchResultNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean =
candidate => candidate.features.getOrElse(EarlybirdSearchResultFeature, None).isEmpty
private val ebFeaturesNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean =
candidate => candidate.features.getOrElse(EarlybirdFeature, None).isEmpty
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture {
): Stitch[Seq[FeatureMap]] = {
val candidatesToHydrate = candidates.filter { candidate =>
val isEmpty =
ebFeaturesNotExistPredicate(candidate) && ebSearchResultNotExistPredicate(candidate)
val isEmpty = ebFeaturesNotExistPredicate(candidate)
if (isEmpty) originalKeyLossCounter.incr() else originalKeyFoundCounter.incr()
isEmpty
}
client((candidatesToHydrate.map(_.candidate.id), query.getRequiredUserId))
.map(handleResponse(query, candidates, _, candidatesToHydrate))
Stitch
.callFuture(client((candidatesToHydrate.map(_.candidate.id), query.getRequiredUserId)))
.map(handleResponse(query, candidates, _))
}
private def handleResponse(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]],
results: KeyValueResult[Long, eb.ThriftSearchResult],
candidatesToHydrate: Seq[CandidateWithFeatures[TweetCandidate]]
results: KeyValueResult[Long, eb.ThriftSearchResult]
): Seq[FeatureMap] = {
val queryFeatureMap = query.features.getOrElse(FeatureMap.empty)
val userLanguages = queryFeatureMap.getOrElse(UserLanguagesFeature, Seq.empty)
val uiLanguageCode = queryFeatureMap.getOrElse(DeviceLanguageFeature, None)
val screenName = queryFeatureMap.getOrElse(UserScreenNameFeature, None)
val followedUserIds = queryFeatureMap.getOrElse(SGSFollowedUsersFeature, Seq.empty).toSet
val searchResults = candidatesToHydrate
.map { candidate =>
val searchResults = candidates
.filter(ebFeaturesNotExistPredicate).map { candidate =>
observedGet(Some(candidate.candidate.id), results)
}.collect {
case Return(Some(value)) => value
}
val allSearchResults = searchResults ++
candidates.filter(!ebSearchResultNotExistPredicate(_)).flatMap { candidate =>
candidate.features
.getOrElse(EarlybirdSearchResultFeature, None)
}
val idToSearchResults = allSearchResults.map(sr => sr.id -> sr).toMap
val tweetIdToEbFeatures = EarlybirdResponseUtil.getTweetThriftFeaturesByTweetId(
val tweetIdToEbFeatures = EarlybirdResponseUtil.getOONTweetThriftFeaturesByTweetId(
searcherUserId = query.getRequiredUserId,
screenName = screenName,
userLanguages = userLanguages,
uiLanguageCode = uiLanguageCode,
followedUserIds = followedUserIds,
mutuallyFollowingUserIds = Set.empty,
searchResults = allSearchResults,
sourceTweetSearchResults = Seq.empty,
searchResults = searchResults
)
candidates.map { candidate =>
val transformedEbFeatures = tweetIdToEbFeatures.get(candidate.candidate.id)
val hydratedEbFeatures = tweetIdToEbFeatures.get(candidate.candidate.id)
val earlybirdFeatures =
if (transformedEbFeatures.nonEmpty) transformedEbFeatures
if (hydratedEbFeatures.nonEmpty) hydratedEbFeatures
else candidate.features.getOrElse(EarlybirdFeature, None)
val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false)
@ -143,7 +122,6 @@ class EarlybirdFeatureHydrator @Inject() (
FeatureMapBuilder()
.add(EarlybirdFeature, earlybirdFeatures)
.add(EarlybirdDataRecordFeature, earlybirdDataRecord)
.add(EarlybirdSearchResultFeature, idToSearchResults.get(candidate.candidate.id))
.add(TweetUrlsFeature, earlybirdFeatures.flatMap(_.urlsList).getOrElse(Seq.empty))
.build()
}

View File

@ -1,10 +1,12 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
@ -15,12 +17,16 @@ import javax.inject.Singleton
@Singleton
case class FeedbackHistoryQueryFeatureHydrator @Inject() (
feedbackHistoryClient: FeedbackHistoryManhattanClient)
extends QueryFeatureHydrator[PipelineQuery] {
extends QueryFeatureHydrator[PipelineQuery]
with Conditionally[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableFeedbackFatigueParam)
override def hydrate(
query: PipelineQuery
): Stitch[FeatureMap] =

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.for_you.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature

View File

@ -0,0 +1,41 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class FollowedTopicsQueryFeatureHydrator @Inject() (
followedTopicsCandidateSource: FollowedTopicsCandidateSource)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
followedTopicsCandidateSource(request)
.map { topics =>
FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
}.handle {
case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
}
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
)
}

View File

@ -0,0 +1,58 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.gizmoduck.{thriftscala => gt}
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.stitch.gizmoduck.Gizmoduck
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("GizmoduckAuthorSafety")
override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
val blueVerifiedStitch = authorIdOption
.map { authorId =>
gizmoduck
.getUserById(
userId = authorId,
queryFields = queryFields
)
.map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
}.getOrElse(Stitch.False)
blueVerifiedStitch.map { isBlueVerified =>
FeatureMapBuilder()
.add(AuthorIsBlueVerifiedFeature, isBlueVerified)
.build()
}
}
}

View File

@ -1,13 +1,15 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.graph_feature_service.{thriftscala => gfs}
import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.IsExtendedReplyFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository
import com.twitter.home_mixer.util.CandidatesUtil
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.home_mixer.util.ReplyRetweetUtil
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
@ -19,7 +21,6 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.two_hop_features.TwoHopFeaturesAdapter
@ -57,19 +58,28 @@ class GraphTwoHopFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture {
// Apply filters to in network candidates for retweets only.
): Stitch[Seq[FeatureMap]] = {
// Apply filters to in network candidates for ExtendedReplyAncestors and retweets.
// ExtendedReplyAncestors should also be in candidates. No filter for oon.
val (inNetworkCandidates, oonCandidates) = candidates.partition { candidate =>
candidate.features.getOrElse(FromInNetworkSourceFeature, false)
candidate.features.getOrElse(InNetworkFeature, false)
}
val inNetworkCandidatesToHydrate =
val inNetworkReplyToAncestorTweet =
ReplyRetweetUtil.replyToAncestorTweetCandidatesMap(inNetworkCandidates)
val inNetworkExtendedReplyAncestors = inNetworkCandidates
.filter(_.features.getOrElse(IsExtendedReplyFeature, false)).flatMap { inNetworkCandidate =>
inNetworkReplyToAncestorTweet.get(inNetworkCandidate.candidate.id)
}.flatten
val inNetworkCandidatesToHydrate = inNetworkExtendedReplyAncestors ++
inNetworkCandidates.filter(_.features.getOrElse(IsRetweetFeature, false))
val candidatesToHydrate = (inNetworkCandidatesToHydrate ++ oonCandidates)
.flatMap(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)).distinct
val response = client((candidatesToHydrate, query.getRequiredUserId))
val response = Stitch.callFuture(client((candidatesToHydrate, query.getRequiredUserId)))
response.map { result =>
candidates.map { candidate =>
@ -77,9 +87,7 @@ class GraphTwoHopFeatureHydrator @Inject() (
val value = observedGet(key = originalAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value)
val followedByUserIds = value.toOption
.flatMap(getFollowedByUserIds(_))
.getOrElse(Seq.empty)
val followedByUserIds = value.toOption.flatMap(getFollowedByUserIds(_)).getOrElse(Seq.empty)
FeatureMapBuilder()
.add(GraphTwoHopFeature, transformedValue)

View File

@ -3,7 +3,6 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -12,8 +11,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
import javax.inject.Inject
import javax.inject.Singleton
@ -21,39 +19,36 @@ import javax.inject.Singleton
@Singleton
case class ImpressionBloomFilterQueryFeatureHydrator[
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
bloomFilterClient: ManhattanStoreClient[
blm.ImpressionBloomFilterKey,
blm.ImpressionBloomFilterSeq
]) extends QueryFeatureHydrator[Query] {
bloomFilter: ImpressionBloomFilter)
extends QueryFeatureHydrator[Query] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
"ImpressionBloomFilter")
private val ImpressionBloomFilterTTL = 7.day
private val ImpressionBloomFilterFalsePositiveRate = 0.002
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
private val SurfaceArea = t.SurfaceArea.HomeTimeline
override def hydrate(query: Query): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
bloomFilterClient
.get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
.map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
.map { bloomFilterSeq =>
val updatedBloomFilterSeq =
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
else {
ImpressionBloomFilter.addSeenTweetIds(
surfaceArea = SurfaceArea,
tweetIds = query.seenTweetIds.get,
bloomFilterSeq = bloomFilterSeq,
timeToLive = ImpressionBloomFilterTTL,
falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam)
)
}
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
}
bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq =>
val updatedBloomFilterSeq =
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
else {
bloomFilter.addElements(
userId = userId,
surfaceArea = SurfaceArea,
tweetIds = query.seenTweetIds.get,
bloomFilterEntrySeq = bloomFilterSeq,
timeToLive = ImpressionBloomFilterTTL,
falsePositiveRate = ImpressionBloomFilterFalsePositiveRate
)
}
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
}
}
override val alerts = Seq(

View File

@ -1,41 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object InNetworkFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
val featureMaps = candidates.map { candidate =>
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
isSelfTweet || followedUserIds.contains(authorId)
}
FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
}
Stitch.value(featureMaps)
}
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.request.HasListId
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -15,18 +15,17 @@ import com.twitter.stitch.socialgraph.SocialGraph
import javax.inject.Inject
import javax.inject.Singleton
case object RecentListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] {
case object ListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] {
override val defaultValue: Seq[Long] = Seq.empty
}
@Singleton
class RecentListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph)
class ListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph)
extends QueryFeatureHydrator[PipelineQuery with HasListId] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("RecentListMembers")
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListMembers")
override val features: Set[Feature[_, _]] = Set(RecentListMembersFeature)
override val features: Set[Feature[_, _]] = Set(ListMembersFeature)
private val MaxRecentMembers = 10
@ -37,7 +36,7 @@ class RecentListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph)
pageRequest = Some(sg.PageRequest(selectAll = Some(true), count = Some(MaxRecentMembers)))
)
socialGraph.ids(request).map(_.ids).map { listMembers =>
FeatureMapBuilder().add(RecentListMembersFeature, listMembers).build()
FeatureMapBuilder().add(ListMembersFeature, listMembers).build()
}
}
}

View File

@ -1,4 +1,5 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer
package functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
@ -13,11 +14,11 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.keyvalue.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch
import com.twitter.util.Future
import javax.inject.Inject
import javax.inject.Named
import javax.inject.Singleton
@ -44,17 +45,25 @@ class MetricCenterUserCountingFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture {
val possiblyAuthorIds = extractKeys(candidates)
val userIds = possiblyAuthorIds.flatten
): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val possiblyAuthorIds = extractKeys(candidates)
val userIds = possiblyAuthorIds.flatten
val response: Future[KeyValueResult[Long, rf.MCUserCountingFeatures]] =
if (userIds.isEmpty) Future.value(KeyValueResult.empty) else client(userIds)
val response: Future[KeyValueResult[Long, rf.MCUserCountingFeatures]] = if (userIds.isEmpty) {
Future.value(KeyValueResult.empty)
} else {
client(userIds)
}
response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
FeatureMapBuilder().add(MetricCenterUserCountingFeature, value).build()
response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
FeatureMapBuilder()
.add(MetricCenterUserCountingFeature, value)
.build()
}
}
}
}

View File

@ -2,10 +2,8 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
@ -29,27 +27,17 @@ import javax.inject.Singleton
@Singleton
case class PersistenceStoreQueryFeatureHydrator @Inject() (
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3],
statsReceiver: StatsReceiver)
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
private val WhoToFollowExcludedUserIdsLimit = 1000
private val ServedTweetIdsDuration = 10.minutes
private val ServedTweetIdsDuration = 1.hour
private val ServedTweetIdsLimit = 100
private val ServedTweetPreviewIdsDuration = 10.hours
private val ServedTweetPreviewIdsLimit = 10
override val features: Set[Feature[_, _]] =
Set(
ServedTweetIdsFeature,
ServedTweetPreviewIdsFeature,
PersistenceEntriesFeature,
WhoToFollowExcludedUserIdsFeature)
Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature)
private val supportedClients = Seq(
ClientPlatform.IPhone,
@ -92,19 +80,8 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
.flatMap(
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
servedTweetIdsSizeStat.add(servedTweetIds.size)
val servedTweetPreviewIds = timelineResponses
.filter(_.clientPlatform == clientPlatform)
.filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
.sortBy(-_.servedTime.inMilliseconds)
.flatMap(_.entries
.filter(_.entityIdType == EntityIdType.TweetPreview)
.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
FeatureMapBuilder()
.add(ServedTweetIdsFeature, servedTweetIds)
.add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
.add(PersistenceEntriesFeature, timelineResponses)
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
.build()

View File

@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.stitch.timelineservice.TimelineService
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
@ -38,10 +37,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
): Stitch[Seq[FeatureMap]] = {
val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
candidate.features
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.map(favoritedBy => favoritedBy -> candidate.candidate.id)
}
@ -60,7 +59,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
candidates.map { candidate =>
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
FeatureMapBuilder()

View File

@ -32,15 +32,11 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
val realGraphScoresFeatures = realGraphFollowedUsers
.getOrElse(Seq.empty)
.sortBy(-_.score)
.map(candidate => candidate.userId -> scaleScore(candidate.score))
.map(candidate => candidate.userId -> candidate.score)
.take(RealGraphCandidateCount)
.toMap
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
}
}
// Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
private def scaleScore(score: Double): Double =
if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphFeatureRepository
import com.twitter.product_mixer.core.feature.Feature

View File

@ -1,8 +1,8 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures
import com.twitter.home_mixer.util.MissingKeyException
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -14,7 +14,6 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter
import com.twitter.timelines.prediction.adapters.real_graph.RealGraphFeaturesAdapter
@ -60,13 +59,13 @@ class RealGraphViewerAuthorFeatureHydrator @Inject() ()
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = OffloadFuturePools.offload {
): Stitch[FeatureMap] = {
val viewerId = query.getRequiredUserId
val realGraphFeatures = query.features
.flatMap(_.getOrElse(RealGraphFeatures, None))
.getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures])
existingFeatures.getOrElse(AuthorIdFeature, None) match {
val result: FeatureMap = existingFeatures.getOrElse(AuthorIdFeature, None) match {
case Some(authorId) =>
val realGraphAuthorFeatures =
getRealGraphViewerAuthorFeatures(viewerId, authorId, realGraphFeatures)
@ -91,6 +90,7 @@ class RealGraphViewerAuthorFeatureHydrator @Inject() ()
.build()
case _ => MissingKeyFeatureMap
}
Stitch(result)
}
private def getRealGraphViewerAuthorFeatures(

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter
import com.twitter.timelines.real_graph.v1.{thriftscala => v1}
@ -44,23 +43,25 @@ class RealGraphViewerRelatedUsersFeatureHydrator @Inject() ()
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = OffloadFuturePools.offload {
): Stitch[FeatureMap] = {
val realGraphQueryFeatures = query.features
.flatMap(_.getOrElse(RealGraphFeatures, None))
.getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures])
val allRelatedUserIds = getRelatedUserIds(existingFeatures)
val realGraphFeatures = RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures(
allRelatedUserIds,
realGraphQueryFeatures
)
val realGraphFeatures =
RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures(
allRelatedUserIds,
realGraphQueryFeatures)
val realGraphFeaturesDataRecord = RealGraphEdgeFeaturesCombineAdapter
.adaptToDataRecords(Some(realGraphFeatures)).asScala.headOption
.getOrElse(new DataRecord)
FeatureMapBuilder()
.add(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord)
.build()
Stitch.value {
FeatureMapBuilder()
.add(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord)
.build()
}
}
private def getRelatedUserIds(features: FeatureMap): Seq[Long] = {

View File

@ -1,22 +1,22 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.realtime_interaction_graph.RealTimeInteractionGraphFeaturesAdapter
import com.twitter.timelines.prediction.features.realtime_interaction_graph.RealTimeInteractionGraphEdgeFeatures
import com.twitter.util.Time
import javax.inject.Inject
import javax.inject.Singleton
import scala.collection.JavaConverters._
@ -31,8 +31,8 @@ object RealTimeInteractionGraphEdgeFeature
class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() ()
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("RealTimeInteractionGraphEdge")
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
"RealTimeInteractionGraphEdge")
override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphEdgeFeature)
@ -41,21 +41,24 @@ class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() ()
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload {
): Stitch[Seq[FeatureMap]] = {
val userVertex =
query.features.flatMap(_.getOrElse(RealTimeInteractionGraphUserVertexQueryFeature, None))
val realTimeInteractionGraphFeaturesMap =
userVertex.map(RealTimeInteractionGraphEdgeFeatures(_, Time.now))
candidates.map { candidate =>
val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId =>
realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId))
Stitch.value {
candidates.map { candidate =>
val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId =>
realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId))
}
FeatureMapBuilder()
.add(
RealTimeInteractionGraphEdgeFeature,
realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head)
.build()
}
val dataRecordFeature =
realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head
FeatureMapBuilder().add(RealTimeInteractionGraphEdgeFeature, dataRecordFeature).build()
}
}
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.google.inject.name.Named
import com.twitter.finagle.stats.StatsReceiver
@ -13,6 +13,7 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.servo.cache.ReadCache
import com.twitter.stitch.Stitch
import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,56 +1,28 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.ContentFeatures
import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.InReplyToContentFeatureAdapter
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.InReplyToEarlybirdAdapter
import com.twitter.home_mixer.util.ReplyRetweetUtil
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures
import com.twitter.timelines.conversation_features.{thriftscala => cf}
import com.twitter.timelines.prediction.adapters.conversation_features.ConversationFeaturesAdapter
import com.twitter.util.Duration
import com.twitter.util.Time
import javax.inject.Inject
import javax.inject.Singleton
import scala.collection.JavaConverters._
object InReplyToTweetHydratedEarlybirdFeature
extends Feature[TweetCandidate, Option[ThriftTweetFeatures]]
object ConversationDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object InReplyToEarlybirdDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object InReplyToTweetypieContentDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
/**
* The purpose of this hydrator is to
* 1) hydrate simple features into replies and their ancestor tweets
@ -64,19 +36,13 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver)
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ReplyTweet")
override val features: Set[Feature[_, _]] = Set(
ConversationDataRecordFeature,
InReplyToTweetHydratedEarlybirdFeature,
InReplyToEarlybirdDataRecordFeature,
InReplyToTweetypieContentDataRecordFeature
ConversationFeature,
InReplyToTweetHydratedEarlybirdFeature
)
private val defaulDataRecord: DataRecord = new DataRecord()
private val DefaultFeatureMap = FeatureMapBuilder()
.add(ConversationDataRecordFeature, defaulDataRecord)
.add(ConversationFeature, None)
.add(InReplyToTweetHydratedEarlybirdFeature, None)
.add(InReplyToEarlybirdDataRecordFeature, defaulDataRecord)
.add(InReplyToTweetypieContentDataRecordFeature, defaulDataRecord)
.build()
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
@ -86,7 +52,7 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload {
): Stitch[Seq[FeatureMap]] = {
val replyToInReplyToTweetMap =
ReplyRetweetUtil.replyTweetIdToInReplyToTweetMap(candidates)
val candidatesWithRepliesHydrated = candidates.map { candidate =>
@ -128,28 +94,16 @@ class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver)
inReplyToTweetEarlyBirdFeature,
updatedReplyConversationFeatures))
}
candidatesWithRepliesAndAncestorTweetsHydrated.map {
case (candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) =>
val conversationDataRecordFeature = updatedConversationFeatures
.map(f => ConversationFeaturesAdapter.adaptToDataRecord(cf.ConversationFeatures.V1(f)))
.getOrElse(defaulDataRecord)
val inReplyToEarlybirdDataRecord =
InReplyToEarlybirdAdapter
.adaptToDataRecords(inReplyToTweetEarlyBirdFeature).asScala.head
val inReplyToContentDataRecord = InReplyToContentFeatureAdapter
.adaptToDataRecords(
inReplyToTweetEarlyBirdFeature.map(ContentFeatures.fromThrift)).asScala.head
FeatureMapBuilder()
.add(ConversationDataRecordFeature, conversationDataRecordFeature)
.add(InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature)
.add(InReplyToEarlybirdDataRecordFeature, inReplyToEarlybirdDataRecord)
.add(InReplyToTweetypieContentDataRecordFeature, inReplyToContentDataRecord)
.build()
case _ => DefaultFeatureMap
}
Stitch.value(
candidatesWithRepliesAndAncestorTweetsHydrated.map {
case (candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) =>
FeatureMapBuilder()
.add(ConversationFeature, updatedConversationFeatures)
.add(InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature)
.build()
case _ => DefaultFeatureMap
}
)
}
private def hydratedReplyCandidate(

View File

@ -17,13 +17,9 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
import com.twitter.search.common.util.lang.ThriftLanguageUtil
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
import java.util.UUID
import javax.inject.Inject
import javax.inject.Singleton
@ -49,9 +45,6 @@ class RequestQueryFeatureHydrator[
PullToRefreshFeature,
RequestJoinIdFeature,
ServedRequestIdFeature,
TimestampFeature,
TimestampGMTDowFeature,
TimestampGMTHourFeature,
ViewerIdFeature
)
@ -74,7 +67,6 @@ class RequestQueryFeatureHydrator[
override def hydrate(query: Query): Stitch[FeatureMap] = {
val requestContext = query.deviceContext.flatMap(_.requestContextValue)
val servedRequestId = UUID.randomUUID.getMostSignificantBits
val timestamp = query.queryTime.inMilliseconds
val featureMap = FeatureMapBuilder()
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
@ -105,15 +97,8 @@ class RequestQueryFeatureHydrator[
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
.add(ServedRequestIdFeature, Some(servedRequestId))
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
.add(TimestampFeature, timestamp)
.add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
.add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
.add(HasDarkRequestFeature, hasDarkRequest)
.add(
ViewerIdFeature,
query.getOptionalUserId
.orElse(query.getGuestId).getOrElse(
throw PipelineFailure(BadRequest, "Missing viewer id")))
.add(ViewerIdFeature, query.getRequiredUserId)
.build()
Stitch.value(featureMap)

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature
@ -22,8 +22,8 @@ object SourceTweetEarlybirdFeature extends Feature[TweetCandidate, Option[Thrift
object RetweetSourceTweetFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("RetweetSourceTweet")
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
"RetweetSourceTweet")
override val features: Set[Feature[_, _]] = Set(
SourceTweetEarlybirdFeature,

View File

@ -0,0 +1,46 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton
case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
socialGraphStitchClient: SocialGraphStitchClient)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SGSFollowedUsers")
override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
private val SocialGraphLimit = 14999
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
val request = sg.IdsRequest(
relationships = Seq(
sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
),
pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
)
socialGraphStitchClient
.ids(request).map(_.ids)
.map { followedUsers =>
FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
}
}
}

View File

@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph
@ -42,7 +41,8 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
): Stitch[Seq[FeatureMap]] = {
val allSocialContextUserIds =
candidates.flatMap { candidate =>
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++

View File

@ -1,5 +1,6 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -13,7 +14,6 @@ import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient
import com.twitter.timelines.configapi.decider.BooleanDeciderParam
@ -29,7 +29,8 @@ object SimClustersEngagementSimilarityFeature
@Singleton
class SimClustersEngagementSimilarityFeatureHydrator @Inject() (
simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient)
simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient,
statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery] {
@ -38,6 +39,10 @@ class SimClustersEngagementSimilarityFeatureHydrator @Inject() (
override val features: Set[Feature[_, _]] = Set(SimClustersEngagementSimilarityFeature)
private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
private val hydratedCandidatesSizeStat = scopedStatsReceiver.stat("hydratedCandidatesSize")
private val simClustersRecentEngagementSimilarityFeaturesAdapter =
new SimClustersRecentEngagementSimilarityFeaturesAdapter
@ -50,14 +55,15 @@ class SimClustersEngagementSimilarityFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture {
): Stitch[Seq[FeatureMap]] = {
val tweetToCandidates = candidates.map(candidate => candidate.candidate.id -> candidate).toMap
val tweetIds = tweetToCandidates.keySet.toSeq
val userId = query.getRequiredUserId
val userTweetEdges = tweetIds.map(tweetId => (userId, tweetId))
simClustersEngagementSimilarityClient
val resultFuture = simClustersEngagementSimilarityClient
.getSimClustersRecentEngagementSimilarityScores(userTweetEdges).map {
simClustersRecentEngagementSimilarityScoresMap =>
hydratedCandidatesSizeStat.add(simClustersRecentEngagementSimilarityScoresMap.size)
candidates.map { candidate =>
val similarityFeatureOpt = simClustersRecentEngagementSimilarityScoresMap
.get(userId -> candidate.candidate.id).flatten
@ -71,5 +77,7 @@ class SimClustersEngagementSimilarityFeatureHydrator @Inject() (
.build()
}
}
Stitch.callFuture(resultFuture)
}
}

View File

@ -0,0 +1,67 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SocialGraphService")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
val idsRequest = createIdsRequest(
userId = viewerId,
relationshipTypes = Set(sg.RelationshipType.Following),
targetIds = Some(distinctNonSelfAuthorIds)
)
socialGraphStitchClient
.ids(request = idsRequest, requestContext = None)
.map { idResult =>
authorIds.map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
val inNetworkAuthorIds = idResult.ids.toSet
val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
}
}
}
private def createIdsRequest(
userId: Long,
relationshipTypes: Set[sg.RelationshipType],
targetIds: Option[Seq[Long]] = None
): sg.IdsRequest = sg.IdsRequest(
relationshipTypes.map { relationshipType =>
sg.SrcRelationship(userId, relationshipType, targets = targetIds)
}.toSeq,
Some(sg.PageRequest(selectAll = Some(true)))
)
}

View File

@ -0,0 +1,162 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.contentrecommender.{thriftscala => cr}
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
import com.twitter.topiclisting.TopicListingViewerContext
import com.twitter.tsp.{thriftscala => tsp}
import javax.inject.Inject
import javax.inject.Singleton
import scala.collection.JavaConverters._
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
object TSPInferredTopicDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TSPInferredTopicFeatureHydrator @Inject() (
topicSocialProofClientColumn: TopicSocialProofClientColumn,
statsReceiver: StatsReceiver,
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
override val features: Set[Feature[_, _]] =
Set(
TSPInferredTopicFeature,
TSPInferredTopicDataRecordFeature,
TopicIdSocialContextFeature,
TopicContextFunctionalityTypeFeature)
private val topK = 3
private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
sid.CandidateTweetSourceId.Simcluster,
sid.CandidateTweetSourceId.CroonTweet
)
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
private val DefaultFeatureMap = FeatureMapBuilder()
.add(TSPInferredTopicFeature, Map.empty[Long, Double])
.add(TSPInferredTopicDataRecordFeature, new DataRecord())
.add(TopicIdSocialContextFeature, None)
.add(TopicContextFunctionalityTypeFeature, None)
.build()
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val tags = candidates.collect {
case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
candidate.candidate.id -> candidate.features
.getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
}.toMap
val topicSocialProofRequest =
tsp.TopicSocialProofRequest(
userId = query.getRequiredUserId,
tweetIds = candidates.map(_.candidate.id).toSet,
displayLocation = cr.DisplayLocation.HomeTimeline,
topicListingSetting = tsp.TopicListingSetting.Followable,
context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
bypassModes = None,
// Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
tags = if (tags.isEmpty) None else Some(tags)
)
topicSocialProofClientColumn.fetcher
.fetch(topicSocialProofRequest)
.map(_.v)
.map {
case Some(response) =>
candidates.map { candidate =>
val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
if (topicWithScores.nonEmpty) {
keyFoundCounter.incr()
val (socialProofId, socialProofFunctionalityType) =
if (candidate.features
.getOrElse(CandidateSourceIdFeature, None)
.exists(sourcesToSetSocialProof.contains)) {
getSocialProof(topicWithScores)
} else {
(None, None)
}
val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
val inferredTopicDataRecord =
InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
FeatureMapBuilder()
.add(TSPInferredTopicFeature, inferredTopicFeatures)
.add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
.add(TopicIdSocialContextFeature, socialProofId)
.add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
.build()
} else {
keyLossCounter.incr()
DefaultFeatureMap
}
}
case _ =>
requestFailCounter.incr()
candidates.map { _ =>
DefaultFeatureMap
}
}
}
private def getSocialProof(
topicWithScores: Seq[tsp.TopicWithScore]
): (Option[Long], Option[TopicContextFunctionalityType]) = {
val followingTopicId = topicWithScores
.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
topicId
}
if (followingTopicId.nonEmpty) {
return (followingTopicId, Some(BasicTopicContextFunctionalityType))
}
val implicitFollowingId = topicWithScores.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
topicId
}
if (implicitFollowingId.nonEmpty) {
return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
}
(None, None)
}
private def convertTopicWithScores(
topicWithScores: Seq[tsp.TopicWithScore],
): Map[Long, Double] = {
topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
}
}

View File

@ -0,0 +1,251 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.FDsl._
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.search.common.features.{thriftscala => sc}
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
import com.twitter.timelines.prediction.features.time_features.TimeFeatures
import com.twitter.util.Duration
import scala.collection.Searching._
object TimeFeaturesDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
Stitch.value {
val richDataRecord = new RichDataRecord()
setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
FeatureMapBuilder()
.add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
.build()
}
}
private def setTimeFeatures(
richDataRecord: RichDataRecord,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
query: PipelineQuery,
): Unit = {
val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
}
private[feature_hydrator] def getTimeFeatures(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
): Option[TimeFeatures] = {
for {
requestTimestampMs <- Some(query.queryTime.inMilliseconds)
tweetId <- Some(candidate.id)
viewerId <- query.getOptionalUserId
tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
}
timeSinceSourceTweetCreation =
existingFeatures
.getOrElse(SourceTweetIdFeature, None)
.flatMap { sourceTweetId =>
timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
requestTimestampMs - sourceTweetCreationTimeMs
}
}
.getOrElse(timeSinceTweetCreation)
if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
} yield {
val timeFeatures = TimeFeatures(
timeSinceTweetCreation = timeSinceTweetCreation,
timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
)
val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
existingFeatures.getOrElse(EarlybirdFeature, None),
timeFeatures,
timeSinceSourceTweetCreation
).getOrElse(timeFeatures)
val nonPollingTimestampsMs =
query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
timeFeaturesWithLastEngagement,
requestTimestampMs,
tweetCreationTimeMs,
nonPollingTimestampsMs
)
timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
}
}
private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
if (SnowflakeId.isSnowflakeId(tweetOrUserId))
Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
else None
}
private def addLastEngagementTimeFeatures(
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
timeFeatures: TimeFeatures,
timeSinceSourceTweetCreation: Long
): Option[TimeFeatures] = {
tweetFeaturesOpt.map { tweetFeatures =>
val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
timeFeatures.copy(
lastFavSinceCreationHrs = lastFavSinceCreationHrs,
lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
lastReplySinceCreationHrs = lastReplySinceCreationHrs,
lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
lastFavSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
lastRetweetSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
lastReplySinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
lastQuoteSinceCreationHrs,
timeSinceSourceTweetCreation
)
)
}
}
private def addNonPollingTimeFeatures(
timeFeatures: TimeFeatures,
requestTimestampMs: Long,
creationTimeMs: Long,
nonPollingTimestampsMs: Option[Seq[Long]]
): Option[TimeFeatures] = {
for {
nonPollingTimestampsMs <- nonPollingTimestampsMs
lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
} yield {
val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
1.0,
timeFeatures.timeSinceTweetCreation
)
/*
* Non-polling timestamps are stored in chronological order.
* The latest timestamps occur first, therefore we need to explicitly search in reverse order.
*/
val nonPollingRequestsSinceTweetCreation =
if (nonPollingTimestampsMs.nonEmpty) {
nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
} else {
0
}
/*
* Calculate the average time between non-polling requests; include
* request time in this calculation as latest timestamp.
*/
val timeBetweenNonPollingRequestsAvg =
(requestTimestampMs - earliestNonPollingTimestampMs) / math
.max(1.0, nonPollingTimestampsMs.size)
val timeFeaturesWithNonPolling = timeFeatures.copy(
timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
tweetAgeRatio = Some(tweetAgeRatio)
)
timeFeaturesWithNonPolling
}
}
private[this] def getTimeSinceLastEngagementHrs(
lastEngagementTimeSinceCreationHrsOpt: Option[Double],
timeSinceTweetCreation: Long
): Option[Double] = {
lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
}
}
private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
val record = richDataRecord.getRecord
.setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
.setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
.setFeatureValueFromOption(
TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
features.timeSinceViewerAccountCreationSecs)
.setFeatureValue(
USER_ID_IS_SNOWFLAKE_ID,
features.timeSinceViewerAccountCreationSecs.isDefined
)
.setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
.setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
.setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
.setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
.setFeatureValueFromOption(
LAST_RETWEET_SINCE_CREATION_HRS,
features.lastRetweetSinceCreationHrs
)
.setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
.setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
/*
* set features whose values are optional as some users do not have non-polling timestamps
*/
features.timeBetweenNonPollingRequestsAvg.foreach(
record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
)
features.timeSinceLastNonPollingRequest.foreach(
record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
)
features.nonPollingRequestsSinceTweetCreation.foreach(
record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
)
features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
}
}

View File

@ -1,7 +1,6 @@
package com.twitter.home_mixer.product.for_you.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller
import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature
import com.twitter.home_mixer.model.request.DeviceContext
import com.twitter.home_mixer.model.request.HasDeviceContext
import com.twitter.home_mixer.service.HomeMixerAlertConfig
@ -17,6 +16,8 @@ import com.twitter.timelineservice.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Singleton
object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton
case class TimelineServiceTweetsQueryFeatureHydrator @Inject() (
timelineService: TimelineService,

View File

@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
extends QueryFeatureHydrator[Query] {
private val TweetImpressionTTL = 2.days
private val TweetImpressionCap = 5000
private val TweetImpressionTTL = 1.day
private val TweetImpressionCap = 3000
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.util.CandidatesUtil
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures
import java.lang.{Long => JLong}
@ -37,10 +36,16 @@ object TweetMetaDataFeatureHydrator
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = OffloadFuturePools.offload {
): Stitch[FeatureMap] = {
val richDataRecord = new RichDataRecord()
setFeatures(richDataRecord, candidate, existingFeatures)
FeatureMapBuilder().add(TweetMetaDataDataRecord, richDataRecord.getRecord).build()
Stitch.value {
FeatureMapBuilder()
.add(TweetMetaDataDataRecord, richDataRecord.getRecord)
.build()
}
}
private def setFeatures(

View File

@ -1,12 +1,12 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.escherbird.{thriftscala => esb}
import com.twitter.finagle.stats.Stat
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ContentFeatureAdapter
import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.ContentFeatureAdapter
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.home_mixer.util.tweetypie.content.FeatureExtractionHelper
import com.twitter.ml.api.DataRecord
@ -64,41 +64,45 @@ class TweetypieContentFeatureHydrator @Inject() (
private val bulkPostTransformerLatencyStat =
statsReceiver.scope(statScope).scope("bulkPostTransformer").stat("latency_ms")
private val DefaultDataRecord: DataRecord = new DataRecord()
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture {
val tweetIdsToHydrate = candidates.map(getCandidateOriginalTweetId).distinct
): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val tweetIdsToHydrate = candidates.map(getCandidateOriginalTweetId).distinct
val response: Future[KeyValueResult[Long, tp.Tweet]] = Stat.timeFuture(bulkRequestLatencyStat) {
if (tweetIdsToHydrate.isEmpty) Future.value(KeyValueResult.empty)
else client(tweetIdsToHydrate)
}
response.flatMap { result =>
Stat.timeFuture(bulkPostTransformerLatencyStat) {
OffloadFuturePools
.parallelize[CandidateWithFeatures[TweetCandidate], Try[(Seq[Long], DataRecord)]](
candidates,
parTransformer(result, _),
parallelism = 32,
default = Return((Seq.empty, DefaultDataRecord))
).map {
_.map {
case Return(result) =>
FeatureMapBuilder()
.add(MediaUnderstandingAnnotationIdsFeature, result._1)
.add(TweetypieContentDataRecordFeature, result._2)
.build()
case Throw(e) =>
FeatureMapBuilder()
.add(MediaUnderstandingAnnotationIdsFeature, Throw(e))
.add(TweetypieContentDataRecordFeature, Throw(e))
.build()
}
val response: Future[KeyValueResult[Long, tp.Tweet]] =
Stat.timeFuture(bulkRequestLatencyStat)(
if (tweetIdsToHydrate.isEmpty) {
Future.value(KeyValueResult.empty)
} else {
client(tweetIdsToHydrate)
}
)
response.flatMap { result =>
Stat.timeFuture(bulkPostTransformerLatencyStat) {
OffloadFuturePools
.parallelize[CandidateWithFeatures[TweetCandidate], Try[(Seq[Long], DataRecord)]](
candidates,
parTransformer(result, _),
parallelism = 32,
default = Return((Seq.empty, new DataRecord))
).map {
_.map {
case Return(result) =>
FeatureMapBuilder()
.add(MediaUnderstandingAnnotationIdsFeature, result._1)
.add(TweetypieContentDataRecordFeature, result._2)
.build()
case Throw(e) =>
FeatureMapBuilder()
.add(MediaUnderstandingAnnotationIdsFeature, Throw(e))
.add(TweetypieContentDataRecordFeature, Throw(e))
.build()
}
}
}
}
}
}
@ -108,6 +112,7 @@ class TweetypieContentFeatureHydrator @Inject() (
candidate: CandidateWithFeatures[TweetCandidate]
): Try[(Seq[Long], DataRecord)] = {
val originalTweetId = Some(getCandidateOriginalTweetId(candidate))
val value = observedGet(key = originalTweetId, keyValueResult = result)
Stat.time(postTransformerLatencyStat)(postTransformer(value))
}

View File

@ -1,9 +1,6 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
@ -13,13 +10,11 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.util.tweetypie.RequestFields
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
@ -34,22 +29,17 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
import com.twitter.stitch.Stitch
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
import com.twitter.tweetypie.{thriftscala => tp}
import com.twitter.util.logging.Logging
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class TweetypieFeatureHydrator @Inject() (
tweetypieStitchClient: TweetypieStitchClient,
statsReceiver: StatsReceiver)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Logging {
class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
override val features: Set[Feature[_, _]] = Set(
AuthorIdFeature,
ExclusiveConversationAuthorIdFeature,
InReplyToTweetIdFeature,
IsHydratedFeature,
IsNsfw,
@ -61,7 +51,6 @@ class TweetypieFeatureHydrator @Inject() (
SourceTweetIdFeature,
SourceUserIdFeature,
TweetTextFeature,
TweetLanguageFeature,
VisibilityReason
)
@ -81,12 +70,9 @@ class TweetypieFeatureHydrator @Inject() (
): Stitch[FeatureMap] = {
val safetyLevel = query.product match {
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
case ForYouProduct =>
val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
case ForYouProduct => rtf.SafetyLevel.TimelineHome
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
}
@ -96,12 +82,9 @@ class TweetypieFeatureHydrator @Inject() (
includeQuotedTweet = true,
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
safetyLevel = Some(safetyLevel),
forUserId = query.getOptionalUserId
forUserId = Some(query.getRequiredUserId)
)
val exclusiveAuthorIdOpt =
existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
val coreData = found.tweet.coreData
@ -123,7 +106,6 @@ class TweetypieFeatureHydrator @Inject() (
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
val tweetText = coreData.map(_.text)
val tweetLanguage = found.tweet.language.map(_.language)
val tweetAuthorId = coreData.map(_.userId)
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
@ -145,7 +127,6 @@ class TweetypieFeatureHydrator @Inject() (
FeatureMapBuilder()
.add(AuthorIdFeature, tweetAuthorId)
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, inReplyToTweetId)
.add(IsHydratedFeature, true)
.add(IsNsfw, Some(isNsfw))
@ -156,24 +137,20 @@ class TweetypieFeatureHydrator @Inject() (
.add(QuotedUserIdFeature, quotedTweetUserId)
.add(SourceTweetIdFeature, retweetedTweetId)
.add(SourceUserIdFeature, retweetedTweetUserId)
.add(TweetLanguageFeature, tweetLanguage)
.add(TweetTextFeature, tweetText)
.add(VisibilityReason, found.suppressReason)
.build()
// If no tweet result found, return default and pre-existing features
case _ =>
DefaultFeatureMap ++ FeatureMapBuilder()
.add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
.add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
.add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
.add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
.add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
.add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
.add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
.build()
DefaultFeatureMap +
(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) +
(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) +
(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) +
(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) +
(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) +
(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) +
(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
}
}
}

View File

@ -1,10 +1,9 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.google.inject.name.Named
import com.twitter.conversions.DurationOps.RichDuration
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature
import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
@ -14,6 +13,7 @@ import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature
import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SemanticAnnotationFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache
@ -47,7 +47,6 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
override val features: Set[Feature[_, _]] = Set(
AuthorIdFeature,
DirectedAtUserIdFeature,
ExclusiveConversationAuthorIdFeature,
HasImageFeature,
HasVideoFeature,
InReplyToTweetIdFeature,
@ -57,6 +56,7 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
MentionUserIdFeature,
QuotedTweetIdFeature,
QuotedUserIdFeature,
SemanticAnnotationFeature,
SourceTweetIdFeature,
SourceUserIdFeature
)
@ -66,7 +66,6 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
private val DefaultFeatureMap = FeatureMapBuilder()
.add(AuthorIdFeature, None)
.add(DirectedAtUserIdFeature, None)
.add(ExclusiveConversationAuthorIdFeature, None)
.add(HasImageFeature, false)
.add(HasVideoFeature, false)
.add(InReplyToTweetIdFeature, None)
@ -76,6 +75,7 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
.add(MentionUserIdFeature, Seq.empty)
.add(QuotedTweetIdFeature, None)
.add(QuotedUserIdFeature, None)
.add(SemanticAnnotationFeature, Seq.empty)
.add(SourceTweetIdFeature, None)
.add(SourceUserIdFeature, None)
.build()
@ -114,13 +114,12 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
val mentions = tweet.mentions.getOrElse(Seq.empty)
val share = coreData.flatMap(_.share)
val reply = coreData.flatMap(_.reply)
val semanticAnnotations =
tweet.escherbirdEntityAnnotations.map(_.entityAnnotations).getOrElse(Seq.empty)
FeatureMapBuilder()
.add(AuthorIdFeature, coreData.map(_.userId))
.add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId)))
.add(
ExclusiveConversationAuthorIdFeature,
tweet.exclusiveTweetControl.map(_.conversationAuthorId))
.add(HasImageFeature, TweetMediaFeaturesExtractor.hasImage(tweet))
.add(HasVideoFeature, TweetMediaFeaturesExtractor.hasVideo(tweet))
.add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId))
@ -130,6 +129,7 @@ class TweetypieStaticEntitiesFeatureHydrator @Inject() (
.add(MentionUserIdFeature, mentions.flatMap(_.userId))
.add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId))
.add(QuotedUserIdFeature, quotedTweet.map(_.userId))
.add(SemanticAnnotationFeature, semanticAnnotations)
.add(SourceTweetIdFeature, share.map(_.sourceStatusId))
.add(SourceUserIdFeature, share.map(_.sourceUserId))
.build()

View File

@ -1,9 +1,9 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureRepository
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter
import com.twitter.home_mixer.util.CandidatesUtil
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollow20200101FeatureRepository
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.{thriftscala => ml}
@ -17,61 +17,69 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.servo.repository.KeyValueResult
import com.twitter.stitch.Stitch
import com.twitter.util.Future
import com.twitter.util.Try
import javax.inject.Inject
import javax.inject.Named
import javax.inject.Singleton
import scala.collection.JavaConverters._
object TwhinAuthorFollowFeature
object TwhinAuthorFollow20220101Feature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TwhinAuthorFollowFeatureHydrator @Inject() (
@Named(TwhinAuthorFollowFeatureRepository)
client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor],
class TwhinAuthorFollow20220101FeatureHydrator @Inject() (
@Named(TwhinAuthorFollow20200101FeatureRepository)
client: KeyValueRepository[Seq[Long], Long, ml.Embedding],
override val statsReceiver: StatsReceiver)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with ObservedKeyValueResultHandler {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("TwhinAuthorFollow")
FeatureHydratorIdentifier("TwhinAuthorFollow20220101")
override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollowFeature)
override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollow20220101Feature)
override val statScope: String = identifier.toString
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture {
val possiblyAuthorIds = extractKeys(candidates)
val authorIds = possiblyAuthorIds.flatten
): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val possiblyAuthorIds = extractKeys(candidates)
val authorIds = possiblyAuthorIds.flatten
val response: Future[KeyValueResult[Long, ml.FloatTensor]] =
if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds)
val response: Future[KeyValueResult[Long, ml.Embedding]] =
if (authorIds.isEmpty) {
Future.value(KeyValueResult.empty)
} else {
client(authorIds)
}
response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value)
response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value)
FeatureMapBuilder().add(TwhinAuthorFollowFeature, transformedValue).build()
FeatureMapBuilder()
.add(TwhinAuthorFollow20220101Feature, transformedValue)
.build()
}
}
}
}
private def postTransformer(embedding: Try[Option[ml.FloatTensor]]): Try[DataRecord] = {
embedding.map { floatTensor =>
TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head
private def postTransformer(embedding: Try[Option[ml.Embedding]]): Try[DataRecord] = {
embedding.map { e =>
TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(e).asScala.head
}
}
@ -79,7 +87,10 @@ class TwhinAuthorFollowFeatureHydrator @Inject() (
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Seq[Option[Long]] = {
candidates.map { candidate =>
CandidatesUtil.getOriginalAuthorId(candidate.features)
candidate.features
.getTry(AuthorIdFeature)
.toOption
.flatten
}
}
}

View File

@ -1,15 +1,17 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserEngagementFeatureRepository
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter
import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions
import com.twitter.ml.api.{thriftscala => ml}
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
@ -20,7 +22,6 @@ import com.twitter.util.Throw
import javax.inject.Inject
import javax.inject.Named
import javax.inject.Singleton
import scala.collection.JavaConverters._
object TwhinUserEngagementFeature
extends DataRecordInAFeature[PipelineQuery]
@ -47,26 +48,33 @@ class TwhinUserEngagementQueryFeatureHydrator @Inject() (
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
Stitch.callFuture(client(Seq(userId))).map { results =>
val embedding: Option[ml.FloatTensor] = results(userId) match {
case Return(value) =>
if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr()
else keyLossCounter.incr()
value
case Throw(_) =>
keyFailureCounter.incr()
None
case _ =>
None
Stitch.callFuture {
client(Seq(userId)).map { results =>
val embedding: Option[ml.FloatTensor] = results(userId) match {
case Return(value) =>
if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr()
else keyLossCounter.incr()
value
case Throw(_) =>
keyFailureCounter.incr()
None
case _ =>
None
}
val dataRecord =
new RichDataRecord(new DataRecord, TwhinUserEngagementEmbeddingsAdapter.getFeatureContext)
embedding.foreach { floatTensor =>
dataRecord.setFeatureValue(
TwhinUserEngagementEmbeddingsAdapter.twhinEmbeddingsFeature,
ScalaToJavaDataRecordConversions.scalaTensor2Java(
ml.GeneralTensor.FloatTensor(floatTensor))
)
}
FeatureMapBuilder()
.add(TwhinUserEngagementFeature, dataRecord.getRecord)
.build()
}
val dataRecord =
TwhinUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head
FeatureMapBuilder()
.add(TwhinUserEngagementFeature, dataRecord)
.build()
}
}
}

View File

@ -1,15 +1,18 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserFollowFeatureRepository
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter
import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions
import com.twitter.ml.api.{thriftscala => ml}
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
@ -20,7 +23,6 @@ import com.twitter.util.Throw
import javax.inject.Inject
import javax.inject.Named
import javax.inject.Singleton
import scala.collection.JavaConverters._
object TwhinUserFollowFeature
extends DataRecordInAFeature[PipelineQuery]
@ -47,24 +49,32 @@ class TwhinUserFollowQueryFeatureHydrator @Inject() (
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
Stitch.callFuture(client(Seq(userId))).map { results =>
val embedding: Option[ml.FloatTensor] = results(userId) match {
case Return(value) =>
if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr()
else keyLossCounter.incr()
value
case Throw(_) =>
keyFailureCounter.incr()
None
case _ =>
None
Stitch.callFuture(
client(Seq(userId)).map { results =>
val embedding: Option[ml.FloatTensor] = results(userId) match {
case Return(value) =>
if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr()
else keyLossCounter.incr()
value
case Throw(_) =>
keyFailureCounter.incr()
None
case _ =>
None
}
val dataRecord =
new RichDataRecord(new DataRecord, TwhinUserFollowEmbeddingsAdapter.getFeatureContext)
embedding.foreach { floatTensor =>
dataRecord.setFeatureValue(
TwhinUserFollowEmbeddingsAdapter.twhinEmbeddingsFeature,
ScalaToJavaDataRecordConversions.scalaTensor2Java(
ml.GeneralTensor
.FloatTensor(floatTensor)))
}
FeatureMapBuilder()
.add(TwhinUserFollowFeature, dataRecord.getRecord)
.build()
}
val dataRecord = TwhinUserFollowEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head
FeatureMapBuilder()
.add(TwhinUserFollowFeature, dataRecord)
.build()
}
)
}
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
@ -12,12 +12,12 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.servo.keyvalue.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch
import com.twitter.util.Future
import com.twitter.util.Try
import javax.inject.Inject
import javax.inject.Named
import javax.inject.Singleton
@ -42,19 +42,27 @@ class UserFollowedTopicIdsFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture {
val possiblyAuthorIds = extractKeys(candidates)
val authorIds = possiblyAuthorIds.flatten
): Stitch[Seq[FeatureMap]] = {
Stitch.callFuture {
val possiblyAuthorIds = extractKeys(candidates)
val authorIds = possiblyAuthorIds.flatten
val response: Future[KeyValueResult[Long, Seq[Long]]] =
if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds)
val response: Future[KeyValueResult[Long, Seq[Long]]] =
if (authorIds.isEmpty) {
Future.value(KeyValueResult.empty)
} else {
client(authorIds)
}
response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value)
response.map { result =>
possiblyAuthorIds.map { possiblyAuthorId =>
val value = observedGet(key = possiblyAuthorId, keyValueResult = result)
val transformedValue = postTransformer(value)
FeatureMapBuilder().add(UserFollowedTopicIdsFeature, transformedValue).build()
FeatureMapBuilder()
.add(UserFollowedTopicIdsFeature, transformedValue)
.build()
}
}
}
}

View File

@ -1,8 +1,6 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesRepository
import com.twitter.home_mixer.util.ObservedKeyValueResultHandler
import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesStore
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
@ -10,8 +8,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.search.common.constants.{thriftscala => scc}
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch
import com.twitter.storehaus.ReadableStore
import javax.inject.Inject
import javax.inject.Named
import javax.inject.Singleton
@ -20,26 +18,17 @@ object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguag
@Singleton
case class UserLanguagesFeatureHydrator @Inject() (
@Named(UserLanguagesRepository) client: KeyValueRepository[Seq[Long], Long, Seq[
scc.ThriftLanguage
]],
statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery]
with ObservedKeyValueResultHandler {
@Named(UserLanguagesStore) store: ReadableStore[Long, Seq[scc.ThriftLanguage]])
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLanguages")
override val features: Set[Feature[_, _]] = Set(UserLanguagesFeature)
override val statScope: String = identifier.toString
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val key = query.getRequiredUserId
Stitch.callFuture(client(Seq(key))).map { result =>
val feature =
observedGet(key = Some(key), keyValueResult = result).map(_.getOrElse(Seq.empty))
Stitch.callFuture(store.get(query.getRequiredUserId)).map { languages =>
FeatureMapBuilder()
.add(UserLanguagesFeature, feature)
.add(UserLanguagesFeature, languages.getOrElse(Seq.empty))
.build()
}
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
@ -9,12 +9,13 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.user_health.v1.{thriftscala => uhv1}
import com.twitter.timelines.user_health.{thriftscala => uh}
import com.twitter.timelines.user_health.v1.{thriftscala => uhv1}
import com.twitter.user_session_store.ReadOnlyUserSessionStore
import com.twitter.user_session_store.ReadRequest
import com.twitter.user_session_store.UserSessionDataset
import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,13 +1,9 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.FavoritedByCountFeature
import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature
import com.twitter.home_mixer.model.HomeFeatures.RepliedByCountFeature
import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.RetweetedByCountFeature
import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository
@ -20,12 +16,12 @@ import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.recos.recos_common.{thriftscala => rc}
import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg}
import com.twitter.servo.keyvalue.KeyValueResult
import com.twitter.servo.repository.KeyValueRepository
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Named
import javax.inject.Singleton
@ -44,10 +40,7 @@ class UtegFeatureHydrator @Inject() (
override val features: Set[Feature[_, _]] = Set(
FavoritedByUserIdsFeature,
RetweetedByEngagerIdsFeature,
RepliedByEngagerIdsFeature,
FavoritedByCountFeature,
RetweetedByCountFeature,
RepliedByCountFeature
RepliedByEngagerIdsFeature
)
override def onlyIf(query: PipelineQuery): Boolean = query.features
@ -56,7 +49,7 @@ class UtegFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture {
): Stitch[Seq[FeatureMap]] = {
val seedUserWeights = query.features.map(_.get(RealGraphInNetworkScoresFeature)).get
val sourceTweetIds = candidates.flatMap(_.features.getOrElse(SourceTweetIdFeature, None))
@ -66,7 +59,9 @@ class UtegFeatureHydrator @Inject() (
val utegQuery = (tweetIdsToSend, (query.getRequiredUserId, seedUserWeights))
client(utegQuery).map(handleResponse(candidates, _))
Stitch
.callFuture(client(utegQuery))
.map(handleResponse(candidates, _))
}
private def handleResponse(
@ -74,7 +69,6 @@ class UtegFeatureHydrator @Inject() (
results: KeyValueResult[Long, uteg.TweetRecommendation],
): Seq[FeatureMap] = {
candidates.map { candidate =>
val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false)
val candidateProof = results(candidate.candidate.id).toOption.flatten
val sourceProof = candidate.features
.getOrElse(SourceTweetIdFeature, None).flatMap(results(_).toOption.flatten)
@ -84,18 +78,10 @@ class UtegFeatureHydrator @Inject() (
val retweetedBy = proofs.flatMap(_.get(rc.SocialProofType.Retweet)).flatten
val repliedBy = proofs.flatMap(_.get(rc.SocialProofType.Reply)).flatten
val (favoritedByCount, retweetedByCount, repliedByCount) =
if (!inNetwork) {
(favoritedBy.size.toDouble, retweetedBy.size.toDouble, repliedBy.size.toDouble)
} else { (0.0, 0.0, 0.0) }
FeatureMapBuilder()
.add(FavoritedByUserIdsFeature, favoritedBy)
.add(RetweetedByEngagerIdsFeature, retweetedBy)
.add(RepliedByEngagerIdsFeature, repliedBy)
.add(FavoritedByCountFeature, favoritedByCount)
.add(RetweetedByCountFeature, retweetedByCount)
.add(RepliedByCountFeature, repliedByCount)
.build()
}
}

View File

@ -0,0 +1,59 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
import com.twitter.ml.api.DataRecordMerger
import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.CompactDataRecordConverter
import com.twitter.ml.api.util.FDsl._
import com.twitter.timelines.author_features.v1.{thriftjava => af}
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
private val originalAuthorAggregatesFeatures =
TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
.buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
private val authorFeatures = originalAuthorAggregatesFeatures ++
Seq(
UserHealthFeatures.AuthorState,
UserHealthFeatures.NumAuthorFollowers,
UserHealthFeatures.NumAuthorConnectDays,
UserHealthFeatures.NumAuthorConnect)
private val featureContext = new FeatureContext(authorFeatures: _*)
override def getFeatureContext: FeatureContext = featureContext
override val commonFeatures: Set[Feature[_]] = Set.empty
private val compactDataRecordConverter = new CompactDataRecordConverter()
private val drMerger = new DataRecordMerger()
override def setFeatures(
authorFeaturesOpt: Option[af.AuthorFeatures],
richDataRecord: RichDataRecord
): Unit = {
authorFeaturesOpt.foreach { authorFeatures =>
val dataRecord = richDataRecord.getRecord
dataRecord.setFeatureValue(
UserHealthFeatures.AuthorState,
authorFeatures.user_health.user_state.getValue.toLong)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorFollowers,
authorFeatures.user_health.num_followers.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnectDays,
authorFeatures.user_health.num_connect_days.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnect,
authorFeatures.user_health.num_connect.toDouble)
val originalAuthorAggregatesDataRecord =
compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
}
}
}

View File

@ -4,7 +4,6 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"src/java/com/twitter/ml/api:api-base",
"src/java/com/twitter/ml/api/util",
"src/scala/com/twitter/ml/api/util",

View File

@ -11,7 +11,6 @@ scala_library(
"src/scala/com/twitter/timelines/prediction/common/adapters",
"src/scala/com/twitter/timelines/prediction/common/adapters:base",
"src/scala/com/twitter/timelines/prediction/features/common",
"src/scala/com/twitter/timelines/prediction/features/conversation_features",
"src/thrift/com/twitter/ml/api:data-java",
"src/thrift/com/twitter/ml/api:data-scala",
],

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content
import com.twitter.home_mixer.model.ContentFeatures
import com.twitter.ml.api.Feature
@ -8,14 +8,11 @@ import com.twitter.ml.api.util.DataRecordConverters._
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
import com.twitter.timelines.prediction.common.adapters.TweetLengthType
import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures
import com.twitter.timelines.prediction.features.conversation_features.ConversationFeatures
import scala.collection.JavaConverters._
object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] {
override val getFeatureContext: FeatureContext = new FeatureContext(
ConversationFeatures.IS_SELF_THREAD_TWEET,
ConversationFeatures.IS_LEAF_IN_SELF_THREAD,
TimelinesSharedFeatures.ASPECT_RATIO_DEN,
TimelinesSharedFeatures.ASPECT_RATIO_NUM,
TimelinesSharedFeatures.BIT_RATE,
@ -82,16 +79,6 @@ object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[Content
): Unit = {
if (contentFeatures.nonEmpty) {
val features = contentFeatures.get
// Conversation Features
richDataRecord.setFeatureValueFromOption(
ConversationFeatures.IS_SELF_THREAD_TWEET,
Some(features.selfThreadMetadata.nonEmpty)
)
richDataRecord.setFeatureValueFromOption(
ConversationFeatures.IS_LEAF_IN_SELF_THREAD,
features.selfThreadMetadata.map(_.isLeaf)
)
// Media Features
richDataRecord.setFeatureValueFromOption(
TimelinesSharedFeatures.ASPECT_RATIO_DEN,

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.earlybird
import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext
@ -305,10 +305,6 @@ object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTwe
richDataRecord.setFeatureValueFromOption(
RecapFeatures.REPLY_COUNT_V2,
features.replyCountV2.map(_.toDouble))
richDataRecord.setFeatureValueFromOption(
RecapFeatures.MENTIONED_SCREEN_NAMES,
features.mentionsList.map(_.toSet.asJava)
)
val urls = features.urlsList.getOrElse(Seq.empty)
richDataRecord.setFeatureValue(
RecapFeatures.URL_DOMAINS,

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic
import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext

Some files were not shown because too many files have changed in this diff Show More