mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-12-22 18:21:51 +01:00
Merge b1bc8b2541
into 72eda9a24f
This commit is contained in:
commit
933daa5d52
@ -18,7 +18,7 @@ public class DistancedItemQueue<U, T> implements Iterable<DistancedItem<T>> {
|
|||||||
private final PriorityQueue<DistancedItem<T>> queue;
|
private final PriorityQueue<DistancedItem<T>> queue;
|
||||||
private final boolean minQueue;
|
private final boolean minQueue;
|
||||||
/**
|
/**
|
||||||
* Creates ontainer for items with their distances.
|
* Creates Container for items with their distances.
|
||||||
*
|
*
|
||||||
* @param origin Origin (reference) point
|
* @param origin Origin (reference) point
|
||||||
* @param initial Initial list of elements to add in the structure
|
* @param initial Initial list of elements to add in the structure
|
||||||
|
@ -87,7 +87,7 @@ trait RecommendationFlow[Target, Candidate <: UniversalNoun[Long]]
|
|||||||
protected def selectRanker(target: Target): Ranker[Target, Candidate]
|
protected def selectRanker(target: Target): Ranker[Target, Candidate]
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* transform the candidates after ranking (e.g. dedupping, grouping and etc)
|
* transform the candidates after ranking (e.g. deduping, grouping and etc)
|
||||||
*/
|
*/
|
||||||
protected def postRankerTransform: Transform[Target, Candidate]
|
protected def postRankerTransform: Transform[Target, Candidate]
|
||||||
|
|
||||||
|
@ -105,7 +105,7 @@ class RepeatedProfileVisitsSource @Inject() (
|
|||||||
val recommendationThreshold = params.getInt(RepeatedProfileVisitsParams.RecommendationThreshold)
|
val recommendationThreshold = params.getInt(RepeatedProfileVisitsParams.RecommendationThreshold)
|
||||||
val bucketingThreshold = params.getInt(RepeatedProfileVisitsParams.BucketingThreshold)
|
val bucketingThreshold = params.getInt(RepeatedProfileVisitsParams.BucketingThreshold)
|
||||||
|
|
||||||
// Get the list of repeatedly visited profilts. Only keep accounts with >= bucketingThreshold visits.
|
// Get the list of repeatedly visited profiles. Only keep accounts with >= bucketingThreshold visits.
|
||||||
val repeatedVisitedAccountsStitch: Stitch[Map[Long, Int]] =
|
val repeatedVisitedAccountsStitch: Stitch[Map[Long, Int]] =
|
||||||
getRepeatedVisitedAccounts(params, userId).map(_.filter(kv => kv._2 >= bucketingThreshold))
|
getRepeatedVisitedAccounts(params, userId).map(_.filter(kv => kv._2 >= bucketingThreshold))
|
||||||
|
|
||||||
|
@ -86,14 +86,14 @@ class TopOrganicFollowsAccountsSource @Inject() (
|
|||||||
debug("candidate source failed identifier = %s".format(identifier), t)
|
debug("candidate source failed identifier = %s".format(identifier), t)
|
||||||
errorStats.incr()
|
errorStats.incr()
|
||||||
})
|
})
|
||||||
.map(transformOrganicFollowAccountssToCandidateSource)
|
.map(transformOrganicFollowAccountsToCandidateSource)
|
||||||
}.getOrElse {
|
}.getOrElse {
|
||||||
noCountryCodeStats.incr()
|
noCountryCodeStats.incr()
|
||||||
Stitch.value(Seq[CandidateUser]())
|
Stitch.value(Seq[CandidateUser]())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def transformOrganicFollowAccountssToCandidateSource(
|
private def transformOrganicFollowAccountsToCandidateSource(
|
||||||
organicFollowsAccounts: Seq[Option[OrganicFollowsAccounts]]
|
organicFollowsAccounts: Seq[Option[OrganicFollowsAccounts]]
|
||||||
): Seq[CandidateUser] = {
|
): Seq[CandidateUser] = {
|
||||||
organicFollowsAccounts
|
organicFollowsAccounts
|
||||||
|
@ -56,7 +56,7 @@ object StratoClientModule extends TwitterModule {
|
|||||||
val WTFPostNuxFeaturesPath = "ml/featureStore/onboarding/wtfPostNuxFeatures.User"
|
val WTFPostNuxFeaturesPath = "ml/featureStore/onboarding/wtfPostNuxFeatures.User"
|
||||||
val ElectionCandidatesPath = "onboarding/electionAccounts"
|
val ElectionCandidatesPath = "onboarding/electionAccounts"
|
||||||
val UserUserGraphPath = "recommendations/userUserGraph"
|
val UserUserGraphPath = "recommendations/userUserGraph"
|
||||||
val WtfDissmissEventsPath = "onboarding/wtfDismissEvents"
|
val WtfDismissEventsPath = "onboarding/wtfDismissEvents"
|
||||||
val RelatableAccountsPath = "onboarding/userrecs/relatableAccounts"
|
val RelatableAccountsPath = "onboarding/userrecs/relatableAccounts"
|
||||||
val ExtendedNetworkCandidatesPath = "search/account_search/extendedNetworkCandidatesMH"
|
val ExtendedNetworkCandidatesPath = "search/account_search/extendedNetworkCandidatesMH"
|
||||||
val LabeledNotificationPath = "frigate/magicrecs/labeledPushRecsAggregated.User"
|
val LabeledNotificationPath = "frigate/magicrecs/labeledPushRecsAggregated.User"
|
||||||
@ -234,7 +234,7 @@ object StratoClientModule extends TwitterModule {
|
|||||||
Unit,
|
Unit,
|
||||||
(Long, (Long, Long)),
|
(Long, (Long, Long)),
|
||||||
WhoToFollowDismissEventDetails
|
WhoToFollowDismissEventDetails
|
||||||
](WtfDissmissEventsPath)
|
](WtfDismissEventsPath)
|
||||||
|
|
||||||
@Provides
|
@Provides
|
||||||
@Singleton
|
@Singleton
|
||||||
|
@ -19,7 +19,7 @@ object GuiceNamedConstants {
|
|||||||
final val TWO_HOP_RANDOM_WALK_FETCHER = "two_hop_random_walk_fetcher"
|
final val TWO_HOP_RANDOM_WALK_FETCHER = "two_hop_random_walk_fetcher"
|
||||||
final val USER_RECOMMENDABILITY_FETCHER = "user_recommendability_fetcher"
|
final val USER_RECOMMENDABILITY_FETCHER = "user_recommendability_fetcher"
|
||||||
final val USER_STATE_FETCHER = "user_state_fetcher"
|
final val USER_STATE_FETCHER = "user_state_fetcher"
|
||||||
final val UTT_ACCOUNT_RECOMMENDATIONS_FETCHER = "utt_account_recomendations_fetcher"
|
final val UTT_ACCOUNT_RECOMMENDATIONS_FETCHER = "utt_account_recommendations_fetcher"
|
||||||
final val UTT_SEED_ACCOUNTS_FETCHER = "utt_seed_accounts_fetcher"
|
final val UTT_SEED_ACCOUNTS_FETCHER = "utt_seed_accounts_fetcher"
|
||||||
|
|
||||||
final val ELECTION_CANDIDATES_FETCHER = "election_candidates_fetcher"
|
final val ELECTION_CANDIDATES_FETCHER = "election_candidates_fetcher"
|
||||||
|
@ -32,8 +32,8 @@ scala_library(
|
|||||||
"src/scala/com/twitter/ml/featurestore/catalog/features/core:socialgraph",
|
"src/scala/com/twitter/ml/featurestore/catalog/features/core:socialgraph",
|
||||||
"src/scala/com/twitter/ml/featurestore/catalog/features/core:user",
|
"src/scala/com/twitter/ml/featurestore/catalog/features/core:user",
|
||||||
"src/scala/com/twitter/ml/featurestore/catalog/features/interests_discovery:user-topic-relationships",
|
"src/scala/com/twitter/ml/featurestore/catalog/features/interests_discovery:user-topic-relationships",
|
||||||
"src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmaries",
|
"src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmaries", # TODO: too many m's
|
||||||
"src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmary-aggregates",
|
"src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmary-aggregates", # TODO: too many m's
|
||||||
"src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:nonmr-ntab-summaries",
|
"src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:nonmr-ntab-summaries",
|
||||||
"src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:mc-user-counting",
|
"src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:mc-user-counting",
|
||||||
"src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline",
|
"src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline",
|
||||||
|
@ -54,7 +54,7 @@ class WeightedCandidateSourceRanker[Target <: HasParams](
|
|||||||
// Note 1: Using map instead mapValue here since mapValue somehow caused infinite loop when used as part of Stream.
|
// Note 1: Using map instead mapValue here since mapValue somehow caused infinite loop when used as part of Stream.
|
||||||
val sortAndShuffledCandidates = input.map {
|
val sortAndShuffledCandidates = input.map {
|
||||||
case (source, candidates) =>
|
case (source, candidates) =>
|
||||||
// Note 2: toList is required here since candidates is a view, and it will result in infinit loop when used as part of Stream.
|
// Note 2: toList is required here since candidates is a view, and it will result in infinite loop when used as part of Stream.
|
||||||
// Note 3: there is no real sorting logic here, it assumes the input is already sorted by candidate sources
|
// Note 3: there is no real sorting logic here, it assumes the input is already sorted by candidate sources
|
||||||
val sortedCandidates = candidates.toList
|
val sortedCandidates = candidates.toList
|
||||||
source -> shuffleFn(sortedCandidates).iterator
|
source -> shuffleFn(sortedCandidates).iterator
|
||||||
|
@ -5,7 +5,7 @@ import com.twitter.timelines.configapi.FSParam
|
|||||||
|
|
||||||
object SamplingTransformParams {
|
object SamplingTransformParams {
|
||||||
|
|
||||||
case object TopKFixed // indicates how many of the fisrt K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score
|
case object TopKFixed // indicates how many of the first K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score
|
||||||
extends FSBoundedParam[Int](
|
extends FSBoundedParam[Int](
|
||||||
name = "post_nux_ml_flow_weighted_sampling_top_k_fixed",
|
name = "post_nux_ml_flow_weighted_sampling_top_k_fixed",
|
||||||
default = 0,
|
default = 0,
|
||||||
|
@ -69,7 +69,7 @@ class PromotedAccountsFlow @Inject() (
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* transform the candidates after ranking (e.g. dedupping, grouping and etc)
|
* transform the candidates after ranking (e.g. deduping, grouping and etc)
|
||||||
*/
|
*/
|
||||||
protected override def postRankerTransform: Transform[
|
protected override def postRankerTransform: Transform[
|
||||||
PromotedAccountsFlowRequest,
|
PromotedAccountsFlowRequest,
|
||||||
|
@ -133,7 +133,7 @@ class ContentRecommenderFlow @Inject() (
|
|||||||
CandidateUser
|
CandidateUser
|
||||||
] = {
|
] = {
|
||||||
new DedupTransform[ContentRecommenderRequest, CandidateUser]
|
new DedupTransform[ContentRecommenderRequest, CandidateUser]
|
||||||
.observe(statsReceiver.scope("dedupping"))
|
.observe(statsReceiver.scope("deduping"))
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override def validateCandidates: Predicate[
|
protected override def validateCandidates: Predicate[
|
||||||
|
@ -29,7 +29,7 @@ import com.twitter.timelines.configapi.HasParams
|
|||||||
* - truncating to the top N merged results for ranking
|
* - truncating to the top N merged results for ranking
|
||||||
* - ML ranker
|
* - ML ranker
|
||||||
* - Interleaving ranker for producer-side experiments
|
* - Interleaving ranker for producer-side experiments
|
||||||
* - impression-based fatigueing
|
* - impression-based fatiguing
|
||||||
*/
|
*/
|
||||||
@Singleton
|
@Singleton
|
||||||
class PostNuxMlCombinedRankerBuilder[
|
class PostNuxMlCombinedRankerBuilder[
|
||||||
|
@ -175,7 +175,7 @@ class PostNuxMlFlow @Inject() (
|
|||||||
|
|
||||||
override protected val postRankerTransform: Transform[PostNuxMlRequest, CandidateUser] = {
|
override protected val postRankerTransform: Transform[PostNuxMlRequest, CandidateUser] = {
|
||||||
new DedupTransform[PostNuxMlRequest, CandidateUser]
|
new DedupTransform[PostNuxMlRequest, CandidateUser]
|
||||||
.observe(statsReceiver.scope("dedupping"))
|
.observe(statsReceiver.scope("deduping"))
|
||||||
.andThen(
|
.andThen(
|
||||||
samplingTransform
|
samplingTransform
|
||||||
.gated(PostNuxMlParams.SamplingTransformEnabled)
|
.gated(PostNuxMlParams.SamplingTransformEnabled)
|
||||||
|
@ -23,5 +23,5 @@ object PostNuxMlFlowFeatureSwitchKeys {
|
|||||||
val EnableSGSPredicate = "post_nux_ml_flow_enable_sgs_predicate"
|
val EnableSGSPredicate = "post_nux_ml_flow_enable_sgs_predicate"
|
||||||
val EnableHssPredicate = "post_nux_ml_flow_enable_hss_predicate"
|
val EnableHssPredicate = "post_nux_ml_flow_enable_hss_predicate"
|
||||||
val GetFollowersFromSgs = "post_nux_ml_flow_get_followers_from_sgs"
|
val GetFollowersFromSgs = "post_nux_ml_flow_get_followers_from_sgs"
|
||||||
val TurnOffMLScorerQFThreshold = "post_nux_ml_flow_turn_off_ml_scorer_threhsold"
|
val TurnOffMLScorerQFThreshold = "post_nux_ml_flow_turn_off_ml_scorer_threshold"
|
||||||
}
|
}
|
||||||
|
@ -16,7 +16,7 @@ import com.twitter.logging.LoggerFactory
|
|||||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||||
import com.twitter.scribelib.marshallers.ClientDataProvider
|
import com.twitter.scribelib.marshallers.ClientDataProvider
|
||||||
import com.twitter.scribelib.marshallers.ExternalRefererDataProvider
|
import com.twitter.scribelib.marshallers.ExternalReferrerDataProvider
|
||||||
import com.twitter.scribelib.marshallers.ScribeSerialization
|
import com.twitter.scribelib.marshallers.ScribeSerialization
|
||||||
import com.twitter.timelines.configapi.HasParams
|
import com.twitter.timelines.configapi.HasParams
|
||||||
import com.twitter.util.Time
|
import com.twitter.util.Time
|
||||||
@ -125,7 +125,7 @@ object FrsLogger {
|
|||||||
/** The id of the current user. When the user is logged out, this method should return None. */
|
/** The id of the current user. When the user is logged out, this method should return None. */
|
||||||
override val userId: Option[Long] = clientContext.userId
|
override val userId: Option[Long] = clientContext.userId
|
||||||
|
|
||||||
/** The id of the guest, which is present in logged-in or loged-out states */
|
/** The id of the guest, which is present in logged-in or logged-out states */
|
||||||
override val guestId: Option[Long] = clientContext.guestId
|
override val guestId: Option[Long] = clientContext.guestId
|
||||||
|
|
||||||
/** The personalization id (pid) of the user, used to personalize Twitter services */
|
/** The personalization id (pid) of the user, used to personalize Twitter services */
|
||||||
@ -153,12 +153,12 @@ object FrsLogger {
|
|||||||
override val isSsl: Option[Boolean] = Some(true)
|
override val isSsl: Option[Boolean] = Some(true)
|
||||||
|
|
||||||
/** The referring URL to the current page for web-based clients, if applicable */
|
/** The referring URL to the current page for web-based clients, if applicable */
|
||||||
override val referer: Option[String] = None
|
override val referrer: Option[String] = None
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The external site, partner, or email that lead to the current Twitter application. Returned value consists of a
|
* The external site, partner, or email that lead to the current Twitter application. Returned value consists of a
|
||||||
* tuple including the encrypted referral data and the type of referral
|
* tuple including the encrypted referral data and the type of referral
|
||||||
*/
|
*/
|
||||||
override val externalReferer: Option[ExternalRefererDataProvider] = None
|
override val externalReferrer: Option[ExternalReferrerDataProvider] = None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -8,7 +8,7 @@ object FlagsModule extends TwitterModule {
|
|||||||
)
|
)
|
||||||
flag[Boolean](
|
flag[Boolean](
|
||||||
name = "interests_opt_out_prod_enabled",
|
name = "interests_opt_out_prod_enabled",
|
||||||
help = "Whether to fetch intersts opt out data from the prod strato column or not"
|
help = "Whether to fetch interests opt out data from the prod strato column or not"
|
||||||
)
|
)
|
||||||
flag[Boolean](
|
flag[Boolean](
|
||||||
name = "log_results",
|
name = "log_results",
|
||||||
|
@ -98,8 +98,8 @@ trait ScribeClientEventSideEffect[
|
|||||||
override val languageCode = query.clientContext.languageCode
|
override val languageCode = query.clientContext.languageCode
|
||||||
override val userAgent = query.clientContext.userAgent
|
override val userAgent = query.clientContext.userAgent
|
||||||
override val isSsl = None
|
override val isSsl = None
|
||||||
override val referer = None
|
override val referrer = None
|
||||||
override val externalReferer = None
|
override val externalReferrer = None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user