This commit is contained in:
Matt Popovich 2023-07-17 21:41:57 -05:00 committed by GitHub
commit 933daa5d52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 24 additions and 24 deletions

View File

@ -18,7 +18,7 @@ public class DistancedItemQueue<U, T> implements Iterable<DistancedItem<T>> {
private final PriorityQueue<DistancedItem<T>> queue; private final PriorityQueue<DistancedItem<T>> queue;
private final boolean minQueue; private final boolean minQueue;
/** /**
* Creates ontainer for items with their distances. * Creates Container for items with their distances.
* *
* @param origin Origin (reference) point * @param origin Origin (reference) point
* @param initial Initial list of elements to add in the structure * @param initial Initial list of elements to add in the structure

View File

@ -87,7 +87,7 @@ trait RecommendationFlow[Target, Candidate <: UniversalNoun[Long]]
protected def selectRanker(target: Target): Ranker[Target, Candidate] protected def selectRanker(target: Target): Ranker[Target, Candidate]
/** /**
* transform the candidates after ranking (e.g. dedupping, grouping and etc) * transform the candidates after ranking (e.g. deduping, grouping and etc)
*/ */
protected def postRankerTransform: Transform[Target, Candidate] protected def postRankerTransform: Transform[Target, Candidate]

View File

@ -105,7 +105,7 @@ class RepeatedProfileVisitsSource @Inject() (
val recommendationThreshold = params.getInt(RepeatedProfileVisitsParams.RecommendationThreshold) val recommendationThreshold = params.getInt(RepeatedProfileVisitsParams.RecommendationThreshold)
val bucketingThreshold = params.getInt(RepeatedProfileVisitsParams.BucketingThreshold) val bucketingThreshold = params.getInt(RepeatedProfileVisitsParams.BucketingThreshold)
// Get the list of repeatedly visited profilts. Only keep accounts with >= bucketingThreshold visits. // Get the list of repeatedly visited profiles. Only keep accounts with >= bucketingThreshold visits.
val repeatedVisitedAccountsStitch: Stitch[Map[Long, Int]] = val repeatedVisitedAccountsStitch: Stitch[Map[Long, Int]] =
getRepeatedVisitedAccounts(params, userId).map(_.filter(kv => kv._2 >= bucketingThreshold)) getRepeatedVisitedAccounts(params, userId).map(_.filter(kv => kv._2 >= bucketingThreshold))

View File

@ -86,14 +86,14 @@ class TopOrganicFollowsAccountsSource @Inject() (
debug("candidate source failed identifier = %s".format(identifier), t) debug("candidate source failed identifier = %s".format(identifier), t)
errorStats.incr() errorStats.incr()
}) })
.map(transformOrganicFollowAccountssToCandidateSource) .map(transformOrganicFollowAccountsToCandidateSource)
}.getOrElse { }.getOrElse {
noCountryCodeStats.incr() noCountryCodeStats.incr()
Stitch.value(Seq[CandidateUser]()) Stitch.value(Seq[CandidateUser]())
} }
} }
private def transformOrganicFollowAccountssToCandidateSource( private def transformOrganicFollowAccountsToCandidateSource(
organicFollowsAccounts: Seq[Option[OrganicFollowsAccounts]] organicFollowsAccounts: Seq[Option[OrganicFollowsAccounts]]
): Seq[CandidateUser] = { ): Seq[CandidateUser] = {
organicFollowsAccounts organicFollowsAccounts

View File

@ -56,7 +56,7 @@ object StratoClientModule extends TwitterModule {
val WTFPostNuxFeaturesPath = "ml/featureStore/onboarding/wtfPostNuxFeatures.User" val WTFPostNuxFeaturesPath = "ml/featureStore/onboarding/wtfPostNuxFeatures.User"
val ElectionCandidatesPath = "onboarding/electionAccounts" val ElectionCandidatesPath = "onboarding/electionAccounts"
val UserUserGraphPath = "recommendations/userUserGraph" val UserUserGraphPath = "recommendations/userUserGraph"
val WtfDissmissEventsPath = "onboarding/wtfDismissEvents" val WtfDismissEventsPath = "onboarding/wtfDismissEvents"
val RelatableAccountsPath = "onboarding/userrecs/relatableAccounts" val RelatableAccountsPath = "onboarding/userrecs/relatableAccounts"
val ExtendedNetworkCandidatesPath = "search/account_search/extendedNetworkCandidatesMH" val ExtendedNetworkCandidatesPath = "search/account_search/extendedNetworkCandidatesMH"
val LabeledNotificationPath = "frigate/magicrecs/labeledPushRecsAggregated.User" val LabeledNotificationPath = "frigate/magicrecs/labeledPushRecsAggregated.User"
@ -234,7 +234,7 @@ object StratoClientModule extends TwitterModule {
Unit, Unit,
(Long, (Long, Long)), (Long, (Long, Long)),
WhoToFollowDismissEventDetails WhoToFollowDismissEventDetails
](WtfDissmissEventsPath) ](WtfDismissEventsPath)
@Provides @Provides
@Singleton @Singleton

View File

@ -19,7 +19,7 @@ object GuiceNamedConstants {
final val TWO_HOP_RANDOM_WALK_FETCHER = "two_hop_random_walk_fetcher" final val TWO_HOP_RANDOM_WALK_FETCHER = "two_hop_random_walk_fetcher"
final val USER_RECOMMENDABILITY_FETCHER = "user_recommendability_fetcher" final val USER_RECOMMENDABILITY_FETCHER = "user_recommendability_fetcher"
final val USER_STATE_FETCHER = "user_state_fetcher" final val USER_STATE_FETCHER = "user_state_fetcher"
final val UTT_ACCOUNT_RECOMMENDATIONS_FETCHER = "utt_account_recomendations_fetcher" final val UTT_ACCOUNT_RECOMMENDATIONS_FETCHER = "utt_account_recommendations_fetcher"
final val UTT_SEED_ACCOUNTS_FETCHER = "utt_seed_accounts_fetcher" final val UTT_SEED_ACCOUNTS_FETCHER = "utt_seed_accounts_fetcher"
final val ELECTION_CANDIDATES_FETCHER = "election_candidates_fetcher" final val ELECTION_CANDIDATES_FETCHER = "election_candidates_fetcher"

View File

@ -32,8 +32,8 @@ scala_library(
"src/scala/com/twitter/ml/featurestore/catalog/features/core:socialgraph", "src/scala/com/twitter/ml/featurestore/catalog/features/core:socialgraph",
"src/scala/com/twitter/ml/featurestore/catalog/features/core:user", "src/scala/com/twitter/ml/featurestore/catalog/features/core:user",
"src/scala/com/twitter/ml/featurestore/catalog/features/interests_discovery:user-topic-relationships", "src/scala/com/twitter/ml/featurestore/catalog/features/interests_discovery:user-topic-relationships",
"src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmaries", "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmaries", # TODO: too many m's
"src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmary-aggregates", "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmary-aggregates", # TODO: too many m's
"src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:nonmr-ntab-summaries", "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:nonmr-ntab-summaries",
"src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:mc-user-counting", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:mc-user-counting",
"src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline",

View File

@ -54,7 +54,7 @@ class WeightedCandidateSourceRanker[Target <: HasParams](
// Note 1: Using map instead mapValue here since mapValue somehow caused infinite loop when used as part of Stream. // Note 1: Using map instead mapValue here since mapValue somehow caused infinite loop when used as part of Stream.
val sortAndShuffledCandidates = input.map { val sortAndShuffledCandidates = input.map {
case (source, candidates) => case (source, candidates) =>
// Note 2: toList is required here since candidates is a view, and it will result in infinit loop when used as part of Stream. // Note 2: toList is required here since candidates is a view, and it will result in infinite loop when used as part of Stream.
// Note 3: there is no real sorting logic here, it assumes the input is already sorted by candidate sources // Note 3: there is no real sorting logic here, it assumes the input is already sorted by candidate sources
val sortedCandidates = candidates.toList val sortedCandidates = candidates.toList
source -> shuffleFn(sortedCandidates).iterator source -> shuffleFn(sortedCandidates).iterator

View File

@ -5,7 +5,7 @@ import com.twitter.timelines.configapi.FSParam
object SamplingTransformParams { object SamplingTransformParams {
case object TopKFixed // indicates how many of the fisrt K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score case object TopKFixed // indicates how many of the first K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score
extends FSBoundedParam[Int]( extends FSBoundedParam[Int](
name = "post_nux_ml_flow_weighted_sampling_top_k_fixed", name = "post_nux_ml_flow_weighted_sampling_top_k_fixed",
default = 0, default = 0,

View File

@ -69,7 +69,7 @@ class PromotedAccountsFlow @Inject() (
} }
/** /**
* transform the candidates after ranking (e.g. dedupping, grouping and etc) * transform the candidates after ranking (e.g. deduping, grouping and etc)
*/ */
protected override def postRankerTransform: Transform[ protected override def postRankerTransform: Transform[
PromotedAccountsFlowRequest, PromotedAccountsFlowRequest,

View File

@ -133,7 +133,7 @@ class ContentRecommenderFlow @Inject() (
CandidateUser CandidateUser
] = { ] = {
new DedupTransform[ContentRecommenderRequest, CandidateUser] new DedupTransform[ContentRecommenderRequest, CandidateUser]
.observe(statsReceiver.scope("dedupping")) .observe(statsReceiver.scope("deduping"))
} }
protected override def validateCandidates: Predicate[ protected override def validateCandidates: Predicate[

View File

@ -29,7 +29,7 @@ import com.twitter.timelines.configapi.HasParams
* - truncating to the top N merged results for ranking * - truncating to the top N merged results for ranking
* - ML ranker * - ML ranker
* - Interleaving ranker for producer-side experiments * - Interleaving ranker for producer-side experiments
* - impression-based fatigueing * - impression-based fatiguing
*/ */
@Singleton @Singleton
class PostNuxMlCombinedRankerBuilder[ class PostNuxMlCombinedRankerBuilder[

View File

@ -175,7 +175,7 @@ class PostNuxMlFlow @Inject() (
override protected val postRankerTransform: Transform[PostNuxMlRequest, CandidateUser] = { override protected val postRankerTransform: Transform[PostNuxMlRequest, CandidateUser] = {
new DedupTransform[PostNuxMlRequest, CandidateUser] new DedupTransform[PostNuxMlRequest, CandidateUser]
.observe(statsReceiver.scope("dedupping")) .observe(statsReceiver.scope("deduping"))
.andThen( .andThen(
samplingTransform samplingTransform
.gated(PostNuxMlParams.SamplingTransformEnabled) .gated(PostNuxMlParams.SamplingTransformEnabled)

View File

@ -23,5 +23,5 @@ object PostNuxMlFlowFeatureSwitchKeys {
val EnableSGSPredicate = "post_nux_ml_flow_enable_sgs_predicate" val EnableSGSPredicate = "post_nux_ml_flow_enable_sgs_predicate"
val EnableHssPredicate = "post_nux_ml_flow_enable_hss_predicate" val EnableHssPredicate = "post_nux_ml_flow_enable_hss_predicate"
val GetFollowersFromSgs = "post_nux_ml_flow_get_followers_from_sgs" val GetFollowersFromSgs = "post_nux_ml_flow_get_followers_from_sgs"
val TurnOffMLScorerQFThreshold = "post_nux_ml_flow_turn_off_ml_scorer_threhsold" val TurnOffMLScorerQFThreshold = "post_nux_ml_flow_turn_off_ml_scorer_threshold"
} }

View File

@ -16,7 +16,7 @@ import com.twitter.logging.LoggerFactory
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
import com.twitter.scribelib.marshallers.ClientDataProvider import com.twitter.scribelib.marshallers.ClientDataProvider
import com.twitter.scribelib.marshallers.ExternalRefererDataProvider import com.twitter.scribelib.marshallers.ExternalReferrerDataProvider
import com.twitter.scribelib.marshallers.ScribeSerialization import com.twitter.scribelib.marshallers.ScribeSerialization
import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.HasParams
import com.twitter.util.Time import com.twitter.util.Time
@ -125,7 +125,7 @@ object FrsLogger {
/** The id of the current user. When the user is logged out, this method should return None. */ /** The id of the current user. When the user is logged out, this method should return None. */
override val userId: Option[Long] = clientContext.userId override val userId: Option[Long] = clientContext.userId
/** The id of the guest, which is present in logged-in or loged-out states */ /** The id of the guest, which is present in logged-in or logged-out states */
override val guestId: Option[Long] = clientContext.guestId override val guestId: Option[Long] = clientContext.guestId
/** The personalization id (pid) of the user, used to personalize Twitter services */ /** The personalization id (pid) of the user, used to personalize Twitter services */
@ -153,12 +153,12 @@ object FrsLogger {
override val isSsl: Option[Boolean] = Some(true) override val isSsl: Option[Boolean] = Some(true)
/** The referring URL to the current page for web-based clients, if applicable */ /** The referring URL to the current page for web-based clients, if applicable */
override val referer: Option[String] = None override val referrer: Option[String] = None
/** /**
* The external site, partner, or email that lead to the current Twitter application. Returned value consists of a * The external site, partner, or email that lead to the current Twitter application. Returned value consists of a
* tuple including the encrypted referral data and the type of referral * tuple including the encrypted referral data and the type of referral
*/ */
override val externalReferer: Option[ExternalRefererDataProvider] = None override val externalReferrer: Option[ExternalReferrerDataProvider] = None
} }
} }

View File

@ -8,7 +8,7 @@ object FlagsModule extends TwitterModule {
) )
flag[Boolean]( flag[Boolean](
name = "interests_opt_out_prod_enabled", name = "interests_opt_out_prod_enabled",
help = "Whether to fetch intersts opt out data from the prod strato column or not" help = "Whether to fetch interests opt out data from the prod strato column or not"
) )
flag[Boolean]( flag[Boolean](
name = "log_results", name = "log_results",

View File

@ -98,8 +98,8 @@ trait ScribeClientEventSideEffect[
override val languageCode = query.clientContext.languageCode override val languageCode = query.clientContext.languageCode
override val userAgent = query.clientContext.userAgent override val userAgent = query.clientContext.userAgent
override val isSsl = None override val isSsl = None
override val referer = None override val referrer = None
override val externalReferer = None override val externalReferrer = None
} }
} }
} }