Mirrors
/
the-algorithm
mirror of https://github.com/twitter/the-algorithm.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[docx] split commit for file 6800 

Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer 2024-01-23 19:22:44 +02:00
parent 2c9fa892a8
commit e90b21ec83
No known key found for this signature in database
GPG Key ID: A50D5B4B599AF8A2
400 changed files with 0 additions and 17751 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ZoneFilteringTest.docx Normal file
View File

Binary file not shown.

50
unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ZoneFilteringTest.scala
View File

@ -1,50 +0,0 @@
package com.twitter.unified_user_actions.service
import com.twitter.inject.Test
import com.twitter.kafka.client.headers.ATLA
import com.twitter.kafka.client.headers.Implicits._
import com.twitter.kafka.client.headers.PDXA
import com.twitter.kafka.client.headers.Zone
import com.twitter.unified_user_actions.service.module.ZoneFiltering
import com.twitter.util.mock.Mockito
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.junit.runner.RunWith
import org.scalatestplus.junit.JUnitRunner
import org.scalatest.prop.TableDrivenPropertyChecks
@RunWith(classOf[JUnitRunner])
class ZoneFilteringTest extends Test with Mockito with TableDrivenPropertyChecks {
trait Fixture {
val consumerRecord =
new ConsumerRecord[Array[Byte], Array[Byte]]("topic", 0, 0l, Array(0), Array(0))
}
test("two DCs filter") {
val zones = Table(
"zone",
Some(ATLA),
Some(PDXA),
None
)
forEvery(zones) { localZoneOpt: Option[Zone] =>
forEvery(zones) { headerZoneOpt: Option[Zone] =>
localZoneOpt.foreach { localZone =>
new Fixture {
headerZoneOpt match {
case Some(headerZone) =>
consumerRecord.headers().setZone(headerZone)
if (headerZone == ATLA && localZone == ATLA)
ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe true
else if (headerZone == PDXA && localZone == PDXA)
ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe true
else
ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe false
case _ =>
ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe true
}
}
}
}
}
}
}

20
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/BUILD
View File

@ -1,20 +0,0 @@
create_thrift_libraries(
org = "com.twitter",
base_name = "unified_user_actions",
sources = ["*.thrift"],
tags = ["bazel-compatible"],
dependency_roots = [
"src/thrift/com/twitter/clientapp/gen:clientapp",
"src/thrift/com/twitter/gizmoduck:thrift",
"src/thrift/com/twitter/gizmoduck:user-thrift",
"src/thrift/com/twitter/search/common:constants",
"src/thrift/com/twitter/socialgraph:thrift",
],
generate_languages = [
"java",
"scala",
"strato",
],
provides_java_name = "unified_user_actions-thrift-java",
provides_scala_name = "unified_user_actions-thrift-scala",
)

BIN
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/BUILD.docx Normal file
View File

Binary file not shown.

BIN
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.docx Normal file
View File

Binary file not shown.

957
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.thrift
View File

@ -1,957 +0,0 @@
namespace java com.twitter.unified_user_actions.thriftjava
#@namespace scala com.twitter.unified_user_actions.thriftscala
#@namespace strato com.twitter.unified_user_actions
include "com/twitter/clientapp/gen/client_app.thrift"
include "com/twitter/reportflow/report_flow_logs.thrift"
include "com/twitter/socialgraph/social_graph_service_write_log.thrift"
include "com/twitter/gizmoduck/user_service.thrift"
/*
* ActionType is typically a three part enum consisting of
* [Origin][Item Type][Action Name]
*
* [Origin] is usually "client" or "server" to indicate how the action was derived.
*
* [Item Type] is singular and refers to the shorthand version of the type of
* Item (e.g. Tweet, Profile, Notification instead of TweetInfo, ProfileInfo, NotificationInfo)
* the action occurred on. Action types and item types should be 1:1, and when an action can be
* performed on multiple types of items, consider granular action types.
*
* [Action Name] is the descriptive name of the user action (e.g. favorite, render impression);
* action names should correspond to UI actions / ML labels (which are typically based on user
* behavior from UI actions)
*
* Below are guidelines around naming of action types:
* a) When an action is coupled to a product surface, be concise in naming such that the
* combination of item type and action name captures the user behavior for the action in the UI. For example,
* for an open on a Notification in the PushNotification product surface that is parsed from client events,
* consider ClientNotificationOpen because the item Notification and the action name Open concisely represent
* the action, and the product surface PushNotification can be identified independently.
*
* b) It is OK to use generic names like Click if needed to distinguish from another action OR
* it is the best way to characterize an action concisely without confusion.
* For example, for ClientTweetClickReply, this refers to actually clicking on the Reply button but not
* Replying, and it is OK to include Click. Another example is Click on a Tweet anywhere (other than the fav,
* reply, etc. buttons), which leads to the TweetDetails page. Avoid generic action names like Click if
* there is a more specific UI aspect to reference and Click is implied, e.g. ClientTweetReport is
* preferred over ClientTweetClickReport and ClientTweetReportClick.
*
* c) Rely on versioning found in the origin when it is present for action names. For example,
* a "V2Impression" is named as such because in behavioral client events, there is
* a "v2Impress" field. See go/bce-v2impress for more details.
*
* d) There is a distinction between "UndoAction" and "Un{Action}" action types.
* An "UndoAction" is fired when a user clicks on the explicit "Undo" button, after they perform an action
* This "Undo" button is a UI element that may be temporary, e.g.,
* - the user waited too long to click the button, the button disappears from the UI (e.g., Undo for Mute, Block)
* - the button does not disappear due to timeout, but becomes unavailable after the user closes a tab
* (e.g, Undo for NotInterestedIn, NotAboutTopic)
* Examples:
- ClientProfileUndoMute: a user clicks the "Undo" button after muting a Profile
- ClientTweetUndoNotInterestedIn: a users clicks the "Undo" button
after clicking "Not interested in this Tweet" button in the caret menu of a Tweet
* An "Un{Action}" is fired when a user reverses a previous action, not by explicitly clicking an "Undo" button,
* but through some other action that allows them to revert.
* Examples:
* - ClientProfileUnmute: a user clicks the "Unmute" button from the caret menu of the Profile they previously muted
* - ClientTweetUnfav: a user unlikes a tweet by clicking on like button again
*
* Examples: ServerTweetFav, ClientTweetRenderImpression, ClientNotificationSeeLessOften
*
* See go/uua-action-type for more details.
*/
enum ActionType {
// 0 - 999 used for actions derived from Server-side sources (e.g. Timelineservice, Tweetypie)
// NOTE: Please match values for corresponding server / client enum members (with offset 1000).
ServerTweetFav = 0
ServerTweetUnfav = 1
// Reserve 2 and 3 for ServerTweetLingerImpression and ServerTweetRenderImpression
ServerTweetCreate = 4
ServerTweetReply = 5
ServerTweetQuote = 6
ServerTweetRetweet = 7
// skip 8-10 since there are no server equivalents for ClickCreate, ClickReply, ClickQuote
// reserve 11-16 for server video engagements
ServerTweetDelete = 17 // User deletes a default tweet
ServerTweetUnreply = 18 // User deletes a reply tweet
ServerTweetUnquote = 19 // User deletes a quote tweet
ServerTweetUnretweet = 20 // User removes an existing retweet
// User edits a tweet. Edit will create a new tweet with editedTweetId = id of the original tweet
// The original tweet or the new tweet from edit can only be a default or quote tweet.
// A user can edit a default tweet to become a quote tweet (by adding the link to another Tweet),
// or edit a quote tweet to remove the quote and make it a default tweet.
// Both the initial tweet and the new tweet created from the edit can be edited, and each time the
// new edit will create a new tweet. All subsequent edits would have the same initial tweet id
// as the TweetInfo.editedTweetId.
// e.g. create Tweet A, edit Tweet A -> Tweet B, edit Tweet B -> Tweet C
// initial tweet id for both Tweet B anc Tweet C would be Tweet A
ServerTweetEdit = 21
// skip 22 for delete an edit if we want to add it in the future
// reserve 30-40 for server topic actions
// 41-70 reserved for all negative engagements and the related positive engagements
// For example, Follow and Unfollow, Mute and Unmute
// This is fired when a user click "Submit" at the end of a "Report Tweet" flow
// ClientTweetReport = 1041 is scribed by HealthClient team, on the client side
// This is scribed by spamacaw, on the server side
// They can be joined on reportFlowId
// See https://confluence.twitter.biz/pages/viewpage.action?spaceKey=HEALTH&title=Understanding+ReportDetails
ServerTweetReport = 41
// reserve 42 for ServerTweetNotInterestedIn
// reserve 43 for ServerTweetUndoNotInterestedIn
// reserve 44 for ServerTweetNotAboutTopic
// reserve 45 for ServerTweetUndoNotAboutTopic
ServerProfileFollow = 50 // User follows a Profile
ServerProfileUnfollow = 51 // User unfollows a Profile
ServerProfileBlock = 52 // User blocks a Profile
ServerProfileUnblock = 53 // User unblocks a Profile
ServerProfileMute = 54 // User mutes a Profile
ServerProfileUnmute = 55 // User unmutes a Profile
// User reports a Profile as Spam / Abuse
// This user action type includes ProfileReportAsSpam and ProfileReportAsAbuse
ServerProfileReport = 56
// reserve 57 for ServerProfileUnReport
// reserve 56-70 for server social graph actions
// 71-90 reserved for click-based events
// reserve 71 for ServerTweetClick
// 1000 - 1999 used for actions derived from Client-side sources (e.g. Client Events, BCE)
// NOTE: Please match values for corresponding server / client enum members (with offset 1000).
// 1000 - 1499 used for legacy client events
ClientTweetFav = 1000
ClientTweetUnfav = 1001
ClientTweetLingerImpression = 1002
// Please note that: Render impression for quoted Tweets would emit 2 events:
// 1 for the quoting Tweet and 1 for the original Tweet!!!
ClientTweetRenderImpression = 1003
// 1004 reserved for ClientTweetCreate
// This is "Send Reply" event to indicate publishing of a reply Tweet as opposed to clicking
// on the reply button to initiate a reply Tweet (captured in ClientTweetClickReply).
// The differences between this and the ServerTweetReply are:
// 1) ServerTweetReply already has the new Tweet Id 2) A sent reply may be lost during transfer
// over the wire and thus may not end up with a follow-up ServerTweetReply.
ClientTweetReply = 1005
// This is the "send quote" event to indicate publishing of a quote tweet as opposed to clicking
// on the quote button to initiate a quote tweet (captured in ClientTweetClickQuote).
// The differences between this and the ServerTweetQuote are:
// 1) ServerTweetQuote already has the new Tweet Id 2) A sent quote may be lost during transfer
// over the wire and thus may not end up with a follow-up ServerTweetQuote.
ClientTweetQuote = 1006
// This is the "retweet" event to indicate publishing of a retweet.
ClientTweetRetweet = 1007
// 1008 reserved for ClientTweetClickCreate
// This is user clicking on the Reply button not actually sending a reply Tweet,
// thus the name ClickReply
ClientTweetClickReply = 1009
// This is user clicking the Quote/RetweetWithComment button not actually sending the quote,
// thus the name ClickQuote
ClientTweetClickQuote = 1010
// 1011 - 1016: Refer to go/cme-scribing and go/interaction-event-spec for details
// This is fired when playback reaches 25% of total track duration. Not valid for live videos.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoPlayback25 = 1011
// This is fired when playback reaches 50% of total track duration. Not valid for live videos.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoPlayback50 = 1012
// This is fired when playback reaches 75% of total track duration. Not valid for live videos.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoPlayback75 = 1013
// This is fired when playback reaches 95% of total track duration. Not valid for live videos.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoPlayback95 = 1014
// This if fired when the video has been played in non-preview
// (i.e. not autoplaying in the timeline) mode, and was not started via auto-advance.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoPlayFromTap = 1015
// This is fired when 50% of the video has been on-screen and playing for 10 consecutive seconds
// or 95% of the video duration, whichever comes first.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoQualityView = 1016
// Fired when either view_threshold or play_from_tap is fired.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoView = 1109
// Fired when 50% of the video has been on-screen and playing for 2 consecutive seconds,
// regardless of video duration.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoMrcView = 1110
// Fired when the video is:
// - Playing for 3 cumulative (not necessarily consecutive) seconds with 100% in view for looping video.
// - Playing for 3 cumulative (not necessarily consecutive) seconds or the video duration, whichever comes first, with 100% in view for non-looping video.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoViewThreshold = 1111
// Fired when the user clicks a generic ‘visit url’ call to action.
ClientTweetVideoCtaUrlClick = 1112
// Fired when the user clicks a ‘watch now’ call to action.
ClientTweetVideoCtaWatchClick = 1113
// 1017 reserved for ClientTweetDelete
// 1018-1019 for Client delete a reply and delete a quote if we want to add them in the future
// This is fired when a user clicks on "Undo retweet" after re-tweeting a tweet
ClientTweetUnretweet = 1020
// 1021 reserved for ClientTweetEdit
// 1022 reserved for Client delete an edit if we want to add it in the future
// This is fired when a user clicks on a photo within a tweet and the photo expands to fit
// the screen.
ClientTweetPhotoExpand = 1023
// This is fired when a user clicks on a profile mention inside a tweet.
ClientTweetClickMentionScreenName = 1024
// 1030 - 1035 for topic actions
// There are multiple cases:
// 1. Follow from the Topic page (or so-called landing page)
// 2. Click on Tweet's caret menu of "Follow (the topic)", it needs to be:
// 1) user follows the Topic already (otherwise there is no "Follow" menu by default),
// 2) and clicked on the "Unfollow Topic" first.
ClientTopicFollow = 1030
// There are multiple cases:
// 1. Unfollow from the Topic page (or so-called landing page)
// 2. Click on Tweet's caret menu of "Unfollow (the topic)" if the user has already followed
// the topic.
ClientTopicUnfollow = 1031
// This is fired when the user clicks the "x" icon next to the topic on their timeline,
// and clicks "Not interested in {TOPIC}" in the pop-up prompt
// Alternatively, they can also click "See more" button to visit the topic page, and click "Not interested" there.
ClientTopicNotInterestedIn = 1032
// This is fired when the user clicks the "Undo" button after clicking "x" or "Not interested" on a Topic
// which is captured in ClientTopicNotInterestedIn
ClientTopicUndoNotInterestedIn = 1033
// 1036-1070 reserved for all negative engagements and the related positive engagements
// For example, Follow and Unfollow, Mute and Unmute
// This is fired when a user clicks on "This Tweet's not helpful" flow in the caret menu
// of a Tweet result on the Search Results Page
ClientTweetNotHelpful = 1036
// This is fired when a user clicks Undo after clicking on
// "This Tweet's not helpful" flow in the caret menu of a Tweet result on the Search Results Page
ClientTweetUndoNotHelpful = 1037
// This is fired when a user starts and/or completes the "Report Tweet" flow in the caret menu of a Tweet
ClientTweetReport = 1041
/*
* 1042-1045 refers to actions that are related to the
* "Not Interested In" button in the caret menu of a Tweet.
*
* ClientTweetNotInterestedIn is fired when a user clicks the
* "Not interested in this Tweet" button in the caret menu of a Tweet.
* A user can undo the ClientTweetNotInterestedIn action by clicking the
* "Undo" button that appears as a prompt in the caret menu, resulting
* in ClientTweetUndoNotInterestedIn being fired.
* If a user chooses to not undo and proceed, they are given multiple choices
* in a prompt to better document why they are not interested in a Tweet.
* For example, if a Tweet is not about a Topic, a user can click
* "This Tweet is not about {TOPIC}" in the provided prompt, resulting in
* in ClientTweetNotAboutTopic being fired.
* A user can undo the ClientTweetNotAboutTopic action by clicking the "Undo"
* button that appears as a subsequent prompt in the caret menu. Undoing this action
* results in the previous UI state, where the user had only marked "Not Interested In" and
* can still undo the original ClientTweetNotInterestedIn action.
* Similarly a user can select "This Tweet isn't recent" action resulting in ClientTweetNotRecent
* and he could undo this action immediately which results in ClientTweetUndoNotRecent
* Similarly a user can select "Show fewer tweets from" action resulting in ClientTweetSeeFewer
* and he could undo this action immediately which results in ClientTweetUndoSeeFewer
*/
ClientTweetNotInterestedIn = 1042
ClientTweetUndoNotInterestedIn = 1043
ClientTweetNotAboutTopic = 1044
ClientTweetUndoNotAboutTopic = 1045
ClientTweetNotRecent = 1046
ClientTweetUndoNotRecent = 1047
ClientTweetSeeFewer = 1048
ClientTweetUndoSeeFewer = 1049
// This is fired when a user follows a profile from the
// profile page header / people module and people tab on the Search Results Page / sidebar on the Home page
// A Profile can also be followed when a user clicks follow in the caret menu of a Tweet
// or follow button on hovering on profile avatar, which is captured in ClientTweetFollowAuthor = 1060
ClientProfileFollow = 1050
// reserve 1050/1051 for client side Follow/Unfollow
// This is fired when a user clicks Block in a Profile page
// A Profile can also be blocked when a user clicks Block in the caret menu of a Tweet,
// which is captured in ClientTweetBlockAuthor = 1062
ClientProfileBlock = 1052
// This is fired when a user clicks unblock in a pop-up prompt right after blocking a profile
// in the profile page or clicks unblock in a drop-down menu in the profile page.
ClientProfileUnblock = 1053
// This is fired when a user clicks Mute in a Profile page
// A Profile can also be muted when a user clicks Mute in the caret menu of a Tweet, which is captured in ClientTweetMuteAuthor = 1064
ClientProfileMute = 1054
// reserve 1055 for client side Unmute
// This is fired when a user clicks "Report User" action from user profile page
ClientProfileReport = 1056
// reserve 1057 for ClientProfileUnreport
// This is fired when a user clicks on a profile from all modules except tweets
// (eg: People Search / people module in Top tab in Search Result Page
// For tweets, the click is captured in ClientTweetClickProfile
ClientProfileClick = 1058
// reserve 1059-1070 for client social graph actions
// This is fired when a user clicks Follow in the caret menu of a Tweet or hovers on the avatar of the tweet
// author and clicks on the Follow button. A profile can also be followed by clicking the Follow button on the
// Profile page and confirm, which is captured in ClientProfileFollow. The event emits two items, one of user type
// and another of tweet type, since the default implementation of BaseClientEvent only looks for Tweet type,
// the other item is dropped which is the expected behaviour
ClientTweetFollowAuthor = 1060
// This is fired when a user clicks Unfollow in the caret menu of a Tweet or hovers on the avatar of the tweet
// author and clicks on the Unfollow button. A profile can also be unfollowed by clicking the Unfollow button on the
// Profile page and confirm, which will be captured in ClientProfileUnfollow. The event emits two items, one of user type
// and another of tweet type, since the default implementation of BaseClientEvent only looks for Tweet type,
// the other item is dropped which is the expected behaviour
ClientTweetUnfollowAuthor = 1061
// This is fired when a user clicks Block in the menu of a Tweet to block the Profile that
// authored this Tweet. A Profile can also be blocked in the Profile page, which is captured
// in ClientProfileBlock = 1052
ClientTweetBlockAuthor = 1062
// This is fired when a user clicks unblock in a pop-up prompt right after blocking an author
// in the drop-down menu of a tweet
ClientTweetUnblockAuthor = 1063
// This is fired when a user clicks Mute in the menu of a Tweet to block the Profile that
// authored this Tweet. A Profile can also be muted in the Profile page, which is captured in ClientProfileMute = 1054
ClientTweetMuteAuthor = 1064
// reserve 1065 for ClientTweetUnmuteAuthor
// 1071-1090 reserved for click-based events
// click-based events are defined as clicks on a UI container (e.g., tweet, profile, etc.), as opposed to clearly named
// button or menu (e.g., follow, block, report, etc.), which requires a specific action name than "click".
// This is fired when a user clicks on a Tweet to open the Tweet details page. Note that for
// Tweets in the Notification Tab product surface, a click can be registered differently
// depending on whether the Tweet is a rendered Tweet (a click results in ClientTweetClick)
// or a wrapper Notification (a click results in ClientNotificationClick).
ClientTweetClick = 1071
// This is fired when a user clicks to view the profile page of a user from a tweet
// Contains a TweetInfo of this tweet
ClientTweetClickProfile = 1072
// This is fired when a user clicks on the "share" icon on a Tweet to open the share menu.
// The user may or may not proceed and finish sharing the Tweet.
ClientTweetClickShare = 1073
// This is fired when a user clicks "Copy link to Tweet" in a menu appeared after hitting
// the "share" icon on a Tweet OR when a user selects share_via -> copy_link after long-click
// a link inside a tweet on a mobile device
ClientTweetShareViaCopyLink = 1074
// This is fired when a user clicks "Send via Direct Message" after
// clicking on the "share" icon on a Tweet to open the share menu.
// The user may or may not proceed and finish Sending the DM.
ClientTweetClickSendViaDirectMessage = 1075
// This is fired when a user clicks "Bookmark" after
// clicking on the "share" icon on a Tweet to open the share menu.
ClientTweetShareViaBookmark = 1076
// This is fired when a user clicks "Remove Tweet from Bookmarks" after
// clicking on the "share" icon on a Tweet to open the share menu.
ClientTweetUnbookmark = 1077
// This is fired when a user clicks on the hashtag in a Tweet.
// The click on hashtag in "What's happening" section gives you other scribe '*:*:sidebar:*:trend:search'
// Currenly we are only filtering for itemType=Tweet. There are other items present in the event where itemType = user
// but those items are in dual-events (events with multiple itemTypes) and happen when you click on a hashtag in a Tweet from someone's profile,
// hence we are ignoring those itemType and only keeping itemType=Tweet.
ClientTweetClickHashtag = 1078
// This is fired when a user clicks "Bookmark" after clicking on the "share" icon on a Tweet to open the share menu, or
// when a user clicks on the 'bookmark' icon on a Tweet (bookmark icon is available to ios only as of March 2023).
// TweetBookmark and TweetShareByBookmark log the same events but serve for individual use cases.
ClientTweetBookmark = 1079
// 1078 - 1089 for all Share related actions.
// This is fired when a user clicks on a link in a tweet.
// The link could be displayed as a URL or embedded in a component such as an image or a card in a tweet.
ClientTweetOpenLink = 1090
// This is fired when a user takes screenshot.
// This is available for mobile clients only.
ClientTweetTakeScreenshot = 1091
// 1100 - 1101: Refer to go/cme-scribing and go/interaction-event-spec for details
// Fired on the first tick of a track regardless of where in the video it is playing.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoPlaybackStart = 1100
// Fired when playback reaches 100% of total track duration.
// Not valid for live videos.
// For looping playback, this is only fired once and does not reset at loop boundaries.
ClientTweetVideoPlaybackComplete = 1101
// A user can select "This Tweet isn't relevant" action resulting in ClientTweetNotRelevant
// and they could undo this action immediately which results in ClientTweetUndoNotRelevant
ClientTweetNotRelevant = 1102
ClientTweetUndoNotRelevant = 1103
// A generic action type to submit feedback for different modules / items ( Tweets / Search Results )
ClientFeedbackPromptSubmit = 1104
// This is fired when a user profile is open in a Profile page
ClientProfileShow = 1105
/*
* This is triggered when a user exits the Twitter platform. The amount of the time spent on the
* platform is recorded in ms that can be used to compute the User Active Seconds (UAS).
*/
ClientAppExit = 1106
/*
* For "card" related actions
*/
ClientCardClick = 1107
ClientCardOpenApp = 1108
ClientCardAppInstallAttempt = 1114
ClientPollCardVote = 1115
/*
* The impressions 1121-1123 together with the ClientTweetRenderImpression 1003 are used by ViewCount
* and UnifiedEngagementCounts as EngagementType.Displayed and EngagementType.Details.
*
* For definitions, please refer to https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/common-internal/analytics/client-event-util/src/main/java/com/twitter/common_internal/analytics/client_event_util/TweetImpressionUtils.java?L14&subtree=true
*/
ClientTweetGalleryImpression = 1121
ClientTweetDetailsImpression = 1122
/**
* This is fired when a user is logged out and follows a profile from the
* profile page / people module from web.
* One can only try to follow from web because iOS and Android do not support logged out browsing as of Jan 2023.
*/
ClientProfileFollowAttempt = 1200
/**
* This is fired when a user is logged out and favourite a tweet from web.
* One can only try to favourite from web, iOS and Android do not support logged out browsing
*/
ClientTweetFavoriteAttempt = 1201
/**
* This is fired when a user is logged out and Retweet a tweet from web.
* One can only try to favourite from web, iOS and Android do not support logged out browsing
*/
ClientTweetRetweetAttempt = 1202
/**
* This is fired when a user is logged out and reply on tweet from web.
* One can only try to favourite from web, iOS and Android do not support logged out browsing
*/
ClientTweetReplyAttempt = 1203
/**
* This is fired when a user is logged out and clicks on login button.
* Currently seem to be generated only on [m5, LiteNativeWrapper]
*/
ClientCTALoginClick = 1204
/**
* This is fired when a user is logged out and login window is shown.
*/
ClientCTALoginStart = 1205
/**
* This is fired when a user is logged out and login is successful.
*/
ClientCTALoginSuccess = 1206
/**
* This is fired when a user is logged out and clicks on signup button.
*/
ClientCTASignupClick = 1207
/**
* This is fired when a user is logged out and signup is successful.
*/
ClientCTASignupSuccess = 1208
// 1400 - 1499 for product surface specific actions
// This is fired when a user opens a Push Notification
ClientNotificationOpen = 1400
// This is fired when a user clicks on a Notification in the Notification Tab
ClientNotificationClick = 1401
// This is fired when a user taps the "See Less Often" caret menu item of a Notification in the Notification Tab
ClientNotificationSeeLessOften = 1402
// This is fired when a user closes or swipes away a Push Notification
ClientNotificationDismiss = 1403
// 1420 - 1439 is reserved for Search Results Page related actions
// 1440 - 1449 is reserved for Typeahead related actions
// This is fired when a user clicks on a typeahead suggestion(queries, events, topics, users)
// in a drop-down menu of a search box or a tweet compose box.
ClientTypeaheadClick = 1440
// 1500 - 1999 used for behavioral client events
// Tweet related impressions
ClientTweetV2Impression = 1500
/* Fullscreen impressions
*
* Android client will always log fullscreen_video impressions, regardless of the media type
* i.e. video, image, MM will all be logged as fullscreen_video
*
* iOS clients will log fullscreen_video or fullscreen_image depending on the media type
* on display when the user exits fullscreen. i.e.
* - image tweet => fullscreen_image
* - video tweet => fullscreen_video
* - MM tweet => fullscreen_video if user exits fullscreen from the video
* => fullscreen_image if user exits fullscreen from the image
*
* Web clients will always log fullscreen_image impressions, regardless of the media type
*
* References
* https://docs.google.com/document/d/1oEt9_Gtz34cmO_JWNag5YKKEq4Q7cJFL-nbHOmhnq1Y
* https://docs.google.com/document/d/1V_7TbfPvTQgtE_91r5SubD7n78JsVR_iToW59gOMrfQ
*/
ClientTweetVideoFullscreenV2Impression = 1501
ClientTweetImageFullscreenV2Impression = 1502
// Profile related impressions
ClientProfileV2Impression = 1600
/*
* Email Notifications: These are actions taken by the user in response to Your Highlights email
* ClientTweetEmailClick refers to the action NotificationType.Click
*/
ClientTweetEmailClick = 5001
/*
* User create via Gizmoduck
*/
ServerUserCreate = 6000
ServerUserUpdate = 6001
/*
* Ads callback engagements
*/
/*
* This engagement is generated when a user Favs a promoted Tweet.
*/
ServerPromotedTweetFav = 7000
/*
* This engagement is generated when a user Unfavs a promoted Tweet that they previously Faved.
*/
ServerPromotedTweetUnfav = 7001
ServerPromotedTweetReply = 7002
ServerPromotedTweetRetweet = 7004
/*
* The block could be performed from the promoted tweet or on the promoted tweet's author's profile
* ads_spend_event data shows majority (~97%) of blocks have an associated promoted tweet id
* So for now we assume the blocks are largely performed from the tweet and following naming convention of ClientTweetBlockAuthor
*/
ServerPromotedTweetBlockAuthor = 7006
ServerPromotedTweetUnblockAuthor = 7007
/*
* This is when a user clicks on the Conversational Card in the Promoted Tweet which
* leads to the Tweet Compose page. The user may or may not send the new Tweet.
*/
ServerPromotedTweetComposeTweet = 7008
/*
* This is when a user clicks on the Promoted Tweet to view its details/replies.
*/
ServerPromotedTweetClick = 7009
/*
* The video ads engagements are divided into two sets: VIDEO_CONTENT_* and VIDEO_AD_*. These engagements
* have similar definitions. VIDEO_CONTENT_* engagements are fired for videos that are part of
* a Tweet. VIDEO_AD_* engagements are fired for a preroll ad. A preroll ad can play on a promoted
* Tweet or on an organic Tweet. go/preroll-matching for more information.
*
* 7011-7013: A Promoted Event is fired when playback reaches 25%, 50%, 75% of total track duration.
* This is for the video on a promoted Tweet.
* Not valid for live videos. Refer go/avscribing.
* For a video that has a preroll ad played before it, the metadata will contain information about
* the preroll ad as well as the video itself. There will be no preroll metadata if there was no
* preroll ad played.
*/
ServerPromotedTweetVideoPlayback25 = 7011
ServerPromotedTweetVideoPlayback50 = 7012
ServerPromotedTweetVideoPlayback75 = 7013
/*
* This is when a user successfully completes the Report flow on a Promoted Tweet.
* It covers reports for all policies from Client Event.
*/
ServerPromotedTweetReport = 7041
/*
* Follow from Ads data stream, it could be from both Tweet or other places
*/
ServerPromotedProfileFollow = 7060
/*
* Follow from Ads data stream, it could be from both Tweet or other places
*/
ServerPromotedProfileUnfollow = 7061
/*
* This is when a user clicks on the mute promoted tweet's author option from the menu.
*/
ServerPromotedTweetMuteAuthor = 7064
/*
* This is fired when a user clicks on the profile image, screen name, or the user name of the
* author of the Promoted Tweet which leads to the author's profile page.
*/
ServerPromotedTweetClickProfile = 7072
/*
* This is fired when a user clicks on a hashtag in the Promoted Tweet.
*/
ServerPromotedTweetClickHashtag = 7078
/*
* This is fired when a user opens link by clicking on a URL in the Promoted Tweet.
*/
ServerPromotedTweetOpenLink = 7079
/*
* This is fired when a user swipes to the next element of the carousel in the Promoted Tweet.
*/
ServerPromotedTweetCarouselSwipeNext = 7091
/*
* This is fired when a user swipes to the previous element of the carousel in the Promoted Tweet.
*/
ServerPromotedTweetCarouselSwipePrevious = 7092
/*
* This event is only for the Promoted Tweets with a web URL.
* It is fired after exiting a WebView from a Promoted Tweet if the user was on the WebView for
* at least 1 second.
*
* See https://confluence.twitter.biz/display/REVENUE/dwell_short for more details.
*/
ServerPromotedTweetLingerImpressionShort = 7093
/*
* This event is only for the Promoted Tweets with a web URL.
* It is fired after exiting a WebView from a Promoted Tweet if the user was on the WebView for
* at least 2 seconds.
*
* See https://confluence.twitter.biz/display/REVENUE/dwell_medium for more details.
*/
ServerPromotedTweetLingerImpressionMedium = 7094
/*
* This event is only for the Promoted Tweets with a web URL.
* It is fired after exiting a WebView from a Promoted Tweet if the user was on the WebView for
* at least 10 seconds.
*
* See https://confluence.twitter.biz/display/REVENUE/dwell_long for more details.
*/
ServerPromotedTweetLingerImpressionLong = 7095
/*
* This is fired when a user navigates to explorer page (taps search magnifying glass on Home page)
* and a Promoted Trend is present and taps ON the promoted spotlight - a video/gif/image in the
* "hero" position (top of the explorer page).
*/
ServerPromotedTweetClickSpotlight = 7096
/*
* This is fired when a user navigates to explorer page (taps search magnifying glass on Home page)
* and a Promoted Trend is present.
*/
ServerPromotedTweetViewSpotlight = 7097
/*
* 7098-7099: Promoted Trends appear in the first or second slots of the “Trends for you” section
* in the Explore tab and “What’s Happening” module on Twitter.com. For more information, check go/ads-takeover.
* 7099: This is fired when a user views a promoted Trend. It should be considered as an impression.
*/
ServerPromotedTrendView = 7098
/*
* 7099: This is fired when a user clicks a promoted Trend. It should be considered as an engagment.
*/
ServerPromotedTrendClick = 7099
/*
* 7131-7133: A Promoted Event fired when playback reaches 25%, 50%, 75% of total track duration.
* This is for the preroll ad that plays before a video on a promoted Tweet.
* Not valid for live videos. Refer go/avscribing.
* This will only contain metadata for the preroll ad.
*/
ServerPromotedTweetVideoAdPlayback25 = 7131
ServerPromotedTweetVideoAdPlayback50 = 7132
ServerPromotedTweetVideoAdPlayback75 = 7133
/*
* 7151-7153: A Promoted Event fired when playback reaches 25%, 50%, 75% of total track duration.
* This is for the preroll ad that plays before a video on an organic Tweet.
* Not valid for live videos. Refer go/avscribing.
* This will only contain metadata for the preroll ad.
*/
ServerTweetVideoAdPlayback25 = 7151
ServerTweetVideoAdPlayback50 = 7152
ServerTweetVideoAdPlayback75 = 7153
ServerPromotedTweetDismissWithoutReason = 7180
ServerPromotedTweetDismissUninteresting = 7181
ServerPromotedTweetDismissRepetitive = 7182
ServerPromotedTweetDismissSpam = 7183
/*
* For FavoriteArchival Events
*/
ServerTweetArchiveFavorite = 8000
ServerTweetUnarchiveFavorite = 8001
/*
* For RetweetArchival Events
*/
ServerTweetArchiveRetweet = 8002
ServerTweetUnarchiveRetweet = 8003
}(persisted='true', hasPersonalData='false')
/*
* This union will be updated when we have a particular
* action that has attributes unique to that particular action
* (e.g. linger impressions have start/end times) and not common
* to all tweet actions.
* Naming convention for TweetActionInfo should be consistent with
* ActionType. For example, `ClientTweetLingerImpression` ActionType enum
* should correspond to `ClientTweetLingerImpression` TweetActionInfo union arm.
* We typically preserve 1:1 mapping between ActionType and TweetActionInfo. However, we make
* exceptions when optimizing for customer requirements. For example, multiple 'ClientTweetVideo*'
* ActionType enums correspond to a single `TweetVideoWatch` TweetActionInfo union arm because
* customers want individual action labels but common information across those labels.
*/
union TweetActionInfo {
// 41 matches enum index ServerTweetReport in ActionType
41: ServerTweetReport serverTweetReport
// 1002 matches enum index ClientTweetLingerImpression in ActionType
1002: ClientTweetLingerImpression clientTweetLingerImpression
// Common metadata for
// 1. "ClientTweetVideo*" ActionTypes with enum indices 1011-1016 and 1100-1101
// 2. "ServerPromotedTweetVideo*" ActionTypes with enum indices 7011-7013 and 7131-7133
// 3. "ServerTweetVideo*" ActionTypes with enum indices 7151-7153
// This is because:
// 1. all the above listed ActionTypes share common metadata
// 2. more modular code as the same struct can be reused
// 3. reduces chance of error while populating and parsing the metadata
// 4. consumers can easily process the metadata
1011: TweetVideoWatch tweetVideoWatch
// 1012: skip
// 1013: skip
// 1014: skip
// 1015: skip
// 1016: skip
// 1024 matches enum index ClientTweetClickMentionScreenName in ActionType
1024: ClientTweetClickMentionScreenName clientTweetClickMentionScreenName
// 1041 matches enum index ClientTweetReport in ActionType
1041: ClientTweetReport clientTweetReport
// 1060 matches enum index ClientTweetFollowAuthor in ActionType
1060: ClientTweetFollowAuthor clientTweetFollowAuthor
// 1061 matches enum index ClientTweetUnfollowAuthor in ActionType
1061: ClientTweetUnfollowAuthor clientTweetUnfollowAuthor
// 1078 matches enum index ClientTweetClickHashtag in ActionType
1078: ClientTweetClickHashtag clientTweetClickHashtag
// 1090 matches enum index ClientTweetOpenLink in ActionType
1090: ClientTweetOpenLink clientTweetOpenLink
// 1091 matches enum index ClientTweetTakeScreenshot in ActionType
1091: ClientTweetTakeScreenshot clientTweetTakeScreenshot
// 1500 matches enum index ClientTweetV2Impression in ActionType
1500: ClientTweetV2Impression clientTweetV2Impression
// 7079 matches enum index ServerPromotedTweetOpenLink in ActionType
7079: ServerPromotedTweetOpenLink serverPromotedTweetOpenLink
}(persisted='true', hasPersonalData='true')
struct ClientTweetOpenLink {
//Url which was clicked.
1: optional string url(personalDataType = 'RawUrlPath')
}(persisted='true', hasPersonalData='true')
struct ServerPromotedTweetOpenLink {
//Url which was clicked.
1: optional string url(personalDataType = 'RawUrlPath')
}(persisted='true', hasPersonalData='true')
struct ClientTweetClickHashtag {
/* Hashtag string which was clicked. The PDP annotation is SearchQuery,
* because clicking on the hashtag triggers a search with the hashtag
*/
1: optional string hashtag(personalDataType = 'SearchQuery')
}(persisted='true', hasPersonalData='true')
struct ClientTweetTakeScreenshot {
//percentage visible height.
1: optional i32 percentVisibleHeight100k
}(persisted='true', hasPersonalData='false')
/*
* See go/ioslingerimpressionbehaviors and go/lingerandroidfaq
* for ios and android client definitions of a linger respectively.
*/
struct ClientTweetLingerImpression {
/* Milliseconds since epoch when the tweet became more than 50% visible. */
1: required i64 lingerStartTimestampMs(personalDataType = 'ImpressionMetadata')
/* Milliseconds since epoch when the tweet became less than 50% visible. */
2: required i64 lingerEndTimestampMs(personalDataType = 'ImpressionMetadata')
}(persisted='true', hasPersonalData='true')
/*
* See go/behavioral-client-events for general behavioral client event (BCE) information
* and go/bce-v2impress for detailed information about BCE impression events.
*
* Unlike ClientTweetLingerImpression, there is no lower bound on the amount of time
* necessary for the impress event to occur. There is also no visibility requirement for a impress
* event to occur.
*/
struct ClientTweetV2Impression {
/* Milliseconds since epoch when the tweet became visible. */
1: required i64 impressStartTimestampMs(personalDataType = 'ImpressionMetadata')
/* Milliseconds since epoch when the tweet became visible. */
2: required i64 impressEndTimestampMs(personalDataType = 'ImpressionMetadata')
/*
* The UI component that hosted this tweet where the impress event happened.
*
* For example, sourceComponent = "tweet" if the impress event happened on a tweet displayed amongst
* a collection of tweets, or sourceComponent = "tweet_details" if the impress event happened on
* a tweet detail UI component.
*/
3: required string sourceComponent(personalDataType = 'WebsitePage')
}(persisted='true', hasPersonalData='true')
/*
* Refer to go/cme-scribing and go/interaction-event-spec for details
*/
struct TweetVideoWatch {
/*
* Type of video included in the Tweet
*/
1: optional client_app.MediaType mediaType(personalDataType = 'MediaFile')
/*
* Whether the video content is "monetizable", i.e.,
* if a preroll ad may be served dynamically when the video plays
*/
2: optional bool isMonetizable(personalDataType = 'MediaFile')
/*
* The owner of the video, provided by playlist.
*
* For ad engagements related to a preroll ad (VIDEO_AD_*),
* this will be the owner of the preroll ad and same as the prerollOwnerId.
*
* For ad engagements related to a regular video (VIDEO_CONTENT_*), this will be the owner of the
* video and not the preroll ad.
*/
3: optional i64 videoOwnerId(personalDataType = 'UserId')
/*
* Identifies the video associated with a card.
*
* For ad Engagements, in the case of engagements related to a preroll ad (VIDEO_AD_*),
* this will be the id of the preroll ad and same as the prerollUuid.
*
* For ad engagements related to a regular video (VIDEO_CONTENT_*), this will be id of the video
* and not the preroll ad.
*/
4: optional string videoUuid(personalDataType = 'MediaId')
/*
* Id of the preroll ad shown before the video
*/
5: optional string prerollUuid(personalDataType = 'MediaId')
/*
* Advertiser id of the preroll ad
*/
6: optional i64 prerollOwnerId(personalDataType = 'UserId')
/*
* for amplify_flayer events, indicates whether preroll or the main video is being played
*/
7: optional string videoType(personalDataType = 'MediaFile')
}(persisted='true', hasPersonalData='true')
struct ClientTweetClickMentionScreenName {
/* Id for the profile (user_id) that was actioned on */
1: required i64 actionProfileId(personalDataType = 'UserId')
/* The handle/screenName of the user. This can't be changed. */
2: required string handle(personalDataType = 'UserName')
}(persisted='true', hasPersonalData='true')
struct ClientTweetReport {
/*
* Whether the "Report Tweet" flow was successfully completed.
* `true` if the flow was completed successfully, `false` otherwise.
*/
1: required bool isReportTweetDone
/*
* report-flow-id is included in Client Event when the "Report Tweet" flow was initiated
* See go/report-flow-ids and
* https://confluence.twitter.biz/pages/viewpage.action?spaceKey=HEALTH&title=Understanding+ReportDetails
*/
2: optional string reportFlowId
}(persisted='true', hasPersonalData='true')
enum TweetAuthorFollowClickSource {
UNKNOWN = 1
CARET_MENU = 2
PROFILE_IMAGE = 3
}
struct ClientTweetFollowAuthor {
/*
* Where did the user click the Follow button on the tweet - from the caret menu("CARET_MENU")
* or via hovering over the profile and clicking on Follow ("PROFILE_IMAGE") - only applicable for web clients
* "UNKNOWN" if the scribe do not match the expected namespace for the above
*/
1: required TweetAuthorFollowClickSource followClickSource
}(persisted='true', hasPersonalData='false')
enum TweetAuthorUnfollowClickSource {
UNKNOWN = 1
CARET_MENU = 2
PROFILE_IMAGE = 3
}
struct ClientTweetUnfollowAuthor {
/*
* Where did the user click the Unfollow button on the tweet - from the caret menu("CARET_MENU")
* or via hovering over the profile and clicking on Unfollow ("PROFILE_IMAGE") - only applicable for web clients
* "UNKNOWN" if the scribe do not match the expected namespace for the above
*/
1: required TweetAuthorUnfollowClickSource unfollowClickSource
}(persisted='true', hasPersonalData='false')
struct ServerTweetReport {
/*
* ReportDetails will be populated when the tweet report was scribed by spamacaw (server side)
* Only for the action submit, all the fields under ReportDetails will be available.
* This is because only after successful submission, we will know the report_type and report_flow_name.
* Reference: https://confluence.twitter.biz/pages/viewpage.action?spaceKey=HEALTH&title=Understanding+ReportDetails
*/
1: optional string reportFlowId
2: optional report_flow_logs.ReportType reportType
}(persisted='true', hasPersonalData='false')
/*
* This union will be updated when we have a particular
* action that has attributes unique to that particular action
* (e.g. linger impressions have start/end times) and not common
* to other profile actions.
*
* Naming convention for ProfileActionInfo should be consistent with
* ActionType. For example, `ClientProfileV2Impression` ActionType enum
* should correspond to `ClientProfileV2Impression` ProfileActionInfo union arm.
*/
union ProfileActionInfo {
// 56 matches enum index ServerProfileReport in ActionType
56: ServerProfileReport serverProfileReport
// 1600 matches enum index ClientProfileV2Impression in ActionType
1600: ClientProfileV2Impression clientProfileV2Impression
// 6001 matches enum index ServerUserUpdate in ActionType
6001: ServerUserUpdate serverUserUpdate
}(persisted='true', hasPersonalData='true')
/*
* See go/behavioral-client-events for general behavioral client event (BCE) information
* and https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.3tu05p92xgxc
* for detailed information about BCE impression event.
*
* Unlike ClientTweetLingerImpression, there is no lower bound on the amount of time
* necessary for the impress event to occur. There is also no visibility requirement for a impress
* event to occur.
*/
struct ClientProfileV2Impression {
/* Milliseconds since epoch when the profile page became visible. */
1: required i64 impressStartTimestampMs(personalDataType = 'ImpressionMetadata')
/* Milliseconds since epoch when the profile page became visible. */
2: required i64 impressEndTimestampMs(personalDataType = 'ImpressionMetadata')
/*
* The UI component that hosted this profile where the impress event happened.
*
* For example, sourceComponent = "profile" if the impress event happened on a profile page
*/
3: required string sourceComponent(personalDataType = 'WebsitePage')
}(persisted='true', hasPersonalData='true')
struct ServerProfileReport {
1: required social_graph_service_write_log.Action reportType(personalDataType = 'ReportType')
}(persisted='true', hasPersonalData='true')
struct ServerUserUpdate {
1: required list<user_service.UpdateDiffItem> updates
2: optional bool success (personalDataType = 'AuditMessage')
}(persisted='true', hasPersonalData='true')

BIN
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/common.docx Normal file
View File

Binary file not shown.

20
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/common.thrift
View File

@ -1,20 +0,0 @@
namespace java com.twitter.unified_user_actions.thriftjava
#@namespace scala com.twitter.unified_user_actions.thriftscala
#@namespace strato com.twitter.unified_user_actions
/*
* Uniquely identifies a user. A user identifier
* for a logged in user should contain a user id
* and a user identifier for a logged out user should
* contain some guest id. A user may have multiple ids.
*/
struct UserIdentifier {
1: optional i64 userId(personalDataType='UserId')
/*
* See http://go/guest-id-cookie-tdd. As of Dec 2021,
* guest id is intended only for essential use cases
* (e.g. logged out preferences, security). Guest id
* marketing is intended for recommendation use cases.
*/
2: optional i64 guestIdMarketing(personalDataType='GuestId')
}(persisted='true', hasPersonalData='true')

BIN
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/item.docx Normal file
View File

Binary file not shown.

294
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/item.thrift
View File

@ -1,294 +0,0 @@
namespace java com.twitter.unified_user_actions.thriftjava
#@namespace scala com.twitter.unified_user_actions.thriftscala
#@namespace strato com.twitter.unified_user_actions
include "com/twitter/unified_user_actions/action_info.thrift"
include "com/twitter/clientapp/gen/client_app.thrift"
/*
* Tweet item information. Some development notes:
* 1. Please keep this top-level struct as minimal as possible to reduce overhead.
* 2. We intentionally avoid nesting action tweet in a separate structure
* to underscore its importance and faciliate extraction of most commonly
* needed fields such as actionTweetId. New fields related to the action tweet
* should generally be prefixed with "actionTweet".
* 3. For the related Tweets, e.g. retweetingTweetId, inReplyToTweetId, etc, we
* mostly only keep their ids for consistency and simplicity.
*/
struct TweetInfo {
/* Id for the tweet that was actioned on */
1: required i64 actionTweetId(personalDataType = 'TweetId')
// Deprecated, please don't re-use!
// 2: optional i64 actionTweetAuthorId(personalDataType = 'UserId')
/* The social proof (i.e. banner) Topic Id that the action Tweet is associated to */
3: optional i64 actionTweetTopicSocialProofId(personalDataType='InferredInterests, ProvidedInterests')
4: optional AuthorInfo actionTweetAuthorInfo
// Fields 1-99 reserved for `actionFooBar` fields
/* Additional details for the action that took place on actionTweetId */
100: optional action_info.TweetActionInfo tweetActionInfo
/* Id of the tweet retweeting the action tweet */
101: optional i64 retweetingTweetId(personalDataType = 'TweetId')
/* Id of the tweet quoting the action Tweet, when the action type is quote */
102: optional i64 quotingTweetId(personalDataType = 'TweetId')
/* Id of the tweet replying to the action Tweet, when the action type is reply */
103: optional i64 replyingTweetId(personalDataType = 'TweetId')
/* Id of the tweet being quoted by the action tweet */
104: optional i64 quotedTweetId(personalDataType = 'TweetId')
/* Id of the tweet being replied to by the action tweet */
105: optional i64 inReplyToTweetId(personalDataType = 'TweetId')
/* Id of the tweet being retweeted by the action tweet, this is just for Unretweet action */
106: optional i64 retweetedTweetId(personalDataType = 'TweetId')
/* Id of the tweet being edited, this is only available for TweetEdit action, and TweetDelete
* action when the deleted tweet was created from Edit. */
107: optional i64 editedTweetId(personalDataType = 'TweetId')
/* Position of a tweet item in a page such as home and tweet detail, and is populated in
* Client Event. */
108: optional i32 tweetPosition
/* PromotedId is provided by ads team for each promoted tweet and is logged in client event */
109: optional string promotedId(personalDataType = 'AdsId')
/* corresponding to inReplyToTweetId */
110: optional i64 inReplyToAuthorId(personalDataType = 'UserId')
/* corresponding to retweetingTweetId */
111: optional i64 retweetingAuthorId(personalDataType = 'UserId')
/* corresponding to quotedTweetId */
112: optional i64 quotedAuthorId(personalDataType = 'UserId')
}(persisted='true', hasPersonalData='true')
/*
* Profile item information. This follows TweetInfo's development notes.
*/
struct ProfileInfo {
/* Id for the profile (user_id) that was actioned on
*
* In a social graph user action, e.g., user1 follows/blocks/mutes user2,
* userIdentifier captures userId of user1 and actionProfileId records
* the userId of user2.
*/
1: required i64 actionProfileId(personalDataType = 'UserId')
// Fields 1-99 reserved for `actionFooBar` fields
/* the full name of the user. max length is 50. */
2: optional string name(personalDataType = 'DisplayName')
/* The handle/screenName of the user. This can't be changed.
*/
3: optional string handle(personalDataType = 'UserName')
/* the "bio" of the user. max length is 160. May contain one or more t.co
* links, which will be hydrated in the UrlEntities substruct if the
* QueryFields.URL_ENTITIES is specified.
*/
4: optional string description(personalDataType = 'Bio')
/* Additional details for the action that took place on actionProfileId */
100: optional action_info.ProfileActionInfo profileActionInfo
}(persisted='true', hasPersonalData='true')
/*
* Topic item information. This follows TweetInfo's development notes.
*/
struct TopicInfo {
/* Id for the Topic that was actioned on */
1: required i64 actionTopicId(personalDataType='InferredInterests, ProvidedInterests')
// Fields 1-99 reserved for `actionFooBar` fields
}(persisted='true', hasPersonalData='true')
/*
* Notification Item information.
*
* See go/phab-d973370-discuss, go/phab-d968144-discuss, and go/uua-action-type for details about
* the schema design for Notification events.
*/
struct NotificationInfo {
/*
* Id of the Notification was actioned on.
*
* Note that this field represents the `impressionId` of a Notification. It has been renamed to
* `notificationId` in UUA so that the name effectively represents the value it holds,
* i.e. a unique id for a Notification and request.
*/
1: required string actionNotificationId(personalDataType='UniversallyUniqueIdentifierUuid')
/*
* Additional information contained in a Notification. This is a `union` arm to differentiate
* among different types of Notifications and store relevant metadata for each type.
*
* For example, a Notification with a single Tweet will hold the Tweet id in `TweetNotification`.
* Similarly, `MultiTweetNotification` is defined for Notiifcations with multiple Tweet ids.
*
* Refer to the definition of `union NotificationContent` below for more details.
*/
2: required NotificationContent content
}(persisted='true', hasPersonalData='true')
/*
* Additional information contained in a Notification.
*/
union NotificationContent {
1: TweetNotification tweetNotification
2: MultiTweetNotification multiTweetNotification
// 3 - 100 reserved for other specific Notification types (for example, profile, event, etc.).
/*
* If a Notification cannot be categorized into any of the types at indices 1 - 100,
* it is considered of `Unknown` type.
*/
101: UnknownNotification unknownNotification
}(persisted='true', hasPersonalData='true')
/*
* Notification contains exactly one `tweetId`.
*/
struct TweetNotification {
1: required i64 tweetId(personalDataType = 'TweetId')
}(persisted='true', hasPersonalData='true')
/*
* Notification contains multiple `tweetIds`.
* For example, user A receives a Notification when user B likes multiple Tweets authored by user A.
*/
struct MultiTweetNotification {
1: required list<i64> tweetIds(personalDataType = 'TweetId')
}(persisted='true', hasPersonalData='true')
/*
* Notification could not be categrized into known types at indices 1 - 100 in `NotificationContent`.
*/
struct UnknownNotification {
// this field is just a placeholder since Sparrow doesn't support empty struct
100: optional bool placeholder
}(persisted='true', hasPersonalData='false')
/*
* Trend Item information for promoted and non-promoted Trends.
*/
struct TrendInfo {
/*
* Identifier for promoted Trends only.
* This is not available for non-promoted Trends and the default value should be set to 0.
*/
1: required i32 actionTrendId(personalDataType= 'TrendId')
/*
* Empty for promoted Trends only.
* This should be set for all non-promoted Trends.
*/
2: optional string actionTrendName
}(persisted='true', hasPersonalData='true')
struct TypeaheadInfo {
/* search query string */
1: required string actionQuery(personalDataType = 'SearchQuery')
2: required TypeaheadActionInfo typeaheadActionInfo
}(persisted='true', hasPersonalData='true')
union TypeaheadActionInfo {
1: UserResult userResult
2: TopicQueryResult topicQueryResult
}(persisted='true', hasPersonalData='true')
struct UserResult {
/* The userId of the profile suggested in the typeahead drop-down, upon which the user took the action */
1: required i64 profileId(personalDataType = 'UserId')
}(persisted='true', hasPersonalData='true')
struct TopicQueryResult {
/* The topic query name suggested in the typeahead drop-down, upon which the user took the action */
1: required string suggestedTopicQuery(personalDataType = 'SearchQuery')
}(persisted='true', hasPersonalData='true')
/*
* Item that captures feedback related information submitted by the user across modules / item (Eg: Search Results / Tweets)
* Design discussion doc: https://docs.google.com/document/d/1UHiCrGzfiXOSymRAUM565KchVLZBAByMwvP4ARxeixY/edit#
*/
struct FeedbackPromptInfo {
1: required FeedbackPromptActionInfo feedbackPromptActionInfo
}(persisted='true', hasPersonalData='true')
union FeedbackPromptActionInfo {
1: DidYouFindItSearch didYouFindItSearch
2: TweetRelevantToSearch tweetRelevantToSearch
}(persisted='true', hasPersonalData='true')
struct DidYouFindItSearch {
1: required string searchQuery(personalDataType= 'SearchQuery')
2: optional bool isRelevant
}(persisted='true', hasPersonalData='true')
struct TweetRelevantToSearch {
1: required string searchQuery(personalDataType= 'SearchQuery')
2: required i64 tweetId
3: optional bool isRelevant
}(persisted='true', hasPersonalData='true')
/*
* For (Tweet) Author info
*/
struct AuthorInfo {
/* In practice, this should be set. Rarely, it may be unset. */
1: optional i64 authorId(personalDataType = 'UserId')
/* i.e. in-network (true) or out-of-network (false) */
2: optional bool isFollowedByActingUser
/* i.e. is a follower (true) or not (false) */
3: optional bool isFollowingActingUser
}(persisted='true', hasPersonalData='true')
/*
* Use for Call to Action events.
*/
struct CTAInfo {
// this field is just a placeholder since Sparrow doesn't support empty struct
100: optional bool placeholder
}(persisted='true', hasPersonalData='false')
/*
* Card Info
*/
struct CardInfo {
1: optional i64 id
2: optional client_app.ItemType itemType
// authorId is deprecated, please use AuthorInfo instead
// 3: optional i64 authorId(personalDataType = 'UserId')
4: optional AuthorInfo actionTweetAuthorInfo
}(persisted='true', hasPersonalData='false')
/*
* When the user exits the app, the time (in millis) spent by them on the platform is recorded as User Active Seconds (UAS).
*/
struct UASInfo {
1: required i64 timeSpentMs
}(persisted='true', hasPersonalData='false')
/*
* Corresponding item for a user action.
* An item should be treated independently if it has different affordances
* (https://www.interaction-design.org/literature/topics/affordances) for the user.
* For example, a Notification has different affordances than a Tweet in the Notification Tab;
* in the former, you can either "click" or "see less often" and in the latter,
* you can perform inline engagements such as "like" or "reply".
* Note that an item may be rendered differently in different contexts, but as long as the
* affordances remain the same or nearly similar, it can be treated as the same item
* (e.g. Tweets can be rendered in slightly different ways in embeds vs in the app).
* Item types (e.g. Tweets, Notifications) and ActionTypes should be 1:1, and when an action can be
* performed on multiple types of items, consider granular action types.
* For example, a user can take the Click action on Tweets and Notifications, and we have
* separate ActionTypes for Tweet Click and Notification Click. This makes it easier to identify all the
* actions associated with a particular item.
*/
union Item {
1: TweetInfo tweetInfo
2: ProfileInfo profileInfo
3: TopicInfo topicInfo
4: NotificationInfo notificationInfo
5: TrendInfo trendInfo
6: CTAInfo ctaInfo
7: FeedbackPromptInfo feedbackPromptInfo
8: TypeaheadInfo typeaheadInfo
9: UASInfo uasInfo
10: CardInfo cardInfo
}(persisted='true', hasPersonalData='true')

BIN
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/keyed_uua.docx Normal file
View File

Binary file not shown.

22
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/keyed_uua.thrift
View File

@ -1,22 +0,0 @@
namespace java com.twitter.unified_user_actions.thriftjava
#@namespace scala com.twitter.unified_user_actions.thriftscala
#@namespace strato com.twitter.unified_user_actions
include "com/twitter/unified_user_actions/action_info.thrift"
include "com/twitter/unified_user_actions/common.thrift"
include "com/twitter/unified_user_actions/metadata.thrift"
/*
* This is mainly for View Counts project, which only require minimum fields for now.
* The name KeyedUuaTweet indicates the value is about a Tweet, not a Moment or other entities.
*/
struct KeyedUuaTweet {
/* A user refers to either a logged in / logged out user */
1: required common.UserIdentifier userIdentifier
/* The tweet that received the action from the user */
2: required i64 tweetId(personalDataType='TweetId')
/* The type of action which took place */
3: required action_info.ActionType actionType
/* Useful for event level analysis and joins */
4: required metadata.EventMetadata eventMetadata
}(persisted='true', hasPersonalData='true')

BIN
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/metadata.docx Normal file
View File

Binary file not shown.

177
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/metadata.thrift
View File

@ -1,177 +0,0 @@
namespace java com.twitter.unified_user_actions.thriftjava
#@namespace scala com.twitter.unified_user_actions.thriftscala
#@namespace strato com.twitter.unified_user_actions
/* Input source */
enum SourceLineage {
/* Client-side. Also known as legacy client events or LCE. */
ClientEvents = 0
/* Client-side. Also known as BCE. */
BehavioralClientEvents = 1
/* Server-side Timelineservice favorites */
ServerTlsFavs = 2
/* Server-side Tweetypie events */
ServerTweetypieEvents = 3
/* Server-side SocialGraph events */
ServerSocialGraphEvents = 4
/* Notification Actions responding to Your Highlights Emails */
EmailNotificationEvents = 5
/**
* Gizmoduck's User Modification events https://docbird.twitter.biz/gizmoduck/user_modifications.html
**/
ServerGizmoduckUserModificationEvents = 6
/**
* Server-side Ads callback engagements
**/
ServerAdsCallbackEngagements = 7
/**
* Server-side favorite archival events
**/
ServerFavoriteArchivalEvents = 8
/**
* Server-side retweet archival events
**/
ServerRetweetArchivalEvents = 9
}(persisted='true', hasPersonalData='false')
/*
* Only available in behavioral client events (BCE).
*
* A breadcrumb tweet is a tweet that was interacted with prior to the current action.
*/
struct BreadcrumbTweet {
/* Id for the tweet that was interacted with prior to the current action */
1: required i64 tweetId(personalDataType = 'TweetId')
/*
* The UI component that hosted the tweet and was interacted with preceeding to the current action.
* - tweet: represents the parent tweet container that wraps the quoted tweet
* - quote_tweet: represents the nested or quoted tweet within the parent container
*
* See more details
* https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.nb7tnjrhqxpm
*/
2: required string sourceComponent(personalDataType = 'WebsitePage')
}(persisted='true', hasPersonalData='true')
/*
* ClientEvent's namespaces. See https://docbird.twitter.biz/client_events/client-event-namespaces.html
*
* - For Legacy Client Events (LCE), it excludes the client part of the
* six part namespace (client:page:section:component:element:action)
* since this part is better captured by clientAppid and clientVersion.
*
* - For Behavioral Client Events (BCE), use clientPlatform to identify the client.
* Additionally, BCE contains an optional subsection to denote the UI component of
* the current action. The ClientEventNamespace.component field will be always empty for
* BCE namespace. There is no straightfoward 1-1 mapping between BCE and LCE namespace.
*/
struct ClientEventNamespace {
1: optional string page(personalDataType = 'AppUsage')
2: optional string section(personalDataType = 'AppUsage')
3: optional string component(personalDataType = 'AppUsage')
4: optional string element(personalDataType = 'AppUsage')
5: optional string action(personalDataType = 'AppUsage')
6: optional string subsection(personalDataType = 'AppUsage')
}(persisted='true', hasPersonalData='true')
/*
* Metadata that is independent of a particular (user, item, action type) tuple
* and mostly shared across user action events.
*/
struct EventMetadata {
/* When the action happened according to whatever source we are reading from */
1: required i64 sourceTimestampMs(personalDataType = 'PrivateTimestamp, PublicTimestamp')
/* When the action was received for processing internally
* (compare with sourceTimestampMs for delay)
*/
2: required i64 receivedTimestampMs
/* Which source is this event derived, e.g. CE, BCE, TimelineFavs */
3: required SourceLineage sourceLineage
/* To be deprecated and replaced by requestJoinId
* Useful for joining with other datasets
* */
4: optional i64 traceId(personalDataType = 'TfeTransactionId')
/*
* This is the language inferred from the request of the user action event (typically user's current client language)
* NOT the language of any Tweet,
* NOT the language that user sets in their profile!!!
*
* - ClientEvents && BehavioralClientEvents: Client UI language or from Gizmoduck which is what user set in Twitter App.
* Please see more at https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/finatra-internal/international/src/main/scala/com/twitter/finatra/international/LanguageIdentifier.scala
* The format should be ISO 639-1.
* - ServerTlsFavs: Client UI language, see more at http://go/languagepriority. The format should be ISO 639-1.
* - ServerTweetypieEvents: UUA sets this to None since there is no request level language info.
*/
5: optional string language(personalDataType = 'InferredLanguage')
/*
* This is the country inferred from the request of the user action event (typically user's current country code)
* NOT the country of any Tweet (by geo-tagging),
* NOT the country set by the user in their profile!!!
*
* - ClientEvents && BehavioralClientEvents: Country code could be IP address (geoduck) or
* User registration country (gizmoduck) and the former takes precedence.
* We don’t know exactly which one is applied, unfortunately,
* see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/finatra-internal/international/src/main/scala/com/twitter/finatra/international/CountryIdentifier.scala
* The format should be ISO_3166-1_alpha-2.
* - ServerTlsFavs: From the request (user’s current location),
* see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/thrift/com/twitter/context/viewer.thrift?L54
* The format should be ISO_3166-1_alpha-2.
* - ServerTweetypieEvents:
* UUA sets this to be consistent with IESource to meet existing use requirement.
* see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/thrift/com/twitter/tweetypie/tweet.thrift?L1001.
* The definitions here conflicts with the intention of UUA to log the request country code
* rather than the signup / geo-tagging country.
*/
6: optional string countryCode(personalDataType = 'InferredCountry')
/* Useful for debugging client application related issues */
7: optional i64 clientAppId(personalDataType = 'AppId')
/* Useful for debugging client application related issues */
8: optional string clientVersion(personalDataType = 'ClientVersion')
/* Useful for filtering */
9: optional ClientEventNamespace clientEventNamespace
/*
* This field is only populated in behavioral client events (BCE).
*
* The client platform such as one of ["iPhone", "iPad", "Mac", "Android", "Web"]
* There can be multiple clientAppIds for the same platform.
*/
10: optional string clientPlatform(personalDataType = 'ClientType')
/*
* This field is only populated in behavioral client events (BCE).
*
* The current UI hierarchy information with human readable labels.
* For example, [home,timeline,tweet] or [tab_bar,home,scrollable_content,tweet]
*
* For more details see https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.uv3md49i0j4j
*/
11: optional list<string> viewHierarchy(personalDataType = 'WebsitePage')
/*
* This field is only populated in behavioral client events (BCE).
*
* The sequence of views (breadcrumb) that was interacted with that caused the user to navigate to
* the current product surface (e.g. profile page) where an action was taken.
*
* The breadcrumb information may only be present for certain preceding product surfaces (e.g. Home Timeline).
* See more details in https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.nb7tnjrhqxpm
*/
12: optional list<string> breadcrumbViews(personalDataType = 'WebsitePage')
/*
* This field is only populated in behavioral client events (BCE).
*
* The sequence of tweets (breadcrumb) that was interacted with that caused the user to navigate to
* current product surface (e.g. profile page) where an action was taken.
*
* The breadcrumb information may only be present for certain preceding product surfaces (e.g. Home Timeline).
* See more details in https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.nb7tnjrhqxpm
*/
13: optional list<BreadcrumbTweet> breadcrumbTweets(personalDataType = 'TweetId')
/*
* A request join id is created by backend services and broadcasted in subsequent calls
* to other downstream services as part of the request path. The requestJoinId is logged
* in server logs and scribed in client events, enabling joins across client and server
* as well as within a given request across backend servers. See go/joinkey-tdd for more
* details.
*/
14: optional i64 requestJoinId(personalDataType = 'TransactionId')
15: optional i64 clientEventTriggeredOn
}(persisted='true', hasPersonalData='true')

BIN
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/product_surface_info.docx Normal file
View File

Binary file not shown.

149
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/product_surface_info.thrift
View File

@ -1,149 +0,0 @@
#@namespace java com.twitter.unified_user_actions.thriftjava
#@namespace scala com.twitter.unified_user_actions.thriftscala
#@namespace strato com.twitter.unified_user_actions
include "com/twitter/unified_user_actions/metadata.thrift"
include "com/twitter/search/common/constants/query.thrift"
include "com/twitter/search/common/constants/result.thrift"
/*
* Represents the product surface on which an action took place.
* See reference that delineates various product surfaces:
* https://docs.google.com/document/d/1PS2ZOyNUoUdO45zxhE7dH3L8KUcqJwo6Vx-XUGGFo6U
* Note: the implementation here may not reflect the above doc exactly.
*/
enum ProductSurface {
// 1 - 19 for Home
HomeTimeline = 1
// 20 - 39 for Notifications
NotificationTab = 20
PushNotification = 21
EmailNotification = 22
// 40 - 59 for Search
SearchResultsPage = 40
SearchTypeahead = 41
// 60 - 79 for Tweet Details Page (Conversation Page)
TweetDetailsPage = 60
// 80 - 99 for Profile Page
ProfilePage = 80
// 100 - 119 for ?
RESERVED_100 = 100
// 120 - 139 for ?
RESERVED_120 = 120
}(persisted='true', hasPersonalData='false')
union ProductSurfaceInfo {
// 1 matches the enum index HomeTimeline in ProductSurface
1: HomeTimelineInfo homeTimelineInfo
// 20 matches the enum index NotificationTab in ProductSurface
20: NotificationTabInfo notificationTabInfo
// 21 matches the enum index PushNotification in ProductSurface
21: PushNotificationInfo pushNotificationInfo
// 22 matches the enum index EmailNotification in ProductSurface
22: EmailNotificationInfo emailNotificationInfo
// 40 matches the enum index SearchResultPage in ProductSurface
40: SearchResultsPageInfo searchResultsPageInfo
// 41 matches the enum index SearchTypeahead in ProductSurface
41: SearchTypeaheadInfo searchTypeaheadInfo
// 60 matches the enum index TweetDetailsPage in ProductSurface
60: TweetDetailsPageInfo tweetDetailsPageInfo
// 80 matches the enum index ProfilePage in ProductSurface
80: ProfilePageInfo profilePageInfo
}(persisted='true', hasPersonalData='false')
/*
* Please keep this minimal to avoid overhead. It should only
* contain high value Home Timeline specific attributes.
*/
struct HomeTimelineInfo {
// suggestType is deprecated, please do't re-use!
// 1: optional i32 suggestType
2: optional string suggestionType
3: optional i32 injectedPosition
}(persisted='true', hasPersonalData='false')
struct NotificationTabInfo {
/*
* Note that this field represents the `impressionId` in a Notification Tab notification.
* It has been renamed to `notificationId` in UUA so that the name effectively represents the
* value it holds, i.e., a unique id for a notification and request.
*/
1: required string notificationId(personalDataType='UniversallyUniqueIdentifierUuid')
}(persisted='true', hasPersonalData='false')
struct PushNotificationInfo {
/*
* Note that this field represents the `impressionId` in a Push Notification.
* It has been renamed to `notificationId` in UUA so that the name effectively represents the
* value it holds, i.e., a unique id for a notification and request.
*/
1: required string notificationId(personalDataType='UniversallyUniqueIdentifierUuid')
}(persisted='true', hasPersonalData='false')
struct EmailNotificationInfo {
/*
* Note that this field represents the `impressionId` in an Email Notification.
* It has been renamed to `notificationId` in UUA so that the name effectively represents the
* value it holds, i.e., a unique id for a notification and request.
*/
1: required string notificationId(personalDataType='UniversallyUniqueIdentifierUuid')
}(persisted='true', hasPersonalData='false')
struct TweetDetailsPageInfo {
// To be deprecated, please don't re-use!
// Only reason to keep it now is Sparrow doesn't take empty struct. Once there is a real
// field we should just comment it out.
1: required list<string> breadcrumbViews(personalDataType = 'WebsitePage')
// Deprecated, please don't re-use!
// 2: required list<metadata.BreadcrumbTweet> breadcrumbTweets(personalDataType = 'TweetId')
}(persisted='true', hasPersonalData='true')
struct ProfilePageInfo {
// To be deprecated, please don't re-use!
// Only reason to keep it now is Sparrow doesn't take empty struct. Once there is a real
// field we should just comment it out.
1: required list<string> breadcrumbViews(personalDataType = 'WebsitePage')
// Deprecated, please don't re-use!
// 2: required list<metadata.BreadcrumbTweet> breadcrumbTweets(personalDataType = 'TweetId')
}(persisted='true', hasPersonalData='true')
struct SearchResultsPageInfo {
// search query string
1: required string query(personalDataType = 'SearchQuery')
// Attribution of the search (e.g. Typed Query, Hashtag Click, etc.)
// see http://go/sgb/src/thrift/com/twitter/search/common/constants/query.thrift for details
2: optional query.ThriftQuerySource querySource
// 0-indexed position of item in list of search results
3: optional i32 itemPosition
// Attribution of the tweet result (e.g. QIG, Earlybird, etc)
// see http://go/sgb/src/thrift/com/twitter/search/common/constants/result.thrift for details
4: optional set<result.TweetResultSource> tweetResultSources
// Attribution of the user result (e.g. ExpertSearch, QIG, etc)
// see http://go/sgb/src/thrift/com/twitter/search/common/constants/result.thrift for details
5: optional set<result.UserResultSource> userResultSources
// The query filter type on the Search Results Page (SRP) when the action took place.
// Clicking on a tab in SRP applies a query filter automatically.
6: optional SearchQueryFilterType queryFilterType
}(persisted='true', hasPersonalData='true')
struct SearchTypeaheadInfo {
// search query string
1: required string query(personalDataType = 'SearchQuery')
// 0-indexed position of item in list of typeahead drop-down
2: optional i32 itemPosition
}(persisted='true', hasPersonalData='true')
enum SearchQueryFilterType {
// filter to top ranked content for a query
TOP = 1
// filter to latest content for a query
LATEST = 2
// filter to user results for a query
PEOPLE = 3
// filter to photo tweet results for a query
PHOTOS = 4
// filter to video tweet results for a query
VIDEOS = 5
}

BIN
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/unified_user_actions.docx Normal file
View File

Binary file not shown.

37
unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/unified_user_actions.thrift
View File

@ -1,37 +0,0 @@
namespace java com.twitter.unified_user_actions.thriftjava
#@namespace scala com.twitter.unified_user_actions.thriftscala
#@namespace strato com.twitter.unified_user_actions
include "com/twitter/unified_user_actions/action_info.thrift"
include "com/twitter/unified_user_actions/common.thrift"
include "com/twitter/unified_user_actions/item.thrift"
include "com/twitter/unified_user_actions/metadata.thrift"
include "com/twitter/unified_user_actions/product_surface_info.thrift"
/*
* A Unified User Action (UUA) is essentially a tuple of
* (user, item, action type, some metadata) with more optional
* information unique to product surfaces when available.
* It represents a user (logged in / out) taking some action (e.g. engagement,
* impression) on an item (e.g. tweet, profile).
*/
struct UnifiedUserAction {
/* A user refers to either a logged in / logged out user */
1: required common.UserIdentifier userIdentifier
/* The item that received the action from the user */
2: required item.Item item
/* The type of action which took place */
3: required action_info.ActionType actionType
/* Useful for event level analysis and joins */
4: required metadata.EventMetadata eventMetadata
/*
* Product surface on which the action occurred. If None,
* it means we can not capture the product surface (e.g. for server-side events).
*/
5: optional product_surface_info.ProductSurface productSurface
/*
* Product specific information like join keys. If None,
* it means we can not capture the product surface information.
*/
6: optional product_surface_info.ProductSurfaceInfo productSurfaceInfo
}(persisted='true', hasPersonalData='true')

15
unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/BUILD.bazel
View File

@ -1,15 +0,0 @@
create_thrift_libraries(
org = "com.twitter",
base_name = "unified_user_actions_spec",
sources = ["*.thrift"],
tags = ["bazel-compatible"],
dependency_roots = [
],
generate_languages = [
"java",
"scala",
"strato",
],
provides_java_name = "unified_user_actions_spec-thrift-java",
provides_scala_name = "unified_user_actions_spec-thrift-scala",
)

BIN
unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/BUILD.docx Normal file
View File

Binary file not shown.

BIN
unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/unified_user_actions.docx Normal file
View File

Binary file not shown.

11
unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/unified_user_actions.thrift
View File

@ -1,11 +0,0 @@
namespace java com.twitter.unified_user_actions.thriftjava
#@namespace scala com.twitter.unified_user_actions.thriftscala
#@namespace strato com.twitter.unified_user_actions
/* Useful for testing UnifiedUserAction-like schema in tests */
struct UnifiedUserActionSpec {
/* A user refers to either a logged out / logged in user */
1: required i64 userId
/* Arbitrary payload */
2: optional string payload
}(hasPersonalData='false')

BIN
user-signal-service/README.docx Normal file
View File

Binary file not shown.

5
user-signal-service/README.md
View File

@ -1,5 +0,0 @@
# User Signal Service #
**User Signal Service** (USS) is a centralized online platform that supplies comprehensive data on user actions and behaviors on Twitter. This information encompasses both explicit signals, such as favoriting, retweeting, and replying, as well as implicit signals, including tweet clicks, video views, profile visits, and more.
To ensure consistency and accuracy, USS gathers these signals from various underlying datasets and online services, processing them into uniform formats. These standardized source signals are then utilized in candidate retrieval and machine learning features for ranking stages.

21
user-signal-service/server/BUILD
View File

@ -1,21 +0,0 @@
jvm_binary(
name = "bin",
basename = "user-signal-service",
main = "com.twitter.usersignalservice.UserSignalServiceStratoFedServerMain",
runtime_platform = "java11",
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/ch/qos/logback:logback-classic",
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
"strato/src/main/scala/com/twitter/strato/logging/logback",
"user-signal-service/server/src/main/resources",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice",
],
)
# Aurora Workflows build phase convention requires a jvm_app named with ${project-name}-app
jvm_app(
name = "user-signal-service-app",
archive = "zip",
binary = ":bin",
)

BIN
user-signal-service/server/BUILD.docx Normal file
View File

Binary file not shown.

7
user-signal-service/server/src/main/resources/BUILD
View File

@ -1,7 +0,0 @@
resources(
sources = [
"*.xml",
"*.yml",
"config/*.yml",
],
)

BIN
user-signal-service/server/src/main/resources/BUILD.docx Normal file
View File

Binary file not shown.

BIN
user-signal-service/server/src/main/resources/config/decider.docx Normal file
View File

Binary file not shown.

6
user-signal-service/server/src/main/resources/config/decider.yml
View File

@ -1,6 +0,0 @@
test_value:
comment: Test Value
default_availability: 10000
dark_traffic_percent:
comment: Percentage of traffic to send to dark traffic destination
default_availability: 0

BIN
user-signal-service/server/src/main/resources/logback.docx Normal file
View File

Binary file not shown.

155
user-signal-service/server/src/main/resources/logback.xml
View File

@ -1,155 +0,0 @@
<configuration>
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
<property name="async_queue_size" value="${queue.size:-50000}"/>
<property name="async_max_flush_time" value="${max.flush.time:-0}"/>
<!-- ===================================================== -->
<!-- Structured Logging -->
<!-- ===================================================== -->
<!-- Only sample 0.1% of the requests -->
<property name="splunk_sampling_rate" value="${splunk_sampling_rate:-0.001}"/>
<include resource="structured-logger-logback.xml"/>
<!-- ===================================================== -->
<!-- Service Config -->
<!-- ===================================================== -->
<property name="DEFAULT_SERVICE_PATTERN"
value="%-16X{transactionId} %logger %msg"/>
<!-- ===================================================== -->
<!-- Common Config -->
<!-- ===================================================== -->
<!-- JUL/JDK14 to Logback bridge -->
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
<!-- Service Log (Rollover every 50MB, max 11 logs) -->
<appender name="SERVICE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.service.output}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>${log.service.output}.%i</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>10</maxIndex>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>50MB</maxFileSize>
</triggeringPolicy>
<encoder>
<pattern>%date %.-3level ${DEFAULT_SERVICE_PATTERN}%n</pattern>
</encoder>
</appender>
<!-- Strato package only log (Rollover every 50MB, max 11 logs) -->
<appender name="STRATO-ONLY" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.strato_only.output}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>${log.strato_only.output}.%i</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>10</maxIndex>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>50MB</maxFileSize>
</triggeringPolicy>
<encoder>
<pattern>%date %.-3level ${DEFAULT_SERVICE_PATTERN}%n</pattern>
</encoder>
</appender>
<!-- LogLens -->
<appender name="LOGLENS" class="com.twitter.loglens.logback.LoglensAppender">
<mdcAdditionalContext>true</mdcAdditionalContext>
<category>loglens</category>
<index>${log.lens.index}</index>
<tag>${log.lens.tag}/service</tag>
<encoder>
<pattern>%msg%n</pattern>
</encoder>
<turboFilter class="ch.qos.logback.classic.turbo.DuplicateMessageFilter">
<cacheSize>500</cacheSize>
<allowedRepetitions>50</allowedRepetitions>
</turboFilter>
<filter class="com.twitter.strato.logging.logback.RegexFilter">
<forLogger>manhattan-client</forLogger>
<excludeRegex>.*InvalidRequest.*</excludeRegex>
</filter>
</appender>
<!-- ===================================================== -->
<!-- Primary Async Appenders -->
<!-- ===================================================== -->
<appender name="ASYNC-SERVICE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>${async_queue_size}</queueSize>
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
<appender-ref ref="SERVICE"/>
</appender>
<appender name="ASYNC-STRATO-ONLY" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>${async_queue_size}</queueSize>
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
<appender-ref ref="STRATO-ONLY"/>
</appender>
<appender name="ASYNC-LOGLENS" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>${async_queue_size}</queueSize>
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
<appender-ref ref="LOGLENS"/>
</appender>
<!-- ===================================================== -->
<!-- Package Config -->
<!-- ===================================================== -->
<!-- Per-Package Config (shared) -->
<logger name="com.twitter" level="info"/>
<!--
By default, we leave the strato package at INFO level.
However, this line allows us to set the entire strato package, or a subset of it, to
a specific level. For example, if you pass -Dstrato_log_package=streaming -Dstrato_log_level=DEBUG
only loggers under com.twitter.strato.streaming.* will be set to DEBUG level. Passing only
-Dstrato_log_level will set all of strato.* to the specified level.
-->
<logger name="com.twitter.strato${strato_log_package:-}" level="${strato_log_level:-INFO}"/>
<logger name="com.twitter.wilyns" level="warn"/>
<logger name="com.twitter.finagle.mux" level="warn"/>
<logger name="com.twitter.finagle.serverset2" level="warn"/>
<logger name="com.twitter.logging.ScribeHandler" level="warn"/>
<logger name="com.twitter.zookeeper.client.internal" level="warn"/>
<logger name="com.twitter.decider.StoreDecider" level="warn"/>
<!-- Per-Package Config (Strato) -->
<logger name="com.twitter.distributedlog.client" level="warn"/>
<logger name="com.twitter.finagle.mtls.authorization.config.AccessControlListConfiguration" level="warn"/>
<logger name="com.twitter.finatra.kafka.common.kerberoshelpers" level="warn"/>
<logger name="com.twitter.finatra.kafka.utils.BootstrapServerUtils" level="warn"/>
<logger name="com.twitter.server.coordinate" level="error"/>
<logger name="com.twitter.zookeeper.client" level="info"/>
<logger name="org.apache.zookeeper" level="error"/>
<logger name="org.apache.zookeeper.ClientCnxn" level="warn"/>
<logger name="ZkSession" level="info"/>
<logger name="OptimisticLockingCache" level="off"/>
<logger name="manhattan-client" level="warn"/>
<logger name="strato.op" level="warn"/>
<logger name="org.apache.kafka.clients.NetworkClient" level="error"/>
<logger name="org.apache.kafka.clients.consumer.internals" level="error"/>
<logger name="org.apache.kafka.clients.producer.internals" level="error"/>
<!-- produce a lot of messages like: Building client authenticator with server name kafka -->
<logger name="org.apache.kafka.common.network" level="warn"/>
<!-- Root Config -->
<root level="${log_level:-INFO}">
<appender-ref ref="ASYNC-SERVICE"/>
<appender-ref ref="ASYNC-LOGLENS"/>
</root>
<!-- Strato package only logging-->
<logger name="com.twitter.strato"
level="info"
additivity="true">
<appender-ref ref="ASYNC-STRATO-ONLY" />
</logger>
</configuration>

9
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/BUILD
View File

@ -1,9 +0,0 @@
scala_library(
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config",
],
)

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/BUILD.docx Normal file
View File

Binary file not shown.

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/UserSignalServiceStratoFedServerMain.docx Normal file
View File

Binary file not shown.

32
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/UserSignalServiceStratoFedServerMain.scala
View File

@ -1,32 +0,0 @@
package com.twitter.usersignalservice
import com.google.inject.Module
import com.twitter.inject.thrift.modules.ThriftClientIdModule
import com.twitter.usersignalservice.columns.UserSignalServiceColumn
import com.twitter.strato.fed._
import com.twitter.strato.fed.server._
import com.twitter.usersignalservice.module.CacheModule
import com.twitter.usersignalservice.module.MHMtlsParamsModule
import com.twitter.usersignalservice.module.SocialGraphServiceClientModule
import com.twitter.usersignalservice.module.TimerModule
object UserSignalServiceStratoFedServerMain extends UserSignalServiceStratoFedServer
trait UserSignalServiceStratoFedServer extends StratoFedServer {
override def dest: String = "/s/user-signal-service/user-signal-service"
override def columns: Seq[Class[_ <: StratoFed.Column]] =
Seq(
classOf[UserSignalServiceColumn]
)
override def modules: Seq[Module] =
Seq(
CacheModule,
MHMtlsParamsModule,
SocialGraphServiceClientModule,
ThriftClientIdModule,
TimerModule,
)
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/AggregatedSignalController.docx Normal file
View File

Binary file not shown.

58
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/AggregatedSignalController.scala
View File

@ -1,58 +0,0 @@
package com.twitter.usersignalservice.base
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.frigate.common.base.Stats
import com.twitter.storehaus.ReadableStore
import com.twitter.twistly.common.UserId
import com.twitter.usersignalservice.base.BaseSignalFetcher.Timeout
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
case class AggregatedSignalController(
signalsAggregationInfo: Seq[SignalAggregatedInfo],
signalsWeightMapInfo: Map[SignalType, Double],
stats: StatsReceiver,
timer: Timer)
extends ReadableStore[Query, Seq[Signal]] {
val name: String = this.getClass.getCanonicalName
val statsReceiver: StatsReceiver = stats.scope(name)
override def get(query: Query): Future[Option[Seq[Signal]]] = {
Stats
.trackItems(statsReceiver) {
val allSignalsFut =
Future
.collect(signalsAggregationInfo.map(_.getSignals(query.userId))).map(_.flatten.flatten)
val aggregatedSignals =
allSignalsFut.map { allSignals =>
allSignals
.groupBy(_.targetInternalId).collect {
case (Some(internalId), signals) =>
val mostRecentEnagementTime = signals.map(_.timestamp).max
val totalWeight =
signals
.map(signal => signalsWeightMapInfo.getOrElse(signal.signalType, 0.0)).sum
(Signal(query.signalType, mostRecentEnagementTime, Some(internalId)), totalWeight)
}.toSeq.sortBy { case (signal, weight) => (-weight, -signal.timestamp) }
.map(_._1)
.take(query.maxResults.getOrElse(Int.MaxValue))
}
aggregatedSignals.map(Some(_))
}.raiseWithin(Timeout)(timer).handle {
case e =>
statsReceiver.counter(e.getClass.getCanonicalName).incr()
Some(Seq.empty[Signal])
}
}
}
case class SignalAggregatedInfo(
signalType: SignalType,
signalFetcher: ReadableStore[Query, Seq[Signal]]) {
def getSignals(userId: UserId): Future[Option[Seq[Signal]]] = {
signalFetcher.get(Query(userId, signalType, None))
}
}

16
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BUILD
View File

@ -1,16 +0,0 @@
scala_library(
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/src/jvm/com/twitter/storehaus:core",
"finagle/finagle-stats",
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store",
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato",
"hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common",
"relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection",
"src/scala/com/twitter/storehaus_internal/manhattan",
"src/scala/com/twitter/twistly/common",
"src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala",
"user-signal-service/thrift/src/main/thrift:thrift-scala",
],
)

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BUILD.docx Normal file
View File

Binary file not shown.

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BaseSignalFetcher.docx Normal file
View File

Binary file not shown.

90
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BaseSignalFetcher.scala
View File

@ -1,90 +0,0 @@
package com.twitter.usersignalservice
package base
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.storehaus.ReadableStore
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.util.Future
import com.twitter.twistly.common.UserId
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.frigate.common.base.Stats
import com.twitter.conversions.DurationOps._
import com.twitter.usersignalservice.thriftscala.ClientIdentifier
import com.twitter.util.Duration
import com.twitter.util.Timer
import java.io.Serializable
case class Query(
userId: UserId,
signalType: SignalType,
maxResults: Option[Int],
clientId: ClientIdentifier = ClientIdentifier.Unknown)
/**
* A trait that defines a standard interface for the signal fetcher
*
* Extends this only when all other traits extending BaseSignalFetcher do not apply to
* your use case.
*/
trait BaseSignalFetcher extends ReadableStore[Query, Seq[Signal]] {
import BaseSignalFetcher._
/**
* This RawSignalType is the output type of `getRawSignals` and the input type of `process`.
* Override it as your own raw signal type to maintain meta data which can be used in the
* step of `process`.
* Note that the RawSignalType is an intermediate data type intended to be small to avoid
* big data chunks being passed over functions or being memcached.
*/
type RawSignalType <: Serializable
def name: String
def statsReceiver: StatsReceiver
def timer: Timer
/**
* This function is called by the top level class to fetch signals. It executes the pipeline to
* fetch raw signals, process and transform the signals. Exceptions and timeout control are
* handled here.
* @param query
* @return Future[Option[Seq[Signal]]]
*/
override def get(query: Query): Future[Option[Seq[Signal]]] = {
val clientStatsReceiver = statsReceiver.scope(query.clientId.name).scope(query.signalType.name)
Stats
.trackItems(clientStatsReceiver) {
val rawSignals = getRawSignals(query.userId)
val signals = process(query, rawSignals)
signals
}.raiseWithin(Timeout)(timer).handle {
case e =>
clientStatsReceiver.scope("FetcherExceptions").counter(e.getClass.getCanonicalName).incr()
EmptyResponse
}
}
/**
* Override this function to define how to fetch the raw signals from any store
* Note that the RawSignalType is an intermediate data type intended to be small to avoid
* big data chunks being passed over functions or being memcached.
* @param userId
* @return Future[Option[Seq[RawSignalType]]]
*/
def getRawSignals(userId: UserId): Future[Option[Seq[RawSignalType]]]
/**
* Override this function to define how to process the raw signals and transform them to signals.
* @param query
* @param rawSignals
* @return Future[Option[Seq[Signal]]]
*/
def process(
query: Query,
rawSignals: Future[Option[Seq[RawSignalType]]]
): Future[Option[Seq[Signal]]]
}
object BaseSignalFetcher {
val Timeout: Duration = 20.milliseconds
val EmptyResponse: Option[Seq[Signal]] = Some(Seq.empty[Signal])
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/FilteredSignalFetcherController.docx Normal file
View File

Binary file not shown.

75
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/FilteredSignalFetcherController.scala
View File

@ -1,75 +0,0 @@
package com.twitter.usersignalservice.base
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.frigate.common.base.Stats
import com.twitter.storehaus.ReadableStore
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
/**
* Combine a BaseSignalFetcher with a map of negative signalFetchers. Filter out the negative
* signals from the signals from BaseSignalFetcher.
*/
case class FilteredSignalFetcherController(
backingSignalFetcher: BaseSignalFetcher,
originSignalType: SignalType,
stats: StatsReceiver,
timer: Timer,
filterSignalFetchers: Map[SignalType, BaseSignalFetcher] =
Map.empty[SignalType, BaseSignalFetcher])
extends ReadableStore[Query, Seq[Signal]] {
val statsReceiver: StatsReceiver = stats.scope(this.getClass.getCanonicalName)
override def get(query: Query): Future[Option[Seq[Signal]]] = {
val clientStatsReceiver = statsReceiver.scope(query.signalType.name).scope(query.clientId.name)
Stats
.trackItems(clientStatsReceiver) {
val backingSignals =
backingSignalFetcher.get(Query(query.userId, originSignalType, None, query.clientId))
val filteredSignals = filter(query, backingSignals)
filteredSignals
}.raiseWithin(BaseSignalFetcher.Timeout)(timer).handle {
case e =>
clientStatsReceiver.scope("FetcherExceptions").counter(e.getClass.getCanonicalName).incr()
BaseSignalFetcher.EmptyResponse
}
}
def filter(
query: Query,
rawSignals: Future[Option[Seq[Signal]]]
): Future[Option[Seq[Signal]]] = {
Stats
.trackItems(statsReceiver) {
val originSignals = rawSignals.map(_.getOrElse(Seq.empty[Signal]))
val filterSignals =
Future
.collect {
filterSignalFetchers.map {
case (signalType, signalFetcher) =>
signalFetcher
.get(Query(query.userId, signalType, None, query.clientId))
.map(_.getOrElse(Seq.empty))
}.toSeq
}.map(_.flatten.toSet)
val filterSignalsSet = filterSignals
.map(_.flatMap(_.targetInternalId))
val originSignalsWithId =
originSignals.map(_.map(signal => (signal, signal.targetInternalId)))
Future.join(originSignalsWithId, filterSignalsSet).map {
case (originSignalsWithId, filterSignalsSet) =>
Some(
originSignalsWithId
.collect {
case (signal, internalIdOpt)
if internalIdOpt.nonEmpty && !filterSignalsSet.contains(internalIdOpt.get) =>
signal
}.take(query.maxResults.getOrElse(Int.MaxValue)))
}
}
}
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/ManhattanSignalFetcher.docx Normal file
View File

Binary file not shown.

66
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/ManhattanSignalFetcher.scala
View File

@ -1,66 +0,0 @@
package com.twitter.usersignalservice
package base
import com.twitter.bijection.Codec
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
import com.twitter.storehaus.ReadableStore
import com.twitter.storehaus_internal.manhattan.ManhattanCluster
import com.twitter.storehaus_internal.manhattan.ManhattanRO
import com.twitter.storehaus_internal.manhattan.ManhattanROConfig
import com.twitter.storehaus_internal.util.HDFSPath
import com.twitter.twistly.common.UserId
import com.twitter.util.Future
import com.twitter.storehaus_internal.util.ApplicationID
import com.twitter.storehaus_internal.util.DatasetName
/**
* A Manhattan signal fetcher extending BaseSignalFetcher to provide an interface to fetch signals
* from a Manhattan dataset.
*
* Extends this when the underlying store is a single Manhattan dataset.
* @tparam ManhattanKeyType
* @tparam ManhattanValueType
*/
trait ManhattanSignalFetcher[ManhattanKeyType, ManhattanValueType] extends BaseSignalFetcher {
/*
Define the meta info of the Manhattan dataset
*/
protected def manhattanAppId: String
protected def manhattanDatasetName: String
protected def manhattanClusterId: ManhattanCluster
protected def manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams
protected def manhattanKeyCodec: Codec[ManhattanKeyType]
protected def manhattanRawSignalCodec: Codec[ManhattanValueType]
/**
* Adaptor to transform the userId to the ManhattanKey
* @param userId
* @return ManhattanKeyType
*/
protected def toManhattanKey(userId: UserId): ManhattanKeyType
/**
* Adaptor to transform the ManhattanValue to the Seq of RawSignalType
* @param manhattanValue
* @return Seq[RawSignalType]
*/
protected def toRawSignals(manhattanValue: ManhattanValueType): Seq[RawSignalType]
protected final lazy val underlyingStore: ReadableStore[UserId, Seq[RawSignalType]] = {
ManhattanRO
.getReadableStoreWithMtls[ManhattanKeyType, ManhattanValueType](
ManhattanROConfig(
HDFSPath(""),
ApplicationID(manhattanAppId),
DatasetName(manhattanDatasetName),
manhattanClusterId),
manhattanKVClientMtlsParams
)(manhattanKeyCodec, manhattanRawSignalCodec)
.composeKeyMapping(userId => toManhattanKey(userId))
.mapValues(manhattanRawSignal => toRawSignals(manhattanRawSignal))
}
override final def getRawSignals(userId: UserId): Future[Option[Seq[RawSignalType]]] =
underlyingStore.get(userId)
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/MemcachedSignalFetcherWrapper.docx Normal file
View File

Binary file not shown.

70
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/MemcachedSignalFetcherWrapper.scala
View File

@ -1,70 +0,0 @@
package com.twitter.usersignalservice
package base
import com.twitter.finagle.memcached.{Client => MemcachedClient}
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.hashing.KeyHasher
import com.twitter.hermit.store.common.ObservedMemcachedReadableStore
import com.twitter.relevance_platform.common.injection.LZ4Injection
import com.twitter.relevance_platform.common.injection.SeqObjectInjection
import com.twitter.storehaus.ReadableStore
import com.twitter.twistly.common.UserId
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.util.Duration
import com.twitter.util.Future
import com.twitter.util.Timer
/**
* Use this wrapper when the latency of the signal fetcher is too high (see BaseSignalFetcher.Timeout
* ) and the results from the signal fetcher don't change often (e.g. results are generated from a
* scalding job scheduled each day).
* @param memcachedClient
* @param baseSignalFetcher
* @param ttl
* @param stats
* @param timer
*/
case class MemcachedSignalFetcherWrapper(
memcachedClient: MemcachedClient,
baseSignalFetcher: BaseSignalFetcher,
ttl: Duration,
stats: StatsReceiver,
keyPrefix: String,
timer: Timer)
extends BaseSignalFetcher {
import MemcachedSignalFetcherWrapper._
override type RawSignalType = baseSignalFetcher.RawSignalType
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(name).scope(baseSignalFetcher.name)
val underlyingStore: ReadableStore[UserId, Seq[RawSignalType]] = {
val cacheUnderlyingStore = new ReadableStore[UserId, Seq[RawSignalType]] {
override def get(userId: UserId): Future[Option[Seq[RawSignalType]]] =
baseSignalFetcher.getRawSignals(userId)
}
ObservedMemcachedReadableStore.fromCacheClient(
backingStore = cacheUnderlyingStore,
cacheClient = memcachedClient,
ttl = ttl)(
valueInjection = LZ4Injection.compose(SeqObjectInjection[RawSignalType]()),
statsReceiver = statsReceiver,
keyToString = { k: UserId =>
s"$keyPrefix:${keyHasher.hashKey(k.toString.getBytes)}"
}
)
}
override def getRawSignals(userId: UserId): Future[Option[Seq[RawSignalType]]] =
underlyingStore.get(userId)
override def process(
query: Query,
rawSignals: Future[Option[Seq[RawSignalType]]]
): Future[Option[Seq[Signal]]] = baseSignalFetcher.process(query, rawSignals)
}
object MemcachedSignalFetcherWrapper {
private val keyHasher: KeyHasher = KeyHasher.FNV1A_64
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/StratoSignalFetcher.docx Normal file
View File

Binary file not shown.

61
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/StratoSignalFetcher.scala
View File

@ -1,61 +0,0 @@
package com.twitter.usersignalservice
package base
import com.twitter.frigate.common.store.strato.StratoFetchableStore
import com.twitter.storehaus.ReadableStore
import com.twitter.strato.client.Client
import com.twitter.strato.data.Conv
import com.twitter.twistly.common.UserId
import com.twitter.util.Future
/**
* A Strato signal fetcher extending BaseSignalFetcher to provide an interface to fetch signals from
* Strato Column.
*
* Extends this when the underlying store is a single Strato column.
* @tparam StratoKeyType
* @tparam StratoViewType
* @tparam StratoValueType
*/
trait StratoSignalFetcher[StratoKeyType, StratoViewType, StratoValueType]
extends BaseSignalFetcher {
/*
Define the meta info of the strato column
*/
def stratoClient: Client
def stratoColumnPath: String
def stratoView: StratoViewType
/**
* Override these vals and remove the implicit key words.
* @return
*/
protected implicit def keyConv: Conv[StratoKeyType]
protected implicit def viewConv: Conv[StratoViewType]
protected implicit def valueConv: Conv[StratoValueType]
/**
* Adapter to transform the userId to the StratoKeyType
* @param userId
* @return StratoKeyType
*/
protected def toStratoKey(userId: UserId): StratoKeyType
/**
* Adapter to transform the StratoValueType to a Seq of RawSignalType
* @param stratoValue
* @return Seq[RawSignalType]
*/
protected def toRawSignals(stratoValue: StratoValueType): Seq[RawSignalType]
protected final lazy val underlyingStore: ReadableStore[UserId, Seq[RawSignalType]] =
StratoFetchableStore
.withView[StratoKeyType, StratoViewType, StratoValueType](
stratoClient,
stratoColumnPath,
stratoView)
.composeKeyMapping(toStratoKey)
.mapValues(toRawSignals)
override final def getRawSignals(userId: UserId): Future[Option[Seq[RawSignalType]]] =
underlyingStore.get(userId)
}

11
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/BUILD
View File

@ -1,11 +0,0 @@
scala_library(
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"src/scala/com/twitter/twistly/common",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service",
"user-signal-service/thrift/src/main/thrift:thrift-scala",
],
)

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/BUILD.docx Normal file
View File

Binary file not shown.

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/UserSignalServiceColumn.docx Normal file
View File

Binary file not shown.

49
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/UserSignalServiceColumn.scala
View File

@ -1,49 +0,0 @@
package com.twitter.usersignalservice.columns
import com.twitter.stitch.NotFound
import com.twitter.stitch.Stitch
import com.twitter.strato.catalog.OpMetadata
import com.twitter.strato.catalog.Ops
import com.twitter.strato.config.Policy
import com.twitter.strato.config.ReadWritePolicy
import com.twitter.strato.data.Conv
import com.twitter.strato.data.Description
import com.twitter.strato.data.Lifecycle
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.usersignalservice.service.UserSignalService
import com.twitter.usersignalservice.thriftscala.BatchSignalRequest
import com.twitter.usersignalservice.thriftscala.BatchSignalResponse
import javax.inject.Inject
class UserSignalServiceColumn @Inject() (userSignalService: UserSignalService)
extends StratoFed.Column(UserSignalServiceColumn.Path)
with StratoFed.Fetch.Stitch {
override val metadata: OpMetadata = OpMetadata(
lifecycle = Some(Lifecycle.Production),
description = Some(Description.PlainText("User Signal Service Federated Column")))
override def ops: Ops = super.ops
override type Key = BatchSignalRequest
override type View = Unit
override type Value = BatchSignalResponse
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[BatchSignalRequest]
override val viewConv: Conv[View] = Conv.ofType
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[BatchSignalResponse]
override def fetch(key: Key, view: View): Stitch[Result[Value]] = {
userSignalService
.userSignalServiceHandlerStoreStitch(key)
.map(result => found(result))
.handle {
case NotFound => missing
}
}
}
object UserSignalServiceColumn {
val Path = "recommendations/user-signal-service/signals"
}

9
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/BUILD
View File

@ -1,9 +0,0 @@
scala_library(
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals",
"user-signal-service/thrift/src/main/thrift:thrift-scala",
],
)

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/BUILD.docx Normal file
View File

Binary file not shown.

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/SignalFetcherConfig.docx Normal file
View File

Binary file not shown.

253
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/SignalFetcherConfig.scala
View File

@ -1,253 +0,0 @@
package com.twitter.usersignalservice.config
import com.twitter.conversions.DurationOps._
import com.twitter.finagle.memcached.{Client => MemcachedClient}
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.storehaus.ReadableStore
import com.twitter.usersignalservice.base.BaseSignalFetcher
import com.twitter.usersignalservice.base.AggregatedSignalController
import com.twitter.usersignalservice.base.FilteredSignalFetcherController
import com.twitter.usersignalservice.base.MemcachedSignalFetcherWrapper
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.base.SignalAggregatedInfo
import com.twitter.usersignalservice.signals.AccountBlocksFetcher
import com.twitter.usersignalservice.signals.AccountFollowsFetcher
import com.twitter.usersignalservice.signals.AccountMutesFetcher
import com.twitter.usersignalservice.signals.NotificationOpenAndClickFetcher
import com.twitter.usersignalservice.signals.OriginalTweetsFetcher
import com.twitter.usersignalservice.signals.ProfileVisitsFetcher
import com.twitter.usersignalservice.signals.ProfileClickFetcher
import com.twitter.usersignalservice.signals.RealGraphOonFetcher
import com.twitter.usersignalservice.signals.ReplyTweetsFetcher
import com.twitter.usersignalservice.signals.RetweetsFetcher
import com.twitter.usersignalservice.signals.TweetClickFetcher
import com.twitter.usersignalservice.signals.TweetFavoritesFetcher
import com.twitter.usersignalservice.signals.TweetSharesFetcher
import com.twitter.usersignalservice.signals.VideoTweetsPlayback50Fetcher
import com.twitter.usersignalservice.signals.VideoTweetsQualityViewFetcher
import com.twitter.usersignalservice.signals.NegativeEngagedUserFetcher
import com.twitter.usersignalservice.signals.NegativeEngagedTweetFetcher
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class SignalFetcherConfig @Inject() (
notificationOpenAndClickFetcher: NotificationOpenAndClickFetcher,
accountFollowsFetcher: AccountFollowsFetcher,
profileVisitsFetcher: ProfileVisitsFetcher,
tweetFavoritesFetcher: TweetFavoritesFetcher,
retweetsFetcher: RetweetsFetcher,
replyTweetsFetcher: ReplyTweetsFetcher,
originalTweetsFetcher: OriginalTweetsFetcher,
tweetSharesFetcher: TweetSharesFetcher,
memcachedClient: MemcachedClient,
realGraphOonFetcher: RealGraphOonFetcher,
tweetClickFetcher: TweetClickFetcher,
videoTweetsPlayback50Fetcher: VideoTweetsPlayback50Fetcher,
videoTweetsQualityViewFetcher: VideoTweetsQualityViewFetcher,
accountMutesFetcher: AccountMutesFetcher,
accountBlocksFetcher: AccountBlocksFetcher,
profileClickFetcher: ProfileClickFetcher,
negativeEngagedTweetFetcher: NegativeEngagedTweetFetcher,
negativeEngagedUserFetcher: NegativeEngagedUserFetcher,
statsReceiver: StatsReceiver,
timer: Timer) {
val MemcachedProfileVisitsFetcher: BaseSignalFetcher =
MemcachedSignalFetcherWrapper(
memcachedClient,
profileVisitsFetcher,
ttl = 8.hours,
statsReceiver,
keyPrefix = "uss:pv",
timer)
val MemcachedAccountFollowsFetcher: BaseSignalFetcher = MemcachedSignalFetcherWrapper(
memcachedClient,
accountFollowsFetcher,
ttl = 5.minute,
statsReceiver,
keyPrefix = "uss:af",
timer)
val GoodTweetClickDdgFetcher: SignalType => FilteredSignalFetcherController = signalType =>
FilteredSignalFetcherController(
tweetClickFetcher,
signalType,
statsReceiver,
timer,
Map(SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher)
)
val GoodProfileClickDdgFetcher: SignalType => FilteredSignalFetcherController = signalType =>
FilteredSignalFetcherController(
profileClickFetcher,
signalType,
statsReceiver,
timer,
Map(SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher)
)
val GoodProfileClickDdgFetcherWithBlocksMutes: SignalType => FilteredSignalFetcherController =
signalType =>
FilteredSignalFetcherController(
profileClickFetcher,
signalType,
statsReceiver,
timer,
Map(
SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher,
SignalType.AccountMute -> accountMutesFetcher,
SignalType.AccountBlock -> accountBlocksFetcher
)
)
val realGraphOonFilteredFetcher: FilteredSignalFetcherController =
FilteredSignalFetcherController(
realGraphOonFetcher,
SignalType.RealGraphOon,
statsReceiver,
timer,
Map(
SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher
)
)
val videoTweetsQualityViewFilteredFetcher: FilteredSignalFetcherController =
FilteredSignalFetcherController(
videoTweetsQualityViewFetcher,
SignalType.VideoView90dQualityV1,
statsReceiver,
timer,
Map(SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher)
)
val videoTweetsPlayback50FilteredFetcher: FilteredSignalFetcherController =
FilteredSignalFetcherController(
videoTweetsPlayback50Fetcher,
SignalType.VideoView90dPlayback50V1,
statsReceiver,
timer,
Map(SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher)
)
val uniformTweetSignalInfo: Seq[SignalAggregatedInfo] = Seq(
SignalAggregatedInfo(SignalType.TweetFavorite, tweetFavoritesFetcher),
SignalAggregatedInfo(SignalType.Retweet, retweetsFetcher),
SignalAggregatedInfo(SignalType.Reply, replyTweetsFetcher),
SignalAggregatedInfo(SignalType.OriginalTweet, originalTweetsFetcher),
SignalAggregatedInfo(SignalType.TweetShareV1, tweetSharesFetcher),
SignalAggregatedInfo(SignalType.VideoView90dQualityV1, videoTweetsQualityViewFilteredFetcher),
)
val uniformProducerSignalInfo: Seq[SignalAggregatedInfo] = Seq(
SignalAggregatedInfo(SignalType.AccountFollow, MemcachedAccountFollowsFetcher),
SignalAggregatedInfo(
SignalType.RepeatedProfileVisit90dMinVisit6V1,
MemcachedProfileVisitsFetcher),
)
val memcachedAccountBlocksFetcher: MemcachedSignalFetcherWrapper = MemcachedSignalFetcherWrapper(
memcachedClient,
accountBlocksFetcher,
ttl = 5.minutes,
statsReceiver,
keyPrefix = "uss:ab",
timer)
val memcachedAccountMutesFetcher: MemcachedSignalFetcherWrapper = MemcachedSignalFetcherWrapper(
memcachedClient,
accountMutesFetcher,
ttl = 5.minutes,
statsReceiver,
keyPrefix = "uss:am",
timer)
val SignalFetcherMapper: Map[SignalType, ReadableStore[Query, Seq[Signal]]] = Map(
/* Raw Signals */
SignalType.AccountFollow -> accountFollowsFetcher,
SignalType.AccountFollowWithDelay -> MemcachedAccountFollowsFetcher,
SignalType.GoodProfileClick -> GoodProfileClickDdgFetcher(SignalType.GoodProfileClick),
SignalType.GoodProfileClick20s -> GoodProfileClickDdgFetcher(SignalType.GoodProfileClick20s),
SignalType.GoodProfileClick30s -> GoodProfileClickDdgFetcher(SignalType.GoodProfileClick30s),
SignalType.GoodProfileClickFiltered -> GoodProfileClickDdgFetcherWithBlocksMutes(
SignalType.GoodProfileClick),
SignalType.GoodProfileClick20sFiltered -> GoodProfileClickDdgFetcherWithBlocksMutes(
SignalType.GoodProfileClick20s),
SignalType.GoodProfileClick30sFiltered -> GoodProfileClickDdgFetcherWithBlocksMutes(
SignalType.GoodProfileClick30s),
SignalType.GoodTweetClick -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick),
SignalType.GoodTweetClick5s -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick5s),
SignalType.GoodTweetClick10s -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick10s),
SignalType.GoodTweetClick30s -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick30s),
SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher,
SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher,
SignalType.NotificationOpenAndClickV1 -> notificationOpenAndClickFetcher,
SignalType.OriginalTweet -> originalTweetsFetcher,
SignalType.OriginalTweet90dV2 -> originalTweetsFetcher,
SignalType.RealGraphOon -> realGraphOonFilteredFetcher,
SignalType.RepeatedProfileVisit14dMinVisit2V1 -> MemcachedProfileVisitsFetcher,
SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative -> FilteredSignalFetcherController(
MemcachedProfileVisitsFetcher,
SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative,
statsReceiver,
timer,
Map(
SignalType.AccountMute -> accountMutesFetcher,
SignalType.AccountBlock -> accountBlocksFetcher)
),
SignalType.RepeatedProfileVisit90dMinVisit6V1 -> MemcachedProfileVisitsFetcher,
SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative -> FilteredSignalFetcherController(
MemcachedProfileVisitsFetcher,
SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative,
statsReceiver,
timer,
Map(
SignalType.AccountMute -> accountMutesFetcher,
SignalType.AccountBlock -> accountBlocksFetcher),
),
SignalType.RepeatedProfileVisit180dMinVisit6V1 -> MemcachedProfileVisitsFetcher,
SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative -> FilteredSignalFetcherController(
MemcachedProfileVisitsFetcher,
SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative,
statsReceiver,
timer,
Map(
SignalType.AccountMute -> accountMutesFetcher,
SignalType.AccountBlock -> accountBlocksFetcher),
),
SignalType.Reply -> replyTweetsFetcher,
SignalType.Reply90dV2 -> replyTweetsFetcher,
SignalType.Retweet -> retweetsFetcher,
SignalType.Retweet90dV2 -> retweetsFetcher,
SignalType.TweetFavorite -> tweetFavoritesFetcher,
SignalType.TweetFavorite90dV2 -> tweetFavoritesFetcher,
SignalType.TweetShareV1 -> tweetSharesFetcher,
SignalType.VideoView90dQualityV1 -> videoTweetsQualityViewFilteredFetcher,
SignalType.VideoView90dPlayback50V1 -> videoTweetsPlayback50FilteredFetcher,
/* Aggregated Signals */
SignalType.ProducerBasedUnifiedEngagementWeightedSignal -> AggregatedSignalController(
uniformProducerSignalInfo,
uniformProducerSignalEngagementAggregation,
statsReceiver,
timer
),
SignalType.TweetBasedUnifiedEngagementWeightedSignal -> AggregatedSignalController(
uniformTweetSignalInfo,
uniformTweetSignalEngagementAggregation,
statsReceiver,
timer
),
SignalType.AdFavorite -> tweetFavoritesFetcher,
/* Negative Signals */
SignalType.AccountBlock -> memcachedAccountBlocksFetcher,
SignalType.AccountMute -> memcachedAccountMutesFetcher,
SignalType.TweetDontLike -> negativeEngagedTweetFetcher,
SignalType.TweetReport -> negativeEngagedTweetFetcher,
SignalType.TweetSeeFewer -> negativeEngagedTweetFetcher,
)
}

14
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/BUILD
View File

@ -1,14 +0,0 @@
scala_library(
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"src/scala/com/twitter/twistly/common",
"src/scala/com/twitter/twistly/store",
"src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module",
"user-signal-service/thrift/src/main/thrift:thrift-scala",
],
)

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/BUILD.docx Normal file
View File

Binary file not shown.

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/UserSignalHandler.docx Normal file
View File

Binary file not shown.

71
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/UserSignalHandler.scala
View File

@ -1,71 +0,0 @@
package com.twitter.usersignalservice.handler
import com.twitter.storehaus.ReadableStore
import com.twitter.usersignalservice.thriftscala.BatchSignalRequest
import com.twitter.usersignalservice.thriftscala.BatchSignalResponse
import com.twitter.util.Future
import com.twitter.conversions.DurationOps._
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.frigate.common.util.StatsUtil
import com.twitter.usersignalservice.config.SignalFetcherConfig
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.thriftscala.ClientIdentifier
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Duration
import com.twitter.util.Timer
import com.twitter.util.TimeoutException
class UserSignalHandler(
signalFetcherConfig: SignalFetcherConfig,
timer: Timer,
stats: StatsReceiver) {
import UserSignalHandler._
val statsReceiver: StatsReceiver = stats.scope("user-signal-service/service")
def getBatchSignalsResponse(request: BatchSignalRequest): Future[Option[BatchSignalResponse]] = {
StatsUtil.trackOptionStats(statsReceiver) {
val allSignals = request.signalRequest.map { signalRequest =>
signalFetcherConfig
.SignalFetcherMapper(signalRequest.signalType)
.get(Query(
userId = request.userId,
signalType = signalRequest.signalType,
maxResults = signalRequest.maxResults.map(_.toInt),
clientId = request.clientId.getOrElse(ClientIdentifier.Unknown)
))
.map(signalOpt => (signalRequest.signalType, signalOpt))
}
Future.collect(allSignals).map { signalsSeq =>
val signalsMap = signalsSeq.map {
case (signalType: SignalType, signalsOpt) =>
(signalType, signalsOpt.getOrElse(EmptySeq))
}.toMap
Some(BatchSignalResponse(signalsMap))
}
}
}
def toReadableStore: ReadableStore[BatchSignalRequest, BatchSignalResponse] = {
new ReadableStore[BatchSignalRequest, BatchSignalResponse] {
override def get(request: BatchSignalRequest): Future[Option[BatchSignalResponse]] = {
getBatchSignalsResponse(request).raiseWithin(UserSignalServiceTimeout)(timer).rescue {
case _: TimeoutException =>
statsReceiver.counter("endpointGetBatchSignals/failure/timeout").incr()
EmptyResponse
case e =>
statsReceiver.counter("endpointGetBatchSignals/failure/" + e.getClass.getName).incr()
EmptyResponse
}
}
}
}
}
object UserSignalHandler {
val UserSignalServiceTimeout: Duration = 25.milliseconds
val EmptySeq: Seq[Nothing] = Seq.empty
val EmptyResponse: Future[Option[BatchSignalResponse]] = Future.value(Some(BatchSignalResponse()))
}

25
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/BUILD
View File

@ -1,25 +0,0 @@
scala_library(
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication",
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client",
"finagle/finagle-core/src/main",
"finagle/finagle-stats",
"finagle/finagle-thrift/src/main/scala",
"hermit/hermit-core/src/main/scala/com/twitter/hermit/predicate/socialgraph",
"relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection",
"servo/service/src/main/scala",
"src/scala/com/twitter/storehaus_internal/manhattan2",
"src/scala/com/twitter/storehaus_internal/memcache",
"src/scala/com/twitter/storehaus_internal/util",
"src/scala/com/twitter/twistly/common",
"src/scala/com/twitter/twistly/store",
"src/thrift/com/twitter/socialgraph:thrift-scala",
"stitch/stitch-storehaus",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"util/util-core:scala",
"util/util-stats/src/main/scala",
],
)

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/BUILD.docx Normal file
View File

Binary file not shown.

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/CacheModule.docx Normal file
View File

Binary file not shown.

34
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/CacheModule.scala
View File

@ -1,34 +0,0 @@
package com.twitter.usersignalservice.module
import com.google.inject.Provides
import javax.inject.Singleton
import com.twitter.finagle.memcached.Client
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.inject.TwitterModule
import com.twitter.conversions.DurationOps._
import com.twitter.storehaus_internal.memcache.MemcacheStore
import com.twitter.storehaus_internal.util.ZkEndPoint
import com.twitter.storehaus_internal.util.ClientName
object CacheModule extends TwitterModule {
private val cacheDest =
flag[String](name = "cache_module.dest", help = "Path to memcache service")
private val timeout =
flag[Int](name = "memcache.timeout", help = "Memcache client timeout")
@Singleton
@Provides
def providesCache(
serviceIdentifier: ServiceIdentifier,
stats: StatsReceiver
): Client =
MemcacheStore.memcachedClient(
name = ClientName("memcache_user_signal_service"),
dest = ZkEndPoint(cacheDest()),
timeout = timeout().milliseconds,
retries = 0,
statsReceiver = stats.scope("memcache"),
serviceIdentifier = serviceIdentifier
)
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/MHMtlsParamsModule.docx Normal file
View File

Binary file not shown.

18
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/MHMtlsParamsModule.scala
View File

@ -1,18 +0,0 @@
package com.twitter.usersignalservice.module
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
import com.twitter.inject.TwitterModule
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
import com.google.inject.Provides
import javax.inject.Singleton
object MHMtlsParamsModule extends TwitterModule {
@Singleton
@Provides
def providesManhattanMtlsParams(
serviceIdentifier: ServiceIdentifier
): ManhattanKVClientMtlsParams = {
ManhattanKVClientMtlsParams(serviceIdentifier)
}
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/SocialGraphServiceClientModule.docx Normal file
View File

Binary file not shown.

40
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/SocialGraphServiceClientModule.scala
View File

@ -1,40 +0,0 @@
package com.twitter.usersignalservice.module
import com.twitter.inject.Injector
import com.twitter.conversions.DurationOps._
import com.twitter.finagle._
import com.twitter.finagle.mux.ClientDiscardedRequestException
import com.twitter.finagle.service.ReqRep
import com.twitter.finagle.service.ResponseClass
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient
import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule
import com.twitter.util.Duration
import com.twitter.util.Throw
import com.twitter.socialgraph.thriftscala.SocialGraphService
object SocialGraphServiceClientModule
extends ThriftMethodBuilderClientModule[
SocialGraphService.ServicePerEndpoint,
SocialGraphService.MethodPerEndpoint
]
with MtlsClient {
override val label = "socialgraph"
override val dest = "/s/socialgraph/socialgraph"
override val requestTimeout: Duration = 30.milliseconds
override def configureThriftMuxClient(
injector: Injector,
client: ThriftMux.Client
): ThriftMux.Client = {
super
.configureThriftMuxClient(injector, client)
.withStatsReceiver(injector.instance[StatsReceiver].scope("clnt"))
.withSessionQualifier
.successRateFailureAccrual(successRate = 0.9, window = 30.seconds)
.withResponseClassifier {
case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable
}
}
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/TimerModule.docx Normal file
View File

Binary file not shown.

12
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/TimerModule.scala
View File

@ -1,12 +0,0 @@
package com.twitter.usersignalservice.module
import com.google.inject.Provides
import com.twitter.finagle.util.DefaultTimer
import com.twitter.inject.TwitterModule
import com.twitter.util.Timer
import javax.inject.Singleton
object TimerModule extends TwitterModule {
@Singleton
@Provides
def providesTimer: Timer = DefaultTimer
}

13
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/BUILD
View File

@ -1,13 +0,0 @@
scala_library(
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"stitch/stitch-storehaus",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module",
"user-signal-service/thrift/src/main/thrift:thrift-scala",
],
)

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/BUILD.docx Normal file
View File

Binary file not shown.

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/UserSignalService.docx Normal file
View File

Binary file not shown.

26
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/UserSignalService.scala
View File

@ -1,26 +0,0 @@
package com.twitter.usersignalservice
package service
import com.google.inject.Inject
import com.google.inject.Singleton
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.stitch.storehaus.StitchOfReadableStore
import com.twitter.usersignalservice.config.SignalFetcherConfig
import com.twitter.usersignalservice.handler.UserSignalHandler
import com.twitter.usersignalservice.thriftscala.BatchSignalRequest
import com.twitter.usersignalservice.thriftscala.BatchSignalResponse
import com.twitter.util.Timer
@Singleton
class UserSignalService @Inject() (
signalFetcherConfig: SignalFetcherConfig,
timer: Timer,
stats: StatsReceiver) {
private val userSignalHandler =
new UserSignalHandler(signalFetcherConfig, timer, stats)
val userSignalServiceHandlerStoreStitch: BatchSignalRequest => com.twitter.stitch.Stitch[
BatchSignalResponse
] = StitchOfReadableStore(userSignalHandler.toReadableStore)
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountBlocksFetcher.docx Normal file
View File

Binary file not shown.

40
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountBlocksFetcher.scala
View File

@ -1,40 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.socialgraph.thriftscala.RelationshipType
import com.twitter.socialgraph.thriftscala.SocialGraphService
import com.twitter.twistly.common.UserId
import com.twitter.usersignalservice.base.BaseSignalFetcher
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.signals.common.SGSUtils
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class AccountBlocksFetcher @Inject() (
sgsClient: SocialGraphService.MethodPerEndpoint,
timer: Timer,
stats: StatsReceiver)
extends BaseSignalFetcher {
override type RawSignalType = Signal
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(this.name)
override def getRawSignals(
userId: UserId
): Future[Option[Seq[RawSignalType]]] = {
SGSUtils.getSGSRawSignals(userId, sgsClient, RelationshipType.Blocking, SignalType.AccountBlock)
}
override def process(
query: Query,
rawSignals: Future[Option[Seq[RawSignalType]]]
): Future[Option[Seq[Signal]]] = {
rawSignals.map(_.map(_.take(query.maxResults.getOrElse(Int.MaxValue))))
}
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountFollowsFetcher.docx Normal file
View File

Binary file not shown.

44
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountFollowsFetcher.scala
View File

@ -1,44 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.socialgraph.thriftscala.RelationshipType
import com.twitter.socialgraph.thriftscala.SocialGraphService
import com.twitter.twistly.common.UserId
import com.twitter.usersignalservice.base.BaseSignalFetcher
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.signals.common.SGSUtils
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class AccountFollowsFetcher @Inject() (
sgsClient: SocialGraphService.MethodPerEndpoint,
timer: Timer,
stats: StatsReceiver)
extends BaseSignalFetcher {
override type RawSignalType = Signal
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(this.name)
override def getRawSignals(
userId: UserId
): Future[Option[Seq[RawSignalType]]] = {
SGSUtils.getSGSRawSignals(
userId,
sgsClient,
RelationshipType.Following,
SignalType.AccountFollow)
}
override def process(
query: Query,
rawSignals: Future[Option[Seq[RawSignalType]]]
): Future[Option[Seq[Signal]]] = {
rawSignals.map(_.map(_.take(query.maxResults.getOrElse(Int.MaxValue))))
}
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountMutesFetcher.docx Normal file
View File

Binary file not shown.

40
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountMutesFetcher.scala
View File

@ -1,40 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.socialgraph.thriftscala.RelationshipType
import com.twitter.socialgraph.thriftscala.SocialGraphService
import com.twitter.twistly.common.UserId
import com.twitter.usersignalservice.base.BaseSignalFetcher
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.signals.common.SGSUtils
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class AccountMutesFetcher @Inject() (
sgsClient: SocialGraphService.MethodPerEndpoint,
timer: Timer,
stats: StatsReceiver)
extends BaseSignalFetcher {
override type RawSignalType = Signal
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(this.name)
override def getRawSignals(
userId: UserId
): Future[Option[Seq[RawSignalType]]] = {
SGSUtils.getSGSRawSignals(userId, sgsClient, RelationshipType.Muting, SignalType.AccountMute)
}
override def process(
query: Query,
rawSignals: Future[Option[Seq[RawSignalType]]]
): Future[Option[Seq[Signal]]] = {
rawSignals.map(_.map(_.take(query.maxResults.getOrElse(Int.MaxValue))))
}
}

34
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/BUILD
View File

@ -1,34 +0,0 @@
scala_library(
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/bijection:scrooge",
"3rdparty/jvm/javax/inject:javax.inject",
"3rdparty/src/jvm/com/twitter/storehaus:core",
"discovery-ds/src/main/thrift/com/twitter/dds/jobs/repeated_profile_visits:profile_visit-scala",
"flock-client/src/main/thrift:thrift-scala",
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato",
"hermit/hermit-core/src/main/scala/com/twitter/hermit/predicate/socialgraph",
"src/scala/com/twitter/scalding_internal/job",
"src/scala/com/twitter/simclusters_v2/common",
"src/scala/com/twitter/storehaus_internal/manhattan",
"src/scala/com/twitter/storehaus_internal/manhattan/config",
"src/scala/com/twitter/storehaus_internal/manhattan2",
"src/scala/com/twitter/storehaus_internal/offline",
"src/scala/com/twitter/storehaus_internal/util",
"src/scala/com/twitter/twistly/common",
"src/thrift/com/twitter/experiments/general_metrics:general_metrics-scala",
"src/thrift/com/twitter/frigate/data_pipeline:frigate-user-history-thrift-scala",
"src/thrift/com/twitter/onboarding/relevance/tweet_engagement:tweet_engagement-scala",
"src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala",
"src/thrift/com/twitter/socialgraph:thrift-scala",
"src/thrift/com/twitter/traffic_attribution:traffic_attribution-scala",
"strato/src/main/scala/com/twitter/strato/client",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base",
"user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common",
"user-signal-service/thrift/src/main/thrift:thrift-scala",
"util/util-core:util-core-util",
"util/util-core/src/main/java/com/twitter/util",
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
],
)

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/BUILD.docx Normal file
View File

Binary file not shown.

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedTweetFetcher.docx Normal file
View File

Binary file not shown.

97
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedTweetFetcher.scala
View File

@ -1,97 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.simclusters_v2.common.UserId
import com.twitter.simclusters_v2.thriftscala.InternalId
import com.twitter.strato.client.Client
import com.twitter.strato.data.Conv
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.twistly.thriftscala.RecentNegativeEngagedTweet
import com.twitter.twistly.thriftscala.TweetNegativeEngagementType
import com.twitter.twistly.thriftscala.UserRecentNegativeEngagedTweets
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.base.StratoSignalFetcher
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class NegativeEngagedTweetFetcher @Inject() (
stratoClient: Client,
timer: Timer,
stats: StatsReceiver)
extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentNegativeEngagedTweets] {
import NegativeEngagedTweetFetcher._
override type RawSignalType = RecentNegativeEngagedTweet
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(name)
override val stratoColumnPath: String = stratoPath
override val stratoView: Unit = None
override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType
override protected val viewConv: Conv[Unit] = Conv.ofType
override protected val valueConv: Conv[UserRecentNegativeEngagedTweets] =
ScroogeConv.fromStruct[UserRecentNegativeEngagedTweets]
override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, defaultVersion)
override protected def toRawSignals(
stratoValue: UserRecentNegativeEngagedTweets
): Seq[RecentNegativeEngagedTweet] = {
stratoValue.recentNegativeEngagedTweets
}
override def process(
query: Query,
rawSignals: Future[Option[Seq[RecentNegativeEngagedTweet]]]
): Future[Option[Seq[Signal]]] = {
rawSignals.map {
_.map { signals =>
signals
.filter(signal => negativeEngagedTweetTypeFilter(query.signalType, signal))
.map { signal =>
Signal(
query.signalType,
signal.engagedAt,
Some(InternalId.TweetId(signal.tweetId))
)
}
.groupBy(_.targetInternalId) // groupBy if there's duplicated authorIds
.mapValues(_.maxBy(_.timestamp))
.values
.toSeq
.sortBy(-_.timestamp)
.take(query.maxResults.getOrElse(Int.MaxValue))
}
}
}
}
object NegativeEngagedTweetFetcher {
val stratoPath = "recommendations/twistly/userRecentNegativeEngagedTweets"
private val defaultVersion = 0L
private def negativeEngagedTweetTypeFilter(
signalType: SignalType,
signal: RecentNegativeEngagedTweet
): Boolean = {
signalType match {
case SignalType.TweetDontLike =>
signal.engagementType == TweetNegativeEngagementType.DontLike
case SignalType.TweetSeeFewer =>
signal.engagementType == TweetNegativeEngagementType.SeeFewer
case SignalType.TweetReport =>
signal.engagementType == TweetNegativeEngagementType.ReportClick
case SignalType.NegativeEngagedTweetId => true
case _ => false
}
}
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedUserFetcher.docx Normal file
View File

Binary file not shown.

79
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedUserFetcher.scala
View File

@ -1,79 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.simclusters_v2.common.UserId
import com.twitter.simclusters_v2.thriftscala.InternalId
import com.twitter.strato.client.Client
import com.twitter.strato.data.Conv
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.twistly.thriftscala.RecentNegativeEngagedTweet
import com.twitter.twistly.thriftscala.UserRecentNegativeEngagedTweets
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.base.StratoSignalFetcher
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class NegativeEngagedUserFetcher @Inject() (
stratoClient: Client,
timer: Timer,
stats: StatsReceiver)
extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentNegativeEngagedTweets] {
import NegativeEngagedUserFetcher._
override type RawSignalType = RecentNegativeEngagedTweet
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(name)
override val stratoColumnPath: String = stratoPath
override val stratoView: Unit = None
override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType
override protected val viewConv: Conv[Unit] = Conv.ofType
override protected val valueConv: Conv[UserRecentNegativeEngagedTweets] =
ScroogeConv.fromStruct[UserRecentNegativeEngagedTweets]
override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, defaultVersion)
override protected def toRawSignals(
stratoValue: UserRecentNegativeEngagedTweets
): Seq[RecentNegativeEngagedTweet] = {
stratoValue.recentNegativeEngagedTweets
}
override def process(
query: Query,
rawSignals: Future[Option[Seq[RecentNegativeEngagedTweet]]]
): Future[Option[Seq[Signal]]] = {
rawSignals.map {
_.map { signals =>
signals
.map { e =>
Signal(
defaultNegativeSignalType,
e.engagedAt,
Some(InternalId.UserId(e.authorId))
)
}
.groupBy(_.targetInternalId) // groupBy if there's duplicated authorIds
.mapValues(_.maxBy(_.timestamp))
.values
.toSeq
.sortBy(-_.timestamp)
}
}
}
}
object NegativeEngagedUserFetcher {
val stratoPath = "recommendations/twistly/userRecentNegativeEngagedTweets"
private val defaultVersion = 0L
private val defaultNegativeSignalType = SignalType.NegativeEngagedUserId
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NotificationOpenAndClickFetcher.docx Normal file
View File

Binary file not shown.

145
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NotificationOpenAndClickFetcher.scala
View File

@ -1,145 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.frigate.common.store.strato.StratoFetchableStore
import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.ClientEngagementEvent
import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.LatestEvents
import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.LatestNegativeEngagementEvents
import com.twitter.simclusters_v2.thriftscala.InternalId
import com.twitter.storehaus.ReadableStore
import com.twitter.strato.client.Client
import com.twitter.twistly.common.TweetId
import com.twitter.twistly.common.UserId
import com.twitter.usersignalservice.base.BaseSignalFetcher
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class NotificationOpenAndClickFetcher @Inject() (
stratoClient: Client,
timer: Timer,
stats: StatsReceiver)
extends BaseSignalFetcher {
import NotificationOpenAndClickFetcher._
override type RawSignalType = ClientEngagementEvent
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(this.name)
private val latestEventsStore: ReadableStore[UserId, LatestEvents] = {
StratoFetchableStore
.withUnitView[UserId, LatestEvents](stratoClient, latestEventStoreColumn)
}
private val notificationNegativeEngagementStore: ReadableStore[UserId, Seq[
NotificationNegativeEngagement
]] = {
StratoFetchableStore
.withUnitView[UserId, LatestNegativeEngagementEvents](
stratoClient,
labeledPushRecsNegativeEngagementsColumn).mapValues(fromLatestNegativeEngagementEvents)
}
override def getRawSignals(
userId: UserId
): Future[Option[Seq[RawSignalType]]] = {
val notificationNegativeEngagementEventsFut =
notificationNegativeEngagementStore.get(userId)
val latestEventsFut = latestEventsStore.get(userId)
Future
.join(latestEventsFut, notificationNegativeEngagementEventsFut).map {
case (latestEventsOpt, latestNegativeEngagementEventsOpt) =>
latestEventsOpt.map { latestEvents =>
// Negative Engagement Events Filter
filterNegativeEngagementEvents(
latestEvents.engagementEvents,
latestNegativeEngagementEventsOpt.getOrElse(Seq.empty),
statsReceiver.scope("filterNegativeEngagementEvents"))
}
}
}
override def process(
query: Query,
rawSignals: Future[Option[Seq[RawSignalType]]]
): Future[Option[Seq[Signal]]] = {
rawSignals.map {
_.map {
_.take(query.maxResults.getOrElse(Int.MaxValue)).map { clientEngagementEvent =>
Signal(
SignalType.NotificationOpenAndClickV1,
timestamp = clientEngagementEvent.timestampMillis,
targetInternalId = Some(InternalId.TweetId(clientEngagementEvent.tweetId))
)
}
}
}
}
}
object NotificationOpenAndClickFetcher {
private val latestEventStoreColumn = "frigate/magicrecs/labeledPushRecsAggregated.User"
private val labeledPushRecsNegativeEngagementsColumn =
"frigate/magicrecs/labeledPushRecsNegativeEngagements.User"
case class NotificationNegativeEngagement(
tweetId: TweetId,
timestampMillis: Long,
isNtabDisliked: Boolean,
isReportTweetClicked: Boolean,
isReportTweetDone: Boolean,
isReportUserClicked: Boolean,
isReportUserDone: Boolean)
def fromLatestNegativeEngagementEvents(
latestNegativeEngagementEvents: LatestNegativeEngagementEvents
): Seq[NotificationNegativeEngagement] = {
latestNegativeEngagementEvents.negativeEngagementEvents.map { event =>
NotificationNegativeEngagement(
event.tweetId,
event.timestampMillis,
event.isNtabDisliked.getOrElse(false),
event.isReportTweetClicked.getOrElse(false),
event.isReportTweetDone.getOrElse(false),
event.isReportUserClicked.getOrElse(false),
event.isReportUserDone.getOrElse(false)
)
}
}
private def filterNegativeEngagementEvents(
engagementEvents: Seq[ClientEngagementEvent],
negativeEvents: Seq[NotificationNegativeEngagement],
statsReceiver: StatsReceiver
): Seq[ClientEngagementEvent] = {
if (negativeEvents.nonEmpty) {
statsReceiver.counter("filterNegativeEngagementEvents").incr()
statsReceiver.stat("eventSizeBeforeFilter").add(engagementEvents.size)
val negativeEngagementIdSet =
negativeEvents.collect {
case event
if event.isNtabDisliked || event.isReportTweetClicked || event.isReportTweetDone || event.isReportUserClicked || event.isReportUserDone =>
event.tweetId
}.toSet
// negative event size
statsReceiver.stat("negativeEventsSize").add(negativeEngagementIdSet.size)
// filter out negative engagement sources
val result = engagementEvents.filterNot { event =>
negativeEngagementIdSet.contains(event.tweetId)
}
statsReceiver.stat("eventSizeAfterFilter").add(result.size)
result
} else engagementEvents
}
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/OriginalTweetsFetcher.docx Normal file
View File

Binary file not shown.

70
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/OriginalTweetsFetcher.scala
View File

@ -1,70 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.simclusters_v2.common.UserId
import com.twitter.simclusters_v2.thriftscala.InternalId
import com.twitter.strato.client.Client
import com.twitter.strato.data.Conv
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.twistly.common.TwistlyProfile
import com.twitter.twistly.thriftscala.EngagementMetadata.OriginalTweetMetadata
import com.twitter.twistly.thriftscala.RecentEngagedTweet
import com.twitter.twistly.thriftscala.UserRecentEngagedTweets
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.base.StratoSignalFetcher
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class OriginalTweetsFetcher @Inject() (
stratoClient: Client,
timer: Timer,
stats: StatsReceiver)
extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentEngagedTweets] {
import OriginalTweetsFetcher._
override type RawSignalType = RecentEngagedTweet
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(name)
override val stratoColumnPath: String =
TwistlyProfile.TwistlyProdProfile.userRecentEngagedStorePath
override val stratoView: Unit = None
override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType
override protected val viewConv: Conv[Unit] = Conv.ofType
override protected val valueConv: Conv[UserRecentEngagedTweets] =
ScroogeConv.fromStruct[UserRecentEngagedTweets]
override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, DefaultVersion)
override protected def toRawSignals(
userRecentEngagedTweets: UserRecentEngagedTweets
): Seq[RawSignalType] =
userRecentEngagedTweets.recentEngagedTweets
override def process(
query: Query,
rawSignals: Future[Option[Seq[RawSignalType]]]
): Future[Option[Seq[Signal]]] = {
rawSignals.map {
_.map { signals =>
val lookBackWindowFilteredSignals =
SignalFilter.lookBackWindow90DayFilter(signals, query.signalType)
lookBackWindowFilteredSignals
.collect {
case RecentEngagedTweet(tweetId, engagedAt, _: OriginalTweetMetadata, _) =>
Signal(query.signalType, engagedAt, Some(InternalId.TweetId(tweetId)))
}.take(query.maxResults.getOrElse(Int.MaxValue))
}
}
}
}
object OriginalTweetsFetcher {
// see com.twitter.twistly.store.UserRecentEngagedTweetsStore
private val DefaultVersion = 0
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileClickFetcher.docx Normal file
View File

Binary file not shown.

98
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileClickFetcher.scala
View File

@ -1,98 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.simclusters_v2.common.UserId
import com.twitter.simclusters_v2.thriftscala.InternalId
import com.twitter.strato.client.Client
import com.twitter.strato.data.Conv
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.twistly.thriftscala.RecentProfileClickImpressEvents
import com.twitter.twistly.thriftscala.ProfileClickImpressEvent
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.base.StratoSignalFetcher
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class ProfileClickFetcher @Inject() (
stratoClient: Client,
timer: Timer,
stats: StatsReceiver)
extends StratoSignalFetcher[(UserId, Long), Unit, RecentProfileClickImpressEvents] {
import ProfileClickFetcher._
override type RawSignalType = ProfileClickImpressEvent
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(name)
override val stratoColumnPath: String = stratoPath
override val stratoView: Unit = None
override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType
override protected val viewConv: Conv[Unit] = Conv.ofType
override protected val valueConv: Conv[RecentProfileClickImpressEvents] =
ScroogeConv.fromStruct[RecentProfileClickImpressEvents]
override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, defaultVersion)
override protected def toRawSignals(
stratoValue: RecentProfileClickImpressEvents
): Seq[ProfileClickImpressEvent] = {
stratoValue.events
}
override def process(
query: Query,
rawSignals: Future[Option[Seq[ProfileClickImpressEvent]]]
): Future[Option[Seq[Signal]]] = {
rawSignals.map { events =>
events
.map { clicks =>
clicks
.filter(dwelltimeFilter(_, query.signalType))
.map(signalFromProfileClick(_, query.signalType))
.sortBy(-_.timestamp)
.take(query.maxResults.getOrElse(Int.MaxValue))
}
}
}
}
object ProfileClickFetcher {
val stratoPath = "recommendations/twistly/userRecentProfileClickImpress"
private val defaultVersion = 0L
private val sec2millis: Int => Long = i => i * 1000L
private val minDwellTimeMap: Map[SignalType, Long] = Map(
SignalType.GoodProfileClick -> sec2millis(10),
SignalType.GoodProfileClick20s -> sec2millis(20),
SignalType.GoodProfileClick30s -> sec2millis(30),
SignalType.GoodProfileClickFiltered -> sec2millis(10),
SignalType.GoodProfileClick20sFiltered -> sec2millis(20),
SignalType.GoodProfileClick30sFiltered -> sec2millis(30),
)
def signalFromProfileClick(
profileClickImpressEvent: ProfileClickImpressEvent,
signalType: SignalType
): Signal = {
Signal(
signalType,
profileClickImpressEvent.engagedAt,
Some(InternalId.UserId(profileClickImpressEvent.entityId))
)
}
def dwelltimeFilter(
profileClickImpressEvent: ProfileClickImpressEvent,
signalType: SignalType
): Boolean = {
val goodClickDwellTime = minDwellTimeMap(signalType)
profileClickImpressEvent.clickImpressEventMetadata.totalDwellTime >= goodClickDwellTime
}
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileVisitsFetcher.docx Normal file
View File

Binary file not shown.

143
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileVisitsFetcher.scala
View File

@ -1,143 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.bijection.Codec
import com.twitter.bijection.scrooge.BinaryScalaCodec
import com.twitter.dds.jobs.repeated_profile_visits.thriftscala.ProfileVisitSet
import com.twitter.dds.jobs.repeated_profile_visits.thriftscala.ProfileVisitorInfo
import com.twitter.experiments.general_metrics.thriftscala.IdType
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.simclusters_v2.thriftscala.InternalId
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
import com.twitter.storehaus_internal.manhattan.Apollo
import com.twitter.storehaus_internal.manhattan.ManhattanCluster
import com.twitter.twistly.common.UserId
import com.twitter.usersignalservice.base.ManhattanSignalFetcher
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
case class ProfileVisitMetadata(
targetId: Option[Long],
totalTargetVisitsInLast14Days: Option[Int],
totalTargetVisitsInLast90Days: Option[Int],
totalTargetVisitsInLast180Days: Option[Int],
latestTargetVisitTimestampInLast90Days: Option[Long])
@Singleton
case class ProfileVisitsFetcher @Inject() (
manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams,
timer: Timer,
stats: StatsReceiver)
extends ManhattanSignalFetcher[ProfileVisitorInfo, ProfileVisitSet] {
import ProfileVisitsFetcher._
override type RawSignalType = ProfileVisitMetadata
override val manhattanAppId: String = MHAppId
override val manhattanDatasetName: String = MHDatasetName
override val manhattanClusterId: ManhattanCluster = Apollo
override val manhattanKeyCodec: Codec[ProfileVisitorInfo] = BinaryScalaCodec(ProfileVisitorInfo)
override val manhattanRawSignalCodec: Codec[ProfileVisitSet] = BinaryScalaCodec(ProfileVisitSet)
override protected def toManhattanKey(userId: UserId): ProfileVisitorInfo =
ProfileVisitorInfo(userId, IdType.User)
override protected def toRawSignals(manhattanValue: ProfileVisitSet): Seq[ProfileVisitMetadata] =
manhattanValue.profileVisitSet
.map {
_.collect {
// only keep the Non-NSFW and not-following profile visits
case profileVisit
if profileVisit.targetId.nonEmpty
// The below check covers 180 days, not only 90 days as the name implies.
// See comment on [[ProfileVisit.latestTargetVisitTimestampInLast90Days]] thrift.
&& profileVisit.latestTargetVisitTimestampInLast90Days.nonEmpty
&& !profileVisit.isTargetNSFW.getOrElse(false)
&& !profileVisit.doesSourceIdFollowTargetId.getOrElse(false) =>
ProfileVisitMetadata(
targetId = profileVisit.targetId,
totalTargetVisitsInLast14Days = profileVisit.totalTargetVisitsInLast14Days,
totalTargetVisitsInLast90Days = profileVisit.totalTargetVisitsInLast90Days,
totalTargetVisitsInLast180Days = profileVisit.totalTargetVisitsInLast180Days,
latestTargetVisitTimestampInLast90Days =
profileVisit.latestTargetVisitTimestampInLast90Days
)
}.toSeq
}.getOrElse(Seq.empty)
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(name)
override def process(
query: Query,
rawSignals: Future[Option[Seq[ProfileVisitMetadata]]]
): Future[Option[Seq[Signal]]] = rawSignals.map { profiles =>
profiles
.map {
_.filter(profileVisitMetadata => visitCountFilter(profileVisitMetadata, query.signalType))
.sortBy(profileVisitMetadata =>
-visitCountMap(query.signalType)(profileVisitMetadata).getOrElse(0))
.map(profileVisitMetadata =>
signalFromProfileVisit(profileVisitMetadata, query.signalType))
.take(query.maxResults.getOrElse(Int.MaxValue))
}
}
}
object ProfileVisitsFetcher {
private val MHAppId = "repeated_profile_visits_aggregated"
private val MHDatasetName = "repeated_profile_visits_aggregated"
private val minVisitCountMap: Map[SignalType, Int] = Map(
SignalType.RepeatedProfileVisit14dMinVisit2V1 -> 2,
SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative -> 2,
SignalType.RepeatedProfileVisit90dMinVisit6V1 -> 6,
SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative -> 6,
SignalType.RepeatedProfileVisit180dMinVisit6V1 -> 6,
SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative -> 6
)
private val visitCountMap: Map[SignalType, ProfileVisitMetadata => Option[Int]] = Map(
SignalType.RepeatedProfileVisit14dMinVisit2V1 ->
((profileVisitMetadata: ProfileVisitMetadata) =>
profileVisitMetadata.totalTargetVisitsInLast14Days),
SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative ->
((profileVisitMetadata: ProfileVisitMetadata) =>
profileVisitMetadata.totalTargetVisitsInLast14Days),
SignalType.RepeatedProfileVisit90dMinVisit6V1 ->
((profileVisitMetadata: ProfileVisitMetadata) =>
profileVisitMetadata.totalTargetVisitsInLast90Days),
SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative ->
((profileVisitMetadata: ProfileVisitMetadata) =>
profileVisitMetadata.totalTargetVisitsInLast90Days),
SignalType.RepeatedProfileVisit180dMinVisit6V1 ->
((profileVisitMetadata: ProfileVisitMetadata) =>
profileVisitMetadata.totalTargetVisitsInLast180Days),
SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative ->
((profileVisitMetadata: ProfileVisitMetadata) =>
profileVisitMetadata.totalTargetVisitsInLast180Days)
)
def signalFromProfileVisit(
profileVisitMetadata: ProfileVisitMetadata,
signalType: SignalType
): Signal = {
Signal(
signalType,
profileVisitMetadata.latestTargetVisitTimestampInLast90Days.get,
profileVisitMetadata.targetId.map(targetId => InternalId.UserId(targetId))
)
}
def visitCountFilter(
profileVisitMetadata: ProfileVisitMetadata,
signalType: SignalType
): Boolean = {
visitCountMap(signalType)(profileVisitMetadata).exists(_ >= minVisitCountMap(signalType))
}
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RealGraphOonFetcher.docx Normal file
View File

Binary file not shown.

70
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RealGraphOonFetcher.scala
View File

@ -1,70 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.simclusters_v2.common.UserId
import com.twitter.simclusters_v2.thriftscala.InternalId
import com.twitter.strato.client.Client
import com.twitter.strato.data.Conv
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.usersignalservice.base.Query
import com.twitter.wtf.candidate.thriftscala.CandidateSeq
import com.twitter.wtf.candidate.thriftscala.Candidate
import com.twitter.usersignalservice.base.StratoSignalFetcher
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.usersignalservice.thriftscala.SignalType
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class RealGraphOonFetcher @Inject() (
stratoClient: Client,
timer: Timer,
stats: StatsReceiver)
extends StratoSignalFetcher[UserId, Unit, CandidateSeq] {
import RealGraphOonFetcher._
override type RawSignalType = Candidate
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(name)
override val stratoColumnPath: String = RealGraphOonFetcher.stratoColumnPath
override val stratoView: Unit = None
override protected val keyConv: Conv[UserId] = Conv.ofType
override protected val viewConv: Conv[Unit] = Conv.ofType
override protected val valueConv: Conv[CandidateSeq] =
ScroogeConv.fromStruct[CandidateSeq]
override protected def toStratoKey(userId: UserId): UserId = userId
override protected def toRawSignals(
realGraphOonCandidates: CandidateSeq
): Seq[RawSignalType] = realGraphOonCandidates.candidates
override def process(
query: Query,
rawSignals: Future[Option[Seq[RawSignalType]]]
): Future[Option[Seq[Signal]]] = {
rawSignals
.map {
_.map(
_.sortBy(-_.score)
.collect {
case c if c.score >= MinRgScore =>
Signal(
SignalType.RealGraphOon,
RealGraphOonFetcher.DefaultTimestamp,
Some(InternalId.UserId(c.userId)))
}.take(query.maxResults.getOrElse(Int.MaxValue)))
}
}
}
object RealGraphOonFetcher {
val stratoColumnPath = "recommendations/real_graph/realGraphScoresOon.User"
// quality threshold for real graph score
private val MinRgScore = 0.0
// no timestamp for RealGraph Candidates, set default as 0L
private val DefaultTimestamp = 0L
}

BIN
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ReplyTweetsFetcher.docx Normal file
View File

Binary file not shown.

70
user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ReplyTweetsFetcher.scala
View File

@ -1,70 +0,0 @@
package com.twitter.usersignalservice.signals
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.simclusters_v2.common.UserId
import com.twitter.simclusters_v2.thriftscala.InternalId
import com.twitter.strato.client.Client
import com.twitter.strato.data.Conv
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.twistly.common.TwistlyProfile
import com.twitter.twistly.thriftscala.EngagementMetadata.ReplyTweetMetadata
import com.twitter.twistly.thriftscala.RecentEngagedTweet
import com.twitter.twistly.thriftscala.UserRecentEngagedTweets
import com.twitter.usersignalservice.base.Query
import com.twitter.usersignalservice.base.StratoSignalFetcher
import com.twitter.usersignalservice.thriftscala.Signal
import com.twitter.util.Future
import com.twitter.util.Timer
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class ReplyTweetsFetcher @Inject() (
stratoClient: Client,
timer: Timer,
stats: StatsReceiver)
extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentEngagedTweets] {
import ReplyTweetsFetcher._
override type RawSignalType = RecentEngagedTweet
override val name: String = this.getClass.getCanonicalName
override val statsReceiver: StatsReceiver = stats.scope(name)
override val stratoColumnPath: String =
TwistlyProfile.TwistlyProdProfile.userRecentEngagedStorePath
override val stratoView: Unit = None
override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType
override protected val viewConv: Conv[Unit] = Conv.ofType
override protected val valueConv: Conv[UserRecentEngagedTweets] =
ScroogeConv.fromStruct[UserRecentEngagedTweets]
override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, DefaultVersion)
override protected def toRawSignals(
userRecentEngagedTweets: UserRecentEngagedTweets
): Seq[RawSignalType] =
userRecentEngagedTweets.recentEngagedTweets
override def process(
query: Query,
rawSignals: Future[Option[Seq[RawSignalType]]]
): Future[Option[Seq[Signal]]] = {
rawSignals.map {
_.map { signals =>
val lookBackWindowFilteredSignals =
SignalFilter.lookBackWindow90DayFilter(signals, query.signalType)
lookBackWindowFilteredSignals
.collect {
case RecentEngagedTweet(tweetId, engagedAt, _: ReplyTweetMetadata, _) =>
Signal(query.signalType, engagedAt, Some(InternalId.TweetId(tweetId)))
}.take(query.maxResults.getOrElse(Int.MaxValue))
}
}
}
}
object ReplyTweetsFetcher {
// see com.twitter.twistly.store.UserRecentEngagedTweetsStore
private val DefaultVersion = 0
}

Some files were not shown because too many files have changed in this diff Show More