diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ZoneFilteringTest.docx b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ZoneFilteringTest.docx new file mode 100644 index 000000000..70a8cbd97 Binary files /dev/null and b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ZoneFilteringTest.docx differ diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ZoneFilteringTest.scala b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ZoneFilteringTest.scala deleted file mode 100644 index 02019fa6d..000000000 --- a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ZoneFilteringTest.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.inject.Test -import com.twitter.kafka.client.headers.ATLA -import com.twitter.kafka.client.headers.Implicits._ -import com.twitter.kafka.client.headers.PDXA -import com.twitter.kafka.client.headers.Zone -import com.twitter.unified_user_actions.service.module.ZoneFiltering -import com.twitter.util.mock.Mockito -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.junit.runner.RunWith -import org.scalatestplus.junit.JUnitRunner -import org.scalatest.prop.TableDrivenPropertyChecks - -@RunWith(classOf[JUnitRunner]) -class ZoneFilteringTest extends Test with Mockito with TableDrivenPropertyChecks { - trait Fixture { - val consumerRecord = - new ConsumerRecord[Array[Byte], Array[Byte]]("topic", 0, 0l, Array(0), Array(0)) - } - - test("two DCs filter") { - val zones = Table( - "zone", - Some(ATLA), - Some(PDXA), - None - ) - forEvery(zones) { localZoneOpt: Option[Zone] => - forEvery(zones) { headerZoneOpt: Option[Zone] => - localZoneOpt.foreach { localZone => - new Fixture { - headerZoneOpt match { - case Some(headerZone) => - consumerRecord.headers().setZone(headerZone) - if (headerZone == ATLA && localZone == ATLA) - ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe true - else if (headerZone == PDXA && localZone == PDXA) - ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe true - else - ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe false - case _ => - ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe true - } - } - } - } - } - } -} diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/BUILD b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/BUILD deleted file mode 100644 index a605859d2..000000000 --- a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -create_thrift_libraries( - org = "com.twitter", - base_name = "unified_user_actions", - sources = ["*.thrift"], - tags = ["bazel-compatible"], - dependency_roots = [ - "src/thrift/com/twitter/clientapp/gen:clientapp", - "src/thrift/com/twitter/gizmoduck:thrift", - "src/thrift/com/twitter/gizmoduck:user-thrift", - "src/thrift/com/twitter/search/common:constants", - "src/thrift/com/twitter/socialgraph:thrift", - ], - generate_languages = [ - "java", - "scala", - "strato", - ], - provides_java_name = "unified_user_actions-thrift-java", - provides_scala_name = "unified_user_actions-thrift-scala", -) diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/BUILD.docx b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/BUILD.docx new file mode 100644 index 000000000..4501c6c4c Binary files /dev/null and b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/BUILD.docx differ diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.docx b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.docx new file mode 100644 index 000000000..9e393baa2 Binary files /dev/null and b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.docx differ diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.thrift b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.thrift deleted file mode 100644 index 1342b5cf7..000000000 --- a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.thrift +++ /dev/null @@ -1,957 +0,0 @@ -namespace java com.twitter.unified_user_actions.thriftjava -#@namespace scala com.twitter.unified_user_actions.thriftscala -#@namespace strato com.twitter.unified_user_actions - -include "com/twitter/clientapp/gen/client_app.thrift" -include "com/twitter/reportflow/report_flow_logs.thrift" -include "com/twitter/socialgraph/social_graph_service_write_log.thrift" -include "com/twitter/gizmoduck/user_service.thrift" - -/* - * ActionType is typically a three part enum consisting of - * [Origin][Item Type][Action Name] - * - * [Origin] is usually "client" or "server" to indicate how the action was derived. - * - * [Item Type] is singular and refers to the shorthand version of the type of - * Item (e.g. Tweet, Profile, Notification instead of TweetInfo, ProfileInfo, NotificationInfo) - * the action occurred on. Action types and item types should be 1:1, and when an action can be - * performed on multiple types of items, consider granular action types. - * - * [Action Name] is the descriptive name of the user action (e.g. favorite, render impression); - * action names should correspond to UI actions / ML labels (which are typically based on user - * behavior from UI actions) - * - * Below are guidelines around naming of action types: - * a) When an action is coupled to a product surface, be concise in naming such that the - * combination of item type and action name captures the user behavior for the action in the UI. For example, - * for an open on a Notification in the PushNotification product surface that is parsed from client events, - * consider ClientNotificationOpen because the item Notification and the action name Open concisely represent - * the action, and the product surface PushNotification can be identified independently. - * - * b) It is OK to use generic names like Click if needed to distinguish from another action OR - * it is the best way to characterize an action concisely without confusion. - * For example, for ClientTweetClickReply, this refers to actually clicking on the Reply button but not - * Replying, and it is OK to include Click. Another example is Click on a Tweet anywhere (other than the fav, - * reply, etc. buttons), which leads to the TweetDetails page. Avoid generic action names like Click if - * there is a more specific UI aspect to reference and Click is implied, e.g. ClientTweetReport is - * preferred over ClientTweetClickReport and ClientTweetReportClick. - * - * c) Rely on versioning found in the origin when it is present for action names. For example, - * a "V2Impression" is named as such because in behavioral client events, there is - * a "v2Impress" field. See go/bce-v2impress for more details. - * - * d) There is a distinction between "UndoAction" and "Un{Action}" action types. - * An "UndoAction" is fired when a user clicks on the explicit "Undo" button, after they perform an action - * This "Undo" button is a UI element that may be temporary, e.g., - * - the user waited too long to click the button, the button disappears from the UI (e.g., Undo for Mute, Block) - * - the button does not disappear due to timeout, but becomes unavailable after the user closes a tab - * (e.g, Undo for NotInterestedIn, NotAboutTopic) - * Examples: - - ClientProfileUndoMute: a user clicks the "Undo" button after muting a Profile - - ClientTweetUndoNotInterestedIn: a users clicks the "Undo" button - after clicking "Not interested in this Tweet" button in the caret menu of a Tweet - * An "Un{Action}" is fired when a user reverses a previous action, not by explicitly clicking an "Undo" button, - * but through some other action that allows them to revert. - * Examples: - * - ClientProfileUnmute: a user clicks the "Unmute" button from the caret menu of the Profile they previously muted - * - ClientTweetUnfav: a user unlikes a tweet by clicking on like button again - * - * Examples: ServerTweetFav, ClientTweetRenderImpression, ClientNotificationSeeLessOften - * - * See go/uua-action-type for more details. - */ -enum ActionType { - // 0 - 999 used for actions derived from Server-side sources (e.g. Timelineservice, Tweetypie) - // NOTE: Please match values for corresponding server / client enum members (with offset 1000). - ServerTweetFav = 0 - ServerTweetUnfav = 1 - // Reserve 2 and 3 for ServerTweetLingerImpression and ServerTweetRenderImpression - - ServerTweetCreate = 4 - ServerTweetReply = 5 - ServerTweetQuote = 6 - ServerTweetRetweet = 7 - // skip 8-10 since there are no server equivalents for ClickCreate, ClickReply, ClickQuote - // reserve 11-16 for server video engagements - - ServerTweetDelete = 17 // User deletes a default tweet - ServerTweetUnreply = 18 // User deletes a reply tweet - ServerTweetUnquote = 19 // User deletes a quote tweet - ServerTweetUnretweet = 20 // User removes an existing retweet - // User edits a tweet. Edit will create a new tweet with editedTweetId = id of the original tweet - // The original tweet or the new tweet from edit can only be a default or quote tweet. - // A user can edit a default tweet to become a quote tweet (by adding the link to another Tweet), - // or edit a quote tweet to remove the quote and make it a default tweet. - // Both the initial tweet and the new tweet created from the edit can be edited, and each time the - // new edit will create a new tweet. All subsequent edits would have the same initial tweet id - // as the TweetInfo.editedTweetId. - // e.g. create Tweet A, edit Tweet A -> Tweet B, edit Tweet B -> Tweet C - // initial tweet id for both Tweet B anc Tweet C would be Tweet A - ServerTweetEdit = 21 - // skip 22 for delete an edit if we want to add it in the future - - // reserve 30-40 for server topic actions - - // 41-70 reserved for all negative engagements and the related positive engagements - // For example, Follow and Unfollow, Mute and Unmute - // This is fired when a user click "Submit" at the end of a "Report Tweet" flow - // ClientTweetReport = 1041 is scribed by HealthClient team, on the client side - // This is scribed by spamacaw, on the server side - // They can be joined on reportFlowId - // See https://confluence.twitter.biz/pages/viewpage.action?spaceKey=HEALTH&title=Understanding+ReportDetails - ServerTweetReport = 41 - - // reserve 42 for ServerTweetNotInterestedIn - // reserve 43 for ServerTweetUndoNotInterestedIn - // reserve 44 for ServerTweetNotAboutTopic - // reserve 45 for ServerTweetUndoNotAboutTopic - - ServerProfileFollow = 50 // User follows a Profile - ServerProfileUnfollow = 51 // User unfollows a Profile - ServerProfileBlock = 52 // User blocks a Profile - ServerProfileUnblock = 53 // User unblocks a Profile - ServerProfileMute = 54 // User mutes a Profile - ServerProfileUnmute = 55 // User unmutes a Profile - // User reports a Profile as Spam / Abuse - // This user action type includes ProfileReportAsSpam and ProfileReportAsAbuse - ServerProfileReport = 56 - // reserve 57 for ServerProfileUnReport - // reserve 56-70 for server social graph actions - - // 71-90 reserved for click-based events - // reserve 71 for ServerTweetClick - - // 1000 - 1999 used for actions derived from Client-side sources (e.g. Client Events, BCE) - // NOTE: Please match values for corresponding server / client enum members (with offset 1000). - // 1000 - 1499 used for legacy client events - ClientTweetFav = 1000 - ClientTweetUnfav = 1001 - ClientTweetLingerImpression = 1002 - // Please note that: Render impression for quoted Tweets would emit 2 events: - // 1 for the quoting Tweet and 1 for the original Tweet!!! - ClientTweetRenderImpression = 1003 - // 1004 reserved for ClientTweetCreate - // This is "Send Reply" event to indicate publishing of a reply Tweet as opposed to clicking - // on the reply button to initiate a reply Tweet (captured in ClientTweetClickReply). - // The differences between this and the ServerTweetReply are: - // 1) ServerTweetReply already has the new Tweet Id 2) A sent reply may be lost during transfer - // over the wire and thus may not end up with a follow-up ServerTweetReply. - ClientTweetReply = 1005 - // This is the "send quote" event to indicate publishing of a quote tweet as opposed to clicking - // on the quote button to initiate a quote tweet (captured in ClientTweetClickQuote). - // The differences between this and the ServerTweetQuote are: - // 1) ServerTweetQuote already has the new Tweet Id 2) A sent quote may be lost during transfer - // over the wire and thus may not end up with a follow-up ServerTweetQuote. - ClientTweetQuote = 1006 - // This is the "retweet" event to indicate publishing of a retweet. - ClientTweetRetweet = 1007 - // 1008 reserved for ClientTweetClickCreate - // This is user clicking on the Reply button not actually sending a reply Tweet, - // thus the name ClickReply - ClientTweetClickReply = 1009 - // This is user clicking the Quote/RetweetWithComment button not actually sending the quote, - // thus the name ClickQuote - ClientTweetClickQuote = 1010 - - // 1011 - 1016: Refer to go/cme-scribing and go/interaction-event-spec for details - // This is fired when playback reaches 25% of total track duration. Not valid for live videos. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoPlayback25 = 1011 - // This is fired when playback reaches 50% of total track duration. Not valid for live videos. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoPlayback50 = 1012 - // This is fired when playback reaches 75% of total track duration. Not valid for live videos. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoPlayback75 = 1013 - // This is fired when playback reaches 95% of total track duration. Not valid for live videos. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoPlayback95 = 1014 - // This if fired when the video has been played in non-preview - // (i.e. not autoplaying in the timeline) mode, and was not started via auto-advance. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoPlayFromTap = 1015 - // This is fired when 50% of the video has been on-screen and playing for 10 consecutive seconds - // or 95% of the video duration, whichever comes first. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoQualityView = 1016 - // Fired when either view_threshold or play_from_tap is fired. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoView = 1109 - // Fired when 50% of the video has been on-screen and playing for 2 consecutive seconds, - // regardless of video duration. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoMrcView = 1110 - // Fired when the video is: - // - Playing for 3 cumulative (not necessarily consecutive) seconds with 100% in view for looping video. - // - Playing for 3 cumulative (not necessarily consecutive) seconds or the video duration, whichever comes first, with 100% in view for non-looping video. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoViewThreshold = 1111 - // Fired when the user clicks a generic ‘visit url’ call to action. - ClientTweetVideoCtaUrlClick = 1112 - // Fired when the user clicks a ‘watch now’ call to action. - ClientTweetVideoCtaWatchClick = 1113 - - // 1017 reserved for ClientTweetDelete - // 1018-1019 for Client delete a reply and delete a quote if we want to add them in the future - - // This is fired when a user clicks on "Undo retweet" after re-tweeting a tweet - ClientTweetUnretweet = 1020 - // 1021 reserved for ClientTweetEdit - // 1022 reserved for Client delete an edit if we want to add it in the future - // This is fired when a user clicks on a photo within a tweet and the photo expands to fit - // the screen. - ClientTweetPhotoExpand = 1023 - - // This is fired when a user clicks on a profile mention inside a tweet. - ClientTweetClickMentionScreenName = 1024 - - // 1030 - 1035 for topic actions - // There are multiple cases: - // 1. Follow from the Topic page (or so-called landing page) - // 2. Click on Tweet's caret menu of "Follow (the topic)", it needs to be: - // 1) user follows the Topic already (otherwise there is no "Follow" menu by default), - // 2) and clicked on the "Unfollow Topic" first. - ClientTopicFollow = 1030 - // There are multiple cases: - // 1. Unfollow from the Topic page (or so-called landing page) - // 2. Click on Tweet's caret menu of "Unfollow (the topic)" if the user has already followed - // the topic. - ClientTopicUnfollow = 1031 - // This is fired when the user clicks the "x" icon next to the topic on their timeline, - // and clicks "Not interested in {TOPIC}" in the pop-up prompt - // Alternatively, they can also click "See more" button to visit the topic page, and click "Not interested" there. - ClientTopicNotInterestedIn = 1032 - // This is fired when the user clicks the "Undo" button after clicking "x" or "Not interested" on a Topic - // which is captured in ClientTopicNotInterestedIn - ClientTopicUndoNotInterestedIn = 1033 - - // 1036-1070 reserved for all negative engagements and the related positive engagements - // For example, Follow and Unfollow, Mute and Unmute - - // This is fired when a user clicks on "This Tweet's not helpful" flow in the caret menu - // of a Tweet result on the Search Results Page - ClientTweetNotHelpful = 1036 - // This is fired when a user clicks Undo after clicking on - // "This Tweet's not helpful" flow in the caret menu of a Tweet result on the Search Results Page - ClientTweetUndoNotHelpful = 1037 - // This is fired when a user starts and/or completes the "Report Tweet" flow in the caret menu of a Tweet - ClientTweetReport = 1041 - /* - * 1042-1045 refers to actions that are related to the - * "Not Interested In" button in the caret menu of a Tweet. - * - * ClientTweetNotInterestedIn is fired when a user clicks the - * "Not interested in this Tweet" button in the caret menu of a Tweet. - * A user can undo the ClientTweetNotInterestedIn action by clicking the - * "Undo" button that appears as a prompt in the caret menu, resulting - * in ClientTweetUndoNotInterestedIn being fired. - * If a user chooses to not undo and proceed, they are given multiple choices - * in a prompt to better document why they are not interested in a Tweet. - * For example, if a Tweet is not about a Topic, a user can click - * "This Tweet is not about {TOPIC}" in the provided prompt, resulting in - * in ClientTweetNotAboutTopic being fired. - * A user can undo the ClientTweetNotAboutTopic action by clicking the "Undo" - * button that appears as a subsequent prompt in the caret menu. Undoing this action - * results in the previous UI state, where the user had only marked "Not Interested In" and - * can still undo the original ClientTweetNotInterestedIn action. - * Similarly a user can select "This Tweet isn't recent" action resulting in ClientTweetNotRecent - * and he could undo this action immediately which results in ClientTweetUndoNotRecent - * Similarly a user can select "Show fewer tweets from" action resulting in ClientTweetSeeFewer - * and he could undo this action immediately which results in ClientTweetUndoSeeFewer - */ - ClientTweetNotInterestedIn = 1042 - ClientTweetUndoNotInterestedIn = 1043 - ClientTweetNotAboutTopic = 1044 - ClientTweetUndoNotAboutTopic = 1045 - ClientTweetNotRecent = 1046 - ClientTweetUndoNotRecent = 1047 - ClientTweetSeeFewer = 1048 - ClientTweetUndoSeeFewer = 1049 - - // This is fired when a user follows a profile from the - // profile page header / people module and people tab on the Search Results Page / sidebar on the Home page - // A Profile can also be followed when a user clicks follow in the caret menu of a Tweet - // or follow button on hovering on profile avatar, which is captured in ClientTweetFollowAuthor = 1060 - ClientProfileFollow = 1050 - // reserve 1050/1051 for client side Follow/Unfollow - // This is fired when a user clicks Block in a Profile page - // A Profile can also be blocked when a user clicks Block in the caret menu of a Tweet, - // which is captured in ClientTweetBlockAuthor = 1062 - ClientProfileBlock = 1052 - // This is fired when a user clicks unblock in a pop-up prompt right after blocking a profile - // in the profile page or clicks unblock in a drop-down menu in the profile page. - ClientProfileUnblock = 1053 - // This is fired when a user clicks Mute in a Profile page - // A Profile can also be muted when a user clicks Mute in the caret menu of a Tweet, which is captured in ClientTweetMuteAuthor = 1064 - ClientProfileMute = 1054 - // reserve 1055 for client side Unmute - // This is fired when a user clicks "Report User" action from user profile page - ClientProfileReport = 1056 - - // reserve 1057 for ClientProfileUnreport - - // This is fired when a user clicks on a profile from all modules except tweets - // (eg: People Search / people module in Top tab in Search Result Page - // For tweets, the click is captured in ClientTweetClickProfile - ClientProfileClick = 1058 - // reserve 1059-1070 for client social graph actions - - // This is fired when a user clicks Follow in the caret menu of a Tweet or hovers on the avatar of the tweet - // author and clicks on the Follow button. A profile can also be followed by clicking the Follow button on the - // Profile page and confirm, which is captured in ClientProfileFollow. The event emits two items, one of user type - // and another of tweet type, since the default implementation of BaseClientEvent only looks for Tweet type, - // the other item is dropped which is the expected behaviour - ClientTweetFollowAuthor = 1060 - - // This is fired when a user clicks Unfollow in the caret menu of a Tweet or hovers on the avatar of the tweet - // author and clicks on the Unfollow button. A profile can also be unfollowed by clicking the Unfollow button on the - // Profile page and confirm, which will be captured in ClientProfileUnfollow. The event emits two items, one of user type - // and another of tweet type, since the default implementation of BaseClientEvent only looks for Tweet type, - // the other item is dropped which is the expected behaviour - ClientTweetUnfollowAuthor = 1061 - - // This is fired when a user clicks Block in the menu of a Tweet to block the Profile that - // authored this Tweet. A Profile can also be blocked in the Profile page, which is captured - // in ClientProfileBlock = 1052 - ClientTweetBlockAuthor = 1062 - // This is fired when a user clicks unblock in a pop-up prompt right after blocking an author - // in the drop-down menu of a tweet - ClientTweetUnblockAuthor = 1063 - - // This is fired when a user clicks Mute in the menu of a Tweet to block the Profile that - // authored this Tweet. A Profile can also be muted in the Profile page, which is captured in ClientProfileMute = 1054 - ClientTweetMuteAuthor = 1064 - - // reserve 1065 for ClientTweetUnmuteAuthor - - // 1071-1090 reserved for click-based events - // click-based events are defined as clicks on a UI container (e.g., tweet, profile, etc.), as opposed to clearly named - // button or menu (e.g., follow, block, report, etc.), which requires a specific action name than "click". - - // This is fired when a user clicks on a Tweet to open the Tweet details page. Note that for - // Tweets in the Notification Tab product surface, a click can be registered differently - // depending on whether the Tweet is a rendered Tweet (a click results in ClientTweetClick) - // or a wrapper Notification (a click results in ClientNotificationClick). - ClientTweetClick = 1071 - // This is fired when a user clicks to view the profile page of a user from a tweet - // Contains a TweetInfo of this tweet - ClientTweetClickProfile = 1072 - // This is fired when a user clicks on the "share" icon on a Tweet to open the share menu. - // The user may or may not proceed and finish sharing the Tweet. - ClientTweetClickShare = 1073 - // This is fired when a user clicks "Copy link to Tweet" in a menu appeared after hitting - // the "share" icon on a Tweet OR when a user selects share_via -> copy_link after long-click - // a link inside a tweet on a mobile device - ClientTweetShareViaCopyLink = 1074 - // This is fired when a user clicks "Send via Direct Message" after - // clicking on the "share" icon on a Tweet to open the share menu. - // The user may or may not proceed and finish Sending the DM. - ClientTweetClickSendViaDirectMessage = 1075 - // This is fired when a user clicks "Bookmark" after - // clicking on the "share" icon on a Tweet to open the share menu. - ClientTweetShareViaBookmark = 1076 - // This is fired when a user clicks "Remove Tweet from Bookmarks" after - // clicking on the "share" icon on a Tweet to open the share menu. - ClientTweetUnbookmark = 1077 - // This is fired when a user clicks on the hashtag in a Tweet. - // The click on hashtag in "What's happening" section gives you other scribe '*:*:sidebar:*:trend:search' - // Currenly we are only filtering for itemType=Tweet. There are other items present in the event where itemType = user - // but those items are in dual-events (events with multiple itemTypes) and happen when you click on a hashtag in a Tweet from someone's profile, - // hence we are ignoring those itemType and only keeping itemType=Tweet. - ClientTweetClickHashtag = 1078 - // This is fired when a user clicks "Bookmark" after clicking on the "share" icon on a Tweet to open the share menu, or - // when a user clicks on the 'bookmark' icon on a Tweet (bookmark icon is available to ios only as of March 2023). - // TweetBookmark and TweetShareByBookmark log the same events but serve for individual use cases. - ClientTweetBookmark = 1079 - - // 1078 - 1089 for all Share related actions. - - // This is fired when a user clicks on a link in a tweet. - // The link could be displayed as a URL or embedded in a component such as an image or a card in a tweet. - ClientTweetOpenLink = 1090 - // This is fired when a user takes screenshot. - // This is available for mobile clients only. - ClientTweetTakeScreenshot = 1091 - - // 1100 - 1101: Refer to go/cme-scribing and go/interaction-event-spec for details - // Fired on the first tick of a track regardless of where in the video it is playing. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoPlaybackStart = 1100 - // Fired when playback reaches 100% of total track duration. - // Not valid for live videos. - // For looping playback, this is only fired once and does not reset at loop boundaries. - ClientTweetVideoPlaybackComplete = 1101 - - // A user can select "This Tweet isn't relevant" action resulting in ClientTweetNotRelevant - // and they could undo this action immediately which results in ClientTweetUndoNotRelevant - ClientTweetNotRelevant = 1102 - ClientTweetUndoNotRelevant = 1103 - - // A generic action type to submit feedback for different modules / items ( Tweets / Search Results ) - ClientFeedbackPromptSubmit = 1104 - - // This is fired when a user profile is open in a Profile page - ClientProfileShow = 1105 - - /* - * This is triggered when a user exits the Twitter platform. The amount of the time spent on the - * platform is recorded in ms that can be used to compute the User Active Seconds (UAS). - */ - ClientAppExit = 1106 - - /* - * For "card" related actions - */ - ClientCardClick = 1107 - ClientCardOpenApp = 1108 - ClientCardAppInstallAttempt = 1114 - ClientPollCardVote = 1115 - - /* - * The impressions 1121-1123 together with the ClientTweetRenderImpression 1003 are used by ViewCount - * and UnifiedEngagementCounts as EngagementType.Displayed and EngagementType.Details. - * - * For definitions, please refer to https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/common-internal/analytics/client-event-util/src/main/java/com/twitter/common_internal/analytics/client_event_util/TweetImpressionUtils.java?L14&subtree=true - */ - ClientTweetGalleryImpression = 1121 - ClientTweetDetailsImpression = 1122 - - /** - * This is fired when a user is logged out and follows a profile from the - * profile page / people module from web. - * One can only try to follow from web because iOS and Android do not support logged out browsing as of Jan 2023. - */ - ClientProfileFollowAttempt = 1200 - - /** - * This is fired when a user is logged out and favourite a tweet from web. - * One can only try to favourite from web, iOS and Android do not support logged out browsing - */ - ClientTweetFavoriteAttempt = 1201 - - /** - * This is fired when a user is logged out and Retweet a tweet from web. - * One can only try to favourite from web, iOS and Android do not support logged out browsing - */ - ClientTweetRetweetAttempt = 1202 - - /** - * This is fired when a user is logged out and reply on tweet from web. - * One can only try to favourite from web, iOS and Android do not support logged out browsing - */ - ClientTweetReplyAttempt = 1203 - - /** - * This is fired when a user is logged out and clicks on login button. - * Currently seem to be generated only on [m5, LiteNativeWrapper] - */ - ClientCTALoginClick = 1204 - /** - * This is fired when a user is logged out and login window is shown. - */ - ClientCTALoginStart = 1205 - /** - * This is fired when a user is logged out and login is successful. - */ - ClientCTALoginSuccess = 1206 - - /** - * This is fired when a user is logged out and clicks on signup button. - */ - ClientCTASignupClick = 1207 - - /** - * This is fired when a user is logged out and signup is successful. - */ - ClientCTASignupSuccess = 1208 - // 1400 - 1499 for product surface specific actions - // This is fired when a user opens a Push Notification - ClientNotificationOpen = 1400 - // This is fired when a user clicks on a Notification in the Notification Tab - ClientNotificationClick = 1401 - // This is fired when a user taps the "See Less Often" caret menu item of a Notification in the Notification Tab - ClientNotificationSeeLessOften = 1402 - // This is fired when a user closes or swipes away a Push Notification - ClientNotificationDismiss = 1403 - - // 1420 - 1439 is reserved for Search Results Page related actions - // 1440 - 1449 is reserved for Typeahead related actions - - // This is fired when a user clicks on a typeahead suggestion(queries, events, topics, users) - // in a drop-down menu of a search box or a tweet compose box. - ClientTypeaheadClick = 1440 - - // 1500 - 1999 used for behavioral client events - // Tweet related impressions - ClientTweetV2Impression = 1500 - /* Fullscreen impressions - * - * Android client will always log fullscreen_video impressions, regardless of the media type - * i.e. video, image, MM will all be logged as fullscreen_video - * - * iOS clients will log fullscreen_video or fullscreen_image depending on the media type - * on display when the user exits fullscreen. i.e. - * - image tweet => fullscreen_image - * - video tweet => fullscreen_video - * - MM tweet => fullscreen_video if user exits fullscreen from the video - * => fullscreen_image if user exits fullscreen from the image - * - * Web clients will always log fullscreen_image impressions, regardless of the media type - * - * References - * https://docs.google.com/document/d/1oEt9_Gtz34cmO_JWNag5YKKEq4Q7cJFL-nbHOmhnq1Y - * https://docs.google.com/document/d/1V_7TbfPvTQgtE_91r5SubD7n78JsVR_iToW59gOMrfQ - */ - ClientTweetVideoFullscreenV2Impression = 1501 - ClientTweetImageFullscreenV2Impression = 1502 - // Profile related impressions - ClientProfileV2Impression = 1600 - /* - * Email Notifications: These are actions taken by the user in response to Your Highlights email - * ClientTweetEmailClick refers to the action NotificationType.Click - */ - ClientTweetEmailClick = 5001 - - /* - * User create via Gizmoduck - */ - ServerUserCreate = 6000 - ServerUserUpdate = 6001 - /* - * Ads callback engagements - */ - /* - * This engagement is generated when a user Favs a promoted Tweet. - */ - ServerPromotedTweetFav = 7000 - /* - * This engagement is generated when a user Unfavs a promoted Tweet that they previously Faved. - */ - ServerPromotedTweetUnfav = 7001 - ServerPromotedTweetReply = 7002 - ServerPromotedTweetRetweet = 7004 - /* - * The block could be performed from the promoted tweet or on the promoted tweet's author's profile - * ads_spend_event data shows majority (~97%) of blocks have an associated promoted tweet id - * So for now we assume the blocks are largely performed from the tweet and following naming convention of ClientTweetBlockAuthor - */ - ServerPromotedTweetBlockAuthor = 7006 - ServerPromotedTweetUnblockAuthor = 7007 - /* - * This is when a user clicks on the Conversational Card in the Promoted Tweet which - * leads to the Tweet Compose page. The user may or may not send the new Tweet. - */ - ServerPromotedTweetComposeTweet = 7008 - /* - * This is when a user clicks on the Promoted Tweet to view its details/replies. - */ - ServerPromotedTweetClick = 7009 - /* - * The video ads engagements are divided into two sets: VIDEO_CONTENT_* and VIDEO_AD_*. These engagements - * have similar definitions. VIDEO_CONTENT_* engagements are fired for videos that are part of - * a Tweet. VIDEO_AD_* engagements are fired for a preroll ad. A preroll ad can play on a promoted - * Tweet or on an organic Tweet. go/preroll-matching for more information. - * - * 7011-7013: A Promoted Event is fired when playback reaches 25%, 50%, 75% of total track duration. - * This is for the video on a promoted Tweet. - * Not valid for live videos. Refer go/avscribing. - * For a video that has a preroll ad played before it, the metadata will contain information about - * the preroll ad as well as the video itself. There will be no preroll metadata if there was no - * preroll ad played. - */ - ServerPromotedTweetVideoPlayback25 = 7011 - ServerPromotedTweetVideoPlayback50 = 7012 - ServerPromotedTweetVideoPlayback75 = 7013 - /* - * This is when a user successfully completes the Report flow on a Promoted Tweet. - * It covers reports for all policies from Client Event. - */ - ServerPromotedTweetReport = 7041 - /* - * Follow from Ads data stream, it could be from both Tweet or other places - */ - ServerPromotedProfileFollow = 7060 - /* - * Follow from Ads data stream, it could be from both Tweet or other places - */ - ServerPromotedProfileUnfollow = 7061 - /* - * This is when a user clicks on the mute promoted tweet's author option from the menu. - */ - ServerPromotedTweetMuteAuthor = 7064 - /* - * This is fired when a user clicks on the profile image, screen name, or the user name of the - * author of the Promoted Tweet which leads to the author's profile page. - */ - ServerPromotedTweetClickProfile = 7072 - /* - * This is fired when a user clicks on a hashtag in the Promoted Tweet. - */ - ServerPromotedTweetClickHashtag = 7078 - /* - * This is fired when a user opens link by clicking on a URL in the Promoted Tweet. - */ - ServerPromotedTweetOpenLink = 7079 - /* - * This is fired when a user swipes to the next element of the carousel in the Promoted Tweet. - */ - ServerPromotedTweetCarouselSwipeNext = 7091 - /* - * This is fired when a user swipes to the previous element of the carousel in the Promoted Tweet. - */ - ServerPromotedTweetCarouselSwipePrevious = 7092 - /* - * This event is only for the Promoted Tweets with a web URL. - * It is fired after exiting a WebView from a Promoted Tweet if the user was on the WebView for - * at least 1 second. - * - * See https://confluence.twitter.biz/display/REVENUE/dwell_short for more details. - */ - ServerPromotedTweetLingerImpressionShort = 7093 - /* - * This event is only for the Promoted Tweets with a web URL. - * It is fired after exiting a WebView from a Promoted Tweet if the user was on the WebView for - * at least 2 seconds. - * - * See https://confluence.twitter.biz/display/REVENUE/dwell_medium for more details. - */ - ServerPromotedTweetLingerImpressionMedium = 7094 - /* - * This event is only for the Promoted Tweets with a web URL. - * It is fired after exiting a WebView from a Promoted Tweet if the user was on the WebView for - * at least 10 seconds. - * - * See https://confluence.twitter.biz/display/REVENUE/dwell_long for more details. - */ - ServerPromotedTweetLingerImpressionLong = 7095 - /* - * This is fired when a user navigates to explorer page (taps search magnifying glass on Home page) - * and a Promoted Trend is present and taps ON the promoted spotlight - a video/gif/image in the - * "hero" position (top of the explorer page). - */ - ServerPromotedTweetClickSpotlight = 7096 - /* - * This is fired when a user navigates to explorer page (taps search magnifying glass on Home page) - * and a Promoted Trend is present. - */ - ServerPromotedTweetViewSpotlight = 7097 - /* - * 7098-7099: Promoted Trends appear in the first or second slots of the “Trends for you” section - * in the Explore tab and “What’s Happening” module on Twitter.com. For more information, check go/ads-takeover. - * 7099: This is fired when a user views a promoted Trend. It should be considered as an impression. - */ - ServerPromotedTrendView = 7098 - /* - * 7099: This is fired when a user clicks a promoted Trend. It should be considered as an engagment. - */ - ServerPromotedTrendClick = 7099 - /* - * 7131-7133: A Promoted Event fired when playback reaches 25%, 50%, 75% of total track duration. - * This is for the preroll ad that plays before a video on a promoted Tweet. - * Not valid for live videos. Refer go/avscribing. - * This will only contain metadata for the preroll ad. - */ - ServerPromotedTweetVideoAdPlayback25 = 7131 - ServerPromotedTweetVideoAdPlayback50 = 7132 - ServerPromotedTweetVideoAdPlayback75 = 7133 - /* - * 7151-7153: A Promoted Event fired when playback reaches 25%, 50%, 75% of total track duration. - * This is for the preroll ad that plays before a video on an organic Tweet. - * Not valid for live videos. Refer go/avscribing. - * This will only contain metadata for the preroll ad. - */ - ServerTweetVideoAdPlayback25 = 7151 - ServerTweetVideoAdPlayback50 = 7152 - ServerTweetVideoAdPlayback75 = 7153 - - ServerPromotedTweetDismissWithoutReason = 7180 - ServerPromotedTweetDismissUninteresting = 7181 - ServerPromotedTweetDismissRepetitive = 7182 - ServerPromotedTweetDismissSpam = 7183 - - - /* - * For FavoriteArchival Events - */ - ServerTweetArchiveFavorite = 8000 - ServerTweetUnarchiveFavorite = 8001 - /* - * For RetweetArchival Events - */ - ServerTweetArchiveRetweet = 8002 - ServerTweetUnarchiveRetweet = 8003 -}(persisted='true', hasPersonalData='false') - -/* - * This union will be updated when we have a particular - * action that has attributes unique to that particular action - * (e.g. linger impressions have start/end times) and not common - * to all tweet actions. - * Naming convention for TweetActionInfo should be consistent with - * ActionType. For example, `ClientTweetLingerImpression` ActionType enum - * should correspond to `ClientTweetLingerImpression` TweetActionInfo union arm. - * We typically preserve 1:1 mapping between ActionType and TweetActionInfo. However, we make - * exceptions when optimizing for customer requirements. For example, multiple 'ClientTweetVideo*' - * ActionType enums correspond to a single `TweetVideoWatch` TweetActionInfo union arm because - * customers want individual action labels but common information across those labels. - */ -union TweetActionInfo { - // 41 matches enum index ServerTweetReport in ActionType - 41: ServerTweetReport serverTweetReport - // 1002 matches enum index ClientTweetLingerImpression in ActionType - 1002: ClientTweetLingerImpression clientTweetLingerImpression - // Common metadata for - // 1. "ClientTweetVideo*" ActionTypes with enum indices 1011-1016 and 1100-1101 - // 2. "ServerPromotedTweetVideo*" ActionTypes with enum indices 7011-7013 and 7131-7133 - // 3. "ServerTweetVideo*" ActionTypes with enum indices 7151-7153 - // This is because: - // 1. all the above listed ActionTypes share common metadata - // 2. more modular code as the same struct can be reused - // 3. reduces chance of error while populating and parsing the metadata - // 4. consumers can easily process the metadata - 1011: TweetVideoWatch tweetVideoWatch - // 1012: skip - // 1013: skip - // 1014: skip - // 1015: skip - // 1016: skip - // 1024 matches enum index ClientTweetClickMentionScreenName in ActionType - 1024: ClientTweetClickMentionScreenName clientTweetClickMentionScreenName - // 1041 matches enum index ClientTweetReport in ActionType - 1041: ClientTweetReport clientTweetReport - // 1060 matches enum index ClientTweetFollowAuthor in ActionType - 1060: ClientTweetFollowAuthor clientTweetFollowAuthor - // 1061 matches enum index ClientTweetUnfollowAuthor in ActionType - 1061: ClientTweetUnfollowAuthor clientTweetUnfollowAuthor - // 1078 matches enum index ClientTweetClickHashtag in ActionType - 1078: ClientTweetClickHashtag clientTweetClickHashtag - // 1090 matches enum index ClientTweetOpenLink in ActionType - 1090: ClientTweetOpenLink clientTweetOpenLink - // 1091 matches enum index ClientTweetTakeScreenshot in ActionType - 1091: ClientTweetTakeScreenshot clientTweetTakeScreenshot - // 1500 matches enum index ClientTweetV2Impression in ActionType - 1500: ClientTweetV2Impression clientTweetV2Impression - // 7079 matches enum index ServerPromotedTweetOpenLink in ActionType - 7079: ServerPromotedTweetOpenLink serverPromotedTweetOpenLink -}(persisted='true', hasPersonalData='true') - - -struct ClientTweetOpenLink { - //Url which was clicked. - 1: optional string url(personalDataType = 'RawUrlPath') -}(persisted='true', hasPersonalData='true') - -struct ServerPromotedTweetOpenLink { - //Url which was clicked. - 1: optional string url(personalDataType = 'RawUrlPath') -}(persisted='true', hasPersonalData='true') - -struct ClientTweetClickHashtag { - /* Hashtag string which was clicked. The PDP annotation is SearchQuery, - * because clicking on the hashtag triggers a search with the hashtag - */ - 1: optional string hashtag(personalDataType = 'SearchQuery') -}(persisted='true', hasPersonalData='true') - -struct ClientTweetTakeScreenshot { - //percentage visible height. - 1: optional i32 percentVisibleHeight100k -}(persisted='true', hasPersonalData='false') - -/* - * See go/ioslingerimpressionbehaviors and go/lingerandroidfaq - * for ios and android client definitions of a linger respectively. - */ -struct ClientTweetLingerImpression { - /* Milliseconds since epoch when the tweet became more than 50% visible. */ - 1: required i64 lingerStartTimestampMs(personalDataType = 'ImpressionMetadata') - /* Milliseconds since epoch when the tweet became less than 50% visible. */ - 2: required i64 lingerEndTimestampMs(personalDataType = 'ImpressionMetadata') -}(persisted='true', hasPersonalData='true') - -/* - * See go/behavioral-client-events for general behavioral client event (BCE) information - * and go/bce-v2impress for detailed information about BCE impression events. - * - * Unlike ClientTweetLingerImpression, there is no lower bound on the amount of time - * necessary for the impress event to occur. There is also no visibility requirement for a impress - * event to occur. - */ -struct ClientTweetV2Impression { - /* Milliseconds since epoch when the tweet became visible. */ - 1: required i64 impressStartTimestampMs(personalDataType = 'ImpressionMetadata') - /* Milliseconds since epoch when the tweet became visible. */ - 2: required i64 impressEndTimestampMs(personalDataType = 'ImpressionMetadata') - /* - * The UI component that hosted this tweet where the impress event happened. - * - * For example, sourceComponent = "tweet" if the impress event happened on a tweet displayed amongst - * a collection of tweets, or sourceComponent = "tweet_details" if the impress event happened on - * a tweet detail UI component. - */ - 3: required string sourceComponent(personalDataType = 'WebsitePage') -}(persisted='true', hasPersonalData='true') - - /* - * Refer to go/cme-scribing and go/interaction-event-spec for details - */ -struct TweetVideoWatch { - /* - * Type of video included in the Tweet - */ - 1: optional client_app.MediaType mediaType(personalDataType = 'MediaFile') - /* - * Whether the video content is "monetizable", i.e., - * if a preroll ad may be served dynamically when the video plays - */ - 2: optional bool isMonetizable(personalDataType = 'MediaFile') - - /* - * The owner of the video, provided by playlist. - * - * For ad engagements related to a preroll ad (VIDEO_AD_*), - * this will be the owner of the preroll ad and same as the prerollOwnerId. - * - * For ad engagements related to a regular video (VIDEO_CONTENT_*), this will be the owner of the - * video and not the preroll ad. - */ - 3: optional i64 videoOwnerId(personalDataType = 'UserId') - - /* - * Identifies the video associated with a card. - * - * For ad Engagements, in the case of engagements related to a preroll ad (VIDEO_AD_*), - * this will be the id of the preroll ad and same as the prerollUuid. - * - * For ad engagements related to a regular video (VIDEO_CONTENT_*), this will be id of the video - * and not the preroll ad. - */ - 4: optional string videoUuid(personalDataType = 'MediaId') - - /* - * Id of the preroll ad shown before the video - */ - 5: optional string prerollUuid(personalDataType = 'MediaId') - - /* - * Advertiser id of the preroll ad - */ - 6: optional i64 prerollOwnerId(personalDataType = 'UserId') - /* - * for amplify_flayer events, indicates whether preroll or the main video is being played - */ - 7: optional string videoType(personalDataType = 'MediaFile') -}(persisted='true', hasPersonalData='true') - -struct ClientTweetClickMentionScreenName { - /* Id for the profile (user_id) that was actioned on */ - 1: required i64 actionProfileId(personalDataType = 'UserId') - /* The handle/screenName of the user. This can't be changed. */ - 2: required string handle(personalDataType = 'UserName') -}(persisted='true', hasPersonalData='true') - -struct ClientTweetReport { - /* - * Whether the "Report Tweet" flow was successfully completed. - * `true` if the flow was completed successfully, `false` otherwise. - */ - 1: required bool isReportTweetDone - /* - * report-flow-id is included in Client Event when the "Report Tweet" flow was initiated - * See go/report-flow-ids and - * https://confluence.twitter.biz/pages/viewpage.action?spaceKey=HEALTH&title=Understanding+ReportDetails - */ - 2: optional string reportFlowId -}(persisted='true', hasPersonalData='true') - -enum TweetAuthorFollowClickSource { - UNKNOWN = 1 - CARET_MENU = 2 - PROFILE_IMAGE = 3 -} - -struct ClientTweetFollowAuthor { - /* - * Where did the user click the Follow button on the tweet - from the caret menu("CARET_MENU") - * or via hovering over the profile and clicking on Follow ("PROFILE_IMAGE") - only applicable for web clients - * "UNKNOWN" if the scribe do not match the expected namespace for the above - */ - 1: required TweetAuthorFollowClickSource followClickSource -}(persisted='true', hasPersonalData='false') - -enum TweetAuthorUnfollowClickSource { - UNKNOWN = 1 - CARET_MENU = 2 - PROFILE_IMAGE = 3 -} - -struct ClientTweetUnfollowAuthor { - /* - * Where did the user click the Unfollow button on the tweet - from the caret menu("CARET_MENU") - * or via hovering over the profile and clicking on Unfollow ("PROFILE_IMAGE") - only applicable for web clients - * "UNKNOWN" if the scribe do not match the expected namespace for the above - */ - 1: required TweetAuthorUnfollowClickSource unfollowClickSource -}(persisted='true', hasPersonalData='false') - -struct ServerTweetReport { - /* - * ReportDetails will be populated when the tweet report was scribed by spamacaw (server side) - * Only for the action submit, all the fields under ReportDetails will be available. - * This is because only after successful submission, we will know the report_type and report_flow_name. - * Reference: https://confluence.twitter.biz/pages/viewpage.action?spaceKey=HEALTH&title=Understanding+ReportDetails - */ - 1: optional string reportFlowId - 2: optional report_flow_logs.ReportType reportType -}(persisted='true', hasPersonalData='false') - -/* - * This union will be updated when we have a particular - * action that has attributes unique to that particular action - * (e.g. linger impressions have start/end times) and not common - * to other profile actions. - * - * Naming convention for ProfileActionInfo should be consistent with - * ActionType. For example, `ClientProfileV2Impression` ActionType enum - * should correspond to `ClientProfileV2Impression` ProfileActionInfo union arm. - */ -union ProfileActionInfo { - // 56 matches enum index ServerProfileReport in ActionType - 56: ServerProfileReport serverProfileReport - // 1600 matches enum index ClientProfileV2Impression in ActionType - 1600: ClientProfileV2Impression clientProfileV2Impression - // 6001 matches enum index ServerUserUpdate in ActionType - 6001: ServerUserUpdate serverUserUpdate -}(persisted='true', hasPersonalData='true') - -/* - * See go/behavioral-client-events for general behavioral client event (BCE) information - * and https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.3tu05p92xgxc - * for detailed information about BCE impression event. - * - * Unlike ClientTweetLingerImpression, there is no lower bound on the amount of time - * necessary for the impress event to occur. There is also no visibility requirement for a impress - * event to occur. - */ -struct ClientProfileV2Impression { - /* Milliseconds since epoch when the profile page became visible. */ - 1: required i64 impressStartTimestampMs(personalDataType = 'ImpressionMetadata') - /* Milliseconds since epoch when the profile page became visible. */ - 2: required i64 impressEndTimestampMs(personalDataType = 'ImpressionMetadata') - /* - * The UI component that hosted this profile where the impress event happened. - * - * For example, sourceComponent = "profile" if the impress event happened on a profile page - */ - 3: required string sourceComponent(personalDataType = 'WebsitePage') -}(persisted='true', hasPersonalData='true') - -struct ServerProfileReport { - 1: required social_graph_service_write_log.Action reportType(personalDataType = 'ReportType') -}(persisted='true', hasPersonalData='true') - -struct ServerUserUpdate { - 1: required list updates - 2: optional bool success (personalDataType = 'AuditMessage') -}(persisted='true', hasPersonalData='true') diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/common.docx b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/common.docx new file mode 100644 index 000000000..e90dfdc79 Binary files /dev/null and b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/common.docx differ diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/common.thrift b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/common.thrift deleted file mode 100644 index cf9efe063..000000000 --- a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/common.thrift +++ /dev/null @@ -1,20 +0,0 @@ -namespace java com.twitter.unified_user_actions.thriftjava -#@namespace scala com.twitter.unified_user_actions.thriftscala -#@namespace strato com.twitter.unified_user_actions - -/* - * Uniquely identifies a user. A user identifier - * for a logged in user should contain a user id - * and a user identifier for a logged out user should - * contain some guest id. A user may have multiple ids. - */ -struct UserIdentifier { - 1: optional i64 userId(personalDataType='UserId') - /* - * See http://go/guest-id-cookie-tdd. As of Dec 2021, - * guest id is intended only for essential use cases - * (e.g. logged out preferences, security). Guest id - * marketing is intended for recommendation use cases. - */ - 2: optional i64 guestIdMarketing(personalDataType='GuestId') -}(persisted='true', hasPersonalData='true') diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/item.docx b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/item.docx new file mode 100644 index 000000000..fc2f9782a Binary files /dev/null and b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/item.docx differ diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/item.thrift b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/item.thrift deleted file mode 100644 index c120e587c..000000000 --- a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/item.thrift +++ /dev/null @@ -1,294 +0,0 @@ -namespace java com.twitter.unified_user_actions.thriftjava -#@namespace scala com.twitter.unified_user_actions.thriftscala -#@namespace strato com.twitter.unified_user_actions - -include "com/twitter/unified_user_actions/action_info.thrift" -include "com/twitter/clientapp/gen/client_app.thrift" - -/* - * Tweet item information. Some development notes: - * 1. Please keep this top-level struct as minimal as possible to reduce overhead. - * 2. We intentionally avoid nesting action tweet in a separate structure - * to underscore its importance and faciliate extraction of most commonly - * needed fields such as actionTweetId. New fields related to the action tweet - * should generally be prefixed with "actionTweet". - * 3. For the related Tweets, e.g. retweetingTweetId, inReplyToTweetId, etc, we - * mostly only keep their ids for consistency and simplicity. - */ -struct TweetInfo { - - /* Id for the tweet that was actioned on */ - 1: required i64 actionTweetId(personalDataType = 'TweetId') - // Deprecated, please don't re-use! - // 2: optional i64 actionTweetAuthorId(personalDataType = 'UserId') - /* The social proof (i.e. banner) Topic Id that the action Tweet is associated to */ - 3: optional i64 actionTweetTopicSocialProofId(personalDataType='InferredInterests, ProvidedInterests') - 4: optional AuthorInfo actionTweetAuthorInfo - - // Fields 1-99 reserved for `actionFooBar` fields - - /* Additional details for the action that took place on actionTweetId */ - 100: optional action_info.TweetActionInfo tweetActionInfo - - /* Id of the tweet retweeting the action tweet */ - 101: optional i64 retweetingTweetId(personalDataType = 'TweetId') - /* Id of the tweet quoting the action Tweet, when the action type is quote */ - 102: optional i64 quotingTweetId(personalDataType = 'TweetId') - /* Id of the tweet replying to the action Tweet, when the action type is reply */ - 103: optional i64 replyingTweetId(personalDataType = 'TweetId') - /* Id of the tweet being quoted by the action tweet */ - 104: optional i64 quotedTweetId(personalDataType = 'TweetId') - /* Id of the tweet being replied to by the action tweet */ - 105: optional i64 inReplyToTweetId(personalDataType = 'TweetId') - /* Id of the tweet being retweeted by the action tweet, this is just for Unretweet action */ - 106: optional i64 retweetedTweetId(personalDataType = 'TweetId') - /* Id of the tweet being edited, this is only available for TweetEdit action, and TweetDelete - * action when the deleted tweet was created from Edit. */ - 107: optional i64 editedTweetId(personalDataType = 'TweetId') - /* Position of a tweet item in a page such as home and tweet detail, and is populated in - * Client Event. */ - 108: optional i32 tweetPosition - /* PromotedId is provided by ads team for each promoted tweet and is logged in client event */ - 109: optional string promotedId(personalDataType = 'AdsId') - /* corresponding to inReplyToTweetId */ - 110: optional i64 inReplyToAuthorId(personalDataType = 'UserId') - /* corresponding to retweetingTweetId */ - 111: optional i64 retweetingAuthorId(personalDataType = 'UserId') - /* corresponding to quotedTweetId */ - 112: optional i64 quotedAuthorId(personalDataType = 'UserId') -}(persisted='true', hasPersonalData='true') - -/* - * Profile item information. This follows TweetInfo's development notes. - */ -struct ProfileInfo { - - /* Id for the profile (user_id) that was actioned on - * - * In a social graph user action, e.g., user1 follows/blocks/mutes user2, - * userIdentifier captures userId of user1 and actionProfileId records - * the userId of user2. - */ - 1: required i64 actionProfileId(personalDataType = 'UserId') - - // Fields 1-99 reserved for `actionFooBar` fields - /* the full name of the user. max length is 50. */ - 2: optional string name(personalDataType = 'DisplayName') - /* The handle/screenName of the user. This can't be changed. - */ - 3: optional string handle(personalDataType = 'UserName') - /* the "bio" of the user. max length is 160. May contain one or more t.co - * links, which will be hydrated in the UrlEntities substruct if the - * QueryFields.URL_ENTITIES is specified. - */ - 4: optional string description(personalDataType = 'Bio') - - /* Additional details for the action that took place on actionProfileId */ - 100: optional action_info.ProfileActionInfo profileActionInfo -}(persisted='true', hasPersonalData='true') - -/* - * Topic item information. This follows TweetInfo's development notes. - */ -struct TopicInfo { - /* Id for the Topic that was actioned on */ - 1: required i64 actionTopicId(personalDataType='InferredInterests, ProvidedInterests') - - // Fields 1-99 reserved for `actionFooBar` fields -}(persisted='true', hasPersonalData='true') - -/* - * Notification Item information. - * - * See go/phab-d973370-discuss, go/phab-d968144-discuss, and go/uua-action-type for details about - * the schema design for Notification events. - */ -struct NotificationInfo { - /* - * Id of the Notification was actioned on. - * - * Note that this field represents the `impressionId` of a Notification. It has been renamed to - * `notificationId` in UUA so that the name effectively represents the value it holds, - * i.e. a unique id for a Notification and request. - */ - 1: required string actionNotificationId(personalDataType='UniversallyUniqueIdentifierUuid') - /* - * Additional information contained in a Notification. This is a `union` arm to differentiate - * among different types of Notifications and store relevant metadata for each type. - * - * For example, a Notification with a single Tweet will hold the Tweet id in `TweetNotification`. - * Similarly, `MultiTweetNotification` is defined for Notiifcations with multiple Tweet ids. - * - * Refer to the definition of `union NotificationContent` below for more details. - */ - 2: required NotificationContent content -}(persisted='true', hasPersonalData='true') - -/* - * Additional information contained in a Notification. - */ -union NotificationContent { - 1: TweetNotification tweetNotification - 2: MultiTweetNotification multiTweetNotification - - // 3 - 100 reserved for other specific Notification types (for example, profile, event, etc.). - - /* - * If a Notification cannot be categorized into any of the types at indices 1 - 100, - * it is considered of `Unknown` type. - */ - 101: UnknownNotification unknownNotification -}(persisted='true', hasPersonalData='true') - -/* - * Notification contains exactly one `tweetId`. - */ -struct TweetNotification { - 1: required i64 tweetId(personalDataType = 'TweetId') -}(persisted='true', hasPersonalData='true') - -/* - * Notification contains multiple `tweetIds`. - * For example, user A receives a Notification when user B likes multiple Tweets authored by user A. - */ -struct MultiTweetNotification { - 1: required list tweetIds(personalDataType = 'TweetId') -}(persisted='true', hasPersonalData='true') - -/* - * Notification could not be categrized into known types at indices 1 - 100 in `NotificationContent`. - */ -struct UnknownNotification { - // this field is just a placeholder since Sparrow doesn't support empty struct - 100: optional bool placeholder -}(persisted='true', hasPersonalData='false') - -/* - * Trend Item information for promoted and non-promoted Trends. - */ -struct TrendInfo { - /* - * Identifier for promoted Trends only. - * This is not available for non-promoted Trends and the default value should be set to 0. - */ - 1: required i32 actionTrendId(personalDataType= 'TrendId') - /* - * Empty for promoted Trends only. - * This should be set for all non-promoted Trends. - */ - 2: optional string actionTrendName -}(persisted='true', hasPersonalData='true') - -struct TypeaheadInfo { - /* search query string */ - 1: required string actionQuery(personalDataType = 'SearchQuery') - 2: required TypeaheadActionInfo typeaheadActionInfo -}(persisted='true', hasPersonalData='true') - -union TypeaheadActionInfo { - 1: UserResult userResult - 2: TopicQueryResult topicQueryResult -}(persisted='true', hasPersonalData='true') - -struct UserResult { - /* The userId of the profile suggested in the typeahead drop-down, upon which the user took the action */ - 1: required i64 profileId(personalDataType = 'UserId') -}(persisted='true', hasPersonalData='true') - -struct TopicQueryResult { - /* The topic query name suggested in the typeahead drop-down, upon which the user took the action */ - 1: required string suggestedTopicQuery(personalDataType = 'SearchQuery') -}(persisted='true', hasPersonalData='true') - - - -/* - * Item that captures feedback related information submitted by the user across modules / item (Eg: Search Results / Tweets) - * Design discussion doc: https://docs.google.com/document/d/1UHiCrGzfiXOSymRAUM565KchVLZBAByMwvP4ARxeixY/edit# - */ -struct FeedbackPromptInfo { - 1: required FeedbackPromptActionInfo feedbackPromptActionInfo -}(persisted='true', hasPersonalData='true') - -union FeedbackPromptActionInfo { - 1: DidYouFindItSearch didYouFindItSearch - 2: TweetRelevantToSearch tweetRelevantToSearch -}(persisted='true', hasPersonalData='true') - -struct DidYouFindItSearch { - 1: required string searchQuery(personalDataType= 'SearchQuery') - 2: optional bool isRelevant -}(persisted='true', hasPersonalData='true') - -struct TweetRelevantToSearch { - 1: required string searchQuery(personalDataType= 'SearchQuery') - 2: required i64 tweetId - 3: optional bool isRelevant -}(persisted='true', hasPersonalData='true') - -/* - * For (Tweet) Author info - */ -struct AuthorInfo { - /* In practice, this should be set. Rarely, it may be unset. */ - 1: optional i64 authorId(personalDataType = 'UserId') - /* i.e. in-network (true) or out-of-network (false) */ - 2: optional bool isFollowedByActingUser - /* i.e. is a follower (true) or not (false) */ - 3: optional bool isFollowingActingUser -}(persisted='true', hasPersonalData='true') - -/* - * Use for Call to Action events. - */ -struct CTAInfo { - // this field is just a placeholder since Sparrow doesn't support empty struct - 100: optional bool placeholder -}(persisted='true', hasPersonalData='false') - -/* - * Card Info - */ -struct CardInfo { - 1: optional i64 id - 2: optional client_app.ItemType itemType - // authorId is deprecated, please use AuthorInfo instead - // 3: optional i64 authorId(personalDataType = 'UserId') - 4: optional AuthorInfo actionTweetAuthorInfo -}(persisted='true', hasPersonalData='false') - -/* - * When the user exits the app, the time (in millis) spent by them on the platform is recorded as User Active Seconds (UAS). - */ -struct UASInfo { - 1: required i64 timeSpentMs -}(persisted='true', hasPersonalData='false') - -/* - * Corresponding item for a user action. - * An item should be treated independently if it has different affordances - * (https://www.interaction-design.org/literature/topics/affordances) for the user. - * For example, a Notification has different affordances than a Tweet in the Notification Tab; - * in the former, you can either "click" or "see less often" and in the latter, - * you can perform inline engagements such as "like" or "reply". - * Note that an item may be rendered differently in different contexts, but as long as the - * affordances remain the same or nearly similar, it can be treated as the same item - * (e.g. Tweets can be rendered in slightly different ways in embeds vs in the app). - * Item types (e.g. Tweets, Notifications) and ActionTypes should be 1:1, and when an action can be - * performed on multiple types of items, consider granular action types. - * For example, a user can take the Click action on Tweets and Notifications, and we have - * separate ActionTypes for Tweet Click and Notification Click. This makes it easier to identify all the - * actions associated with a particular item. - */ -union Item { - 1: TweetInfo tweetInfo - 2: ProfileInfo profileInfo - 3: TopicInfo topicInfo - 4: NotificationInfo notificationInfo - 5: TrendInfo trendInfo - 6: CTAInfo ctaInfo - 7: FeedbackPromptInfo feedbackPromptInfo - 8: TypeaheadInfo typeaheadInfo - 9: UASInfo uasInfo - 10: CardInfo cardInfo -}(persisted='true', hasPersonalData='true') diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/keyed_uua.docx b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/keyed_uua.docx new file mode 100644 index 000000000..dac3a64ed Binary files /dev/null and b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/keyed_uua.docx differ diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/keyed_uua.thrift b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/keyed_uua.thrift deleted file mode 100644 index 98c64609c..000000000 --- a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/keyed_uua.thrift +++ /dev/null @@ -1,22 +0,0 @@ -namespace java com.twitter.unified_user_actions.thriftjava -#@namespace scala com.twitter.unified_user_actions.thriftscala -#@namespace strato com.twitter.unified_user_actions - -include "com/twitter/unified_user_actions/action_info.thrift" -include "com/twitter/unified_user_actions/common.thrift" -include "com/twitter/unified_user_actions/metadata.thrift" - -/* - * This is mainly for View Counts project, which only require minimum fields for now. - * The name KeyedUuaTweet indicates the value is about a Tweet, not a Moment or other entities. - */ -struct KeyedUuaTweet { - /* A user refers to either a logged in / logged out user */ - 1: required common.UserIdentifier userIdentifier - /* The tweet that received the action from the user */ - 2: required i64 tweetId(personalDataType='TweetId') - /* The type of action which took place */ - 3: required action_info.ActionType actionType - /* Useful for event level analysis and joins */ - 4: required metadata.EventMetadata eventMetadata -}(persisted='true', hasPersonalData='true') diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/metadata.docx b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/metadata.docx new file mode 100644 index 000000000..2cf2e4915 Binary files /dev/null and b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/metadata.docx differ diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/metadata.thrift b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/metadata.thrift deleted file mode 100644 index 47644b6f8..000000000 --- a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/metadata.thrift +++ /dev/null @@ -1,177 +0,0 @@ -namespace java com.twitter.unified_user_actions.thriftjava -#@namespace scala com.twitter.unified_user_actions.thriftscala -#@namespace strato com.twitter.unified_user_actions - -/* Input source */ -enum SourceLineage { - /* Client-side. Also known as legacy client events or LCE. */ - ClientEvents = 0 - /* Client-side. Also known as BCE. */ - BehavioralClientEvents = 1 - /* Server-side Timelineservice favorites */ - ServerTlsFavs = 2 - /* Server-side Tweetypie events */ - ServerTweetypieEvents = 3 - /* Server-side SocialGraph events */ - ServerSocialGraphEvents = 4 - /* Notification Actions responding to Your Highlights Emails */ - EmailNotificationEvents = 5 - /** - * Gizmoduck's User Modification events https://docbird.twitter.biz/gizmoduck/user_modifications.html - **/ - ServerGizmoduckUserModificationEvents = 6 - /** - * Server-side Ads callback engagements - **/ - ServerAdsCallbackEngagements = 7 - /** - * Server-side favorite archival events - **/ - ServerFavoriteArchivalEvents = 8 - /** - * Server-side retweet archival events - **/ - ServerRetweetArchivalEvents = 9 -}(persisted='true', hasPersonalData='false') - -/* - * Only available in behavioral client events (BCE). - * - * A breadcrumb tweet is a tweet that was interacted with prior to the current action. - */ -struct BreadcrumbTweet { - /* Id for the tweet that was interacted with prior to the current action */ - 1: required i64 tweetId(personalDataType = 'TweetId') - /* - * The UI component that hosted the tweet and was interacted with preceeding to the current action. - * - tweet: represents the parent tweet container that wraps the quoted tweet - * - quote_tweet: represents the nested or quoted tweet within the parent container - * - * See more details - * https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.nb7tnjrhqxpm - */ - 2: required string sourceComponent(personalDataType = 'WebsitePage') -}(persisted='true', hasPersonalData='true') - -/* - * ClientEvent's namespaces. See https://docbird.twitter.biz/client_events/client-event-namespaces.html - * - * - For Legacy Client Events (LCE), it excludes the client part of the - * six part namespace (client:page:section:component:element:action) - * since this part is better captured by clientAppid and clientVersion. - * - * - For Behavioral Client Events (BCE), use clientPlatform to identify the client. - * Additionally, BCE contains an optional subsection to denote the UI component of - * the current action. The ClientEventNamespace.component field will be always empty for - * BCE namespace. There is no straightfoward 1-1 mapping between BCE and LCE namespace. - */ -struct ClientEventNamespace { - 1: optional string page(personalDataType = 'AppUsage') - 2: optional string section(personalDataType = 'AppUsage') - 3: optional string component(personalDataType = 'AppUsage') - 4: optional string element(personalDataType = 'AppUsage') - 5: optional string action(personalDataType = 'AppUsage') - 6: optional string subsection(personalDataType = 'AppUsage') -}(persisted='true', hasPersonalData='true') - -/* - * Metadata that is independent of a particular (user, item, action type) tuple - * and mostly shared across user action events. - */ -struct EventMetadata { - /* When the action happened according to whatever source we are reading from */ - 1: required i64 sourceTimestampMs(personalDataType = 'PrivateTimestamp, PublicTimestamp') - /* When the action was received for processing internally - * (compare with sourceTimestampMs for delay) - */ - 2: required i64 receivedTimestampMs - /* Which source is this event derived, e.g. CE, BCE, TimelineFavs */ - 3: required SourceLineage sourceLineage - /* To be deprecated and replaced by requestJoinId - * Useful for joining with other datasets - * */ - 4: optional i64 traceId(personalDataType = 'TfeTransactionId') - /* - * This is the language inferred from the request of the user action event (typically user's current client language) - * NOT the language of any Tweet, - * NOT the language that user sets in their profile!!! - * - * - ClientEvents && BehavioralClientEvents: Client UI language or from Gizmoduck which is what user set in Twitter App. - * Please see more at https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/finatra-internal/international/src/main/scala/com/twitter/finatra/international/LanguageIdentifier.scala - * The format should be ISO 639-1. - * - ServerTlsFavs: Client UI language, see more at http://go/languagepriority. The format should be ISO 639-1. - * - ServerTweetypieEvents: UUA sets this to None since there is no request level language info. - */ - 5: optional string language(personalDataType = 'InferredLanguage') - /* - * This is the country inferred from the request of the user action event (typically user's current country code) - * NOT the country of any Tweet (by geo-tagging), - * NOT the country set by the user in their profile!!! - * - * - ClientEvents && BehavioralClientEvents: Country code could be IP address (geoduck) or - * User registration country (gizmoduck) and the former takes precedence. - * We don’t know exactly which one is applied, unfortunately, - * see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/finatra-internal/international/src/main/scala/com/twitter/finatra/international/CountryIdentifier.scala - * The format should be ISO_3166-1_alpha-2. - * - ServerTlsFavs: From the request (user’s current location), - * see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/thrift/com/twitter/context/viewer.thrift?L54 - * The format should be ISO_3166-1_alpha-2. - * - ServerTweetypieEvents: - * UUA sets this to be consistent with IESource to meet existing use requirement. - * see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/thrift/com/twitter/tweetypie/tweet.thrift?L1001. - * The definitions here conflicts with the intention of UUA to log the request country code - * rather than the signup / geo-tagging country. - */ - 6: optional string countryCode(personalDataType = 'InferredCountry') - /* Useful for debugging client application related issues */ - 7: optional i64 clientAppId(personalDataType = 'AppId') - /* Useful for debugging client application related issues */ - 8: optional string clientVersion(personalDataType = 'ClientVersion') - /* Useful for filtering */ - 9: optional ClientEventNamespace clientEventNamespace - /* - * This field is only populated in behavioral client events (BCE). - * - * The client platform such as one of ["iPhone", "iPad", "Mac", "Android", "Web"] - * There can be multiple clientAppIds for the same platform. - */ - 10: optional string clientPlatform(personalDataType = 'ClientType') - /* - * This field is only populated in behavioral client events (BCE). - * - * The current UI hierarchy information with human readable labels. - * For example, [home,timeline,tweet] or [tab_bar,home,scrollable_content,tweet] - * - * For more details see https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.uv3md49i0j4j - */ - 11: optional list viewHierarchy(personalDataType = 'WebsitePage') - /* - * This field is only populated in behavioral client events (BCE). - * - * The sequence of views (breadcrumb) that was interacted with that caused the user to navigate to - * the current product surface (e.g. profile page) where an action was taken. - * - * The breadcrumb information may only be present for certain preceding product surfaces (e.g. Home Timeline). - * See more details in https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.nb7tnjrhqxpm - */ - 12: optional list breadcrumbViews(personalDataType = 'WebsitePage') - /* - * This field is only populated in behavioral client events (BCE). - * - * The sequence of tweets (breadcrumb) that was interacted with that caused the user to navigate to - * current product surface (e.g. profile page) where an action was taken. - * - * The breadcrumb information may only be present for certain preceding product surfaces (e.g. Home Timeline). - * See more details in https://docs.google.com/document/d/16CdSRpsmUUd17yoFH9min3nLBqDVawx4DaZoiqSfCHI/edit#heading=h.nb7tnjrhqxpm - */ - 13: optional list breadcrumbTweets(personalDataType = 'TweetId') - /* - * A request join id is created by backend services and broadcasted in subsequent calls - * to other downstream services as part of the request path. The requestJoinId is logged - * in server logs and scribed in client events, enabling joins across client and server - * as well as within a given request across backend servers. See go/joinkey-tdd for more - * details. - */ - 14: optional i64 requestJoinId(personalDataType = 'TransactionId') - 15: optional i64 clientEventTriggeredOn -}(persisted='true', hasPersonalData='true') diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/product_surface_info.docx b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/product_surface_info.docx new file mode 100644 index 000000000..f58aad252 Binary files /dev/null and b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/product_surface_info.docx differ diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/product_surface_info.thrift b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/product_surface_info.thrift deleted file mode 100644 index 524097885..000000000 --- a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/product_surface_info.thrift +++ /dev/null @@ -1,149 +0,0 @@ -#@namespace java com.twitter.unified_user_actions.thriftjava -#@namespace scala com.twitter.unified_user_actions.thriftscala -#@namespace strato com.twitter.unified_user_actions - -include "com/twitter/unified_user_actions/metadata.thrift" -include "com/twitter/search/common/constants/query.thrift" -include "com/twitter/search/common/constants/result.thrift" - - -/* - * Represents the product surface on which an action took place. - * See reference that delineates various product surfaces: - * https://docs.google.com/document/d/1PS2ZOyNUoUdO45zxhE7dH3L8KUcqJwo6Vx-XUGGFo6U - * Note: the implementation here may not reflect the above doc exactly. - */ -enum ProductSurface { - // 1 - 19 for Home - HomeTimeline = 1 - // 20 - 39 for Notifications - NotificationTab = 20 - PushNotification = 21 - EmailNotification = 22 - // 40 - 59 for Search - SearchResultsPage = 40 - SearchTypeahead = 41 - // 60 - 79 for Tweet Details Page (Conversation Page) - TweetDetailsPage = 60 - // 80 - 99 for Profile Page - ProfilePage = 80 - // 100 - 119 for ? - RESERVED_100 = 100 - // 120 - 139 for ? - RESERVED_120 = 120 -}(persisted='true', hasPersonalData='false') - -union ProductSurfaceInfo { - // 1 matches the enum index HomeTimeline in ProductSurface - 1: HomeTimelineInfo homeTimelineInfo - // 20 matches the enum index NotificationTab in ProductSurface - 20: NotificationTabInfo notificationTabInfo - // 21 matches the enum index PushNotification in ProductSurface - 21: PushNotificationInfo pushNotificationInfo - // 22 matches the enum index EmailNotification in ProductSurface - 22: EmailNotificationInfo emailNotificationInfo - // 40 matches the enum index SearchResultPage in ProductSurface - 40: SearchResultsPageInfo searchResultsPageInfo - // 41 matches the enum index SearchTypeahead in ProductSurface - 41: SearchTypeaheadInfo searchTypeaheadInfo - // 60 matches the enum index TweetDetailsPage in ProductSurface - 60: TweetDetailsPageInfo tweetDetailsPageInfo - // 80 matches the enum index ProfilePage in ProductSurface - 80: ProfilePageInfo profilePageInfo -}(persisted='true', hasPersonalData='false') - -/* - * Please keep this minimal to avoid overhead. It should only - * contain high value Home Timeline specific attributes. - */ -struct HomeTimelineInfo { - // suggestType is deprecated, please do't re-use! - // 1: optional i32 suggestType - 2: optional string suggestionType - 3: optional i32 injectedPosition -}(persisted='true', hasPersonalData='false') - -struct NotificationTabInfo { - /* - * Note that this field represents the `impressionId` in a Notification Tab notification. - * It has been renamed to `notificationId` in UUA so that the name effectively represents the - * value it holds, i.e., a unique id for a notification and request. - */ - 1: required string notificationId(personalDataType='UniversallyUniqueIdentifierUuid') -}(persisted='true', hasPersonalData='false') - -struct PushNotificationInfo { - /* - * Note that this field represents the `impressionId` in a Push Notification. - * It has been renamed to `notificationId` in UUA so that the name effectively represents the - * value it holds, i.e., a unique id for a notification and request. - */ - 1: required string notificationId(personalDataType='UniversallyUniqueIdentifierUuid') -}(persisted='true', hasPersonalData='false') - -struct EmailNotificationInfo { - /* - * Note that this field represents the `impressionId` in an Email Notification. - * It has been renamed to `notificationId` in UUA so that the name effectively represents the - * value it holds, i.e., a unique id for a notification and request. - */ - 1: required string notificationId(personalDataType='UniversallyUniqueIdentifierUuid') -}(persisted='true', hasPersonalData='false') - - -struct TweetDetailsPageInfo { - // To be deprecated, please don't re-use! - // Only reason to keep it now is Sparrow doesn't take empty struct. Once there is a real - // field we should just comment it out. - 1: required list breadcrumbViews(personalDataType = 'WebsitePage') - // Deprecated, please don't re-use! - // 2: required list breadcrumbTweets(personalDataType = 'TweetId') -}(persisted='true', hasPersonalData='true') - -struct ProfilePageInfo { - // To be deprecated, please don't re-use! - // Only reason to keep it now is Sparrow doesn't take empty struct. Once there is a real - // field we should just comment it out. - 1: required list breadcrumbViews(personalDataType = 'WebsitePage') - // Deprecated, please don't re-use! - // 2: required list breadcrumbTweets(personalDataType = 'TweetId') -}(persisted='true', hasPersonalData='true') - -struct SearchResultsPageInfo { - // search query string - 1: required string query(personalDataType = 'SearchQuery') - // Attribution of the search (e.g. Typed Query, Hashtag Click, etc.) - // see http://go/sgb/src/thrift/com/twitter/search/common/constants/query.thrift for details - 2: optional query.ThriftQuerySource querySource - // 0-indexed position of item in list of search results - 3: optional i32 itemPosition - // Attribution of the tweet result (e.g. QIG, Earlybird, etc) - // see http://go/sgb/src/thrift/com/twitter/search/common/constants/result.thrift for details - 4: optional set tweetResultSources - // Attribution of the user result (e.g. ExpertSearch, QIG, etc) - // see http://go/sgb/src/thrift/com/twitter/search/common/constants/result.thrift for details - 5: optional set userResultSources - // The query filter type on the Search Results Page (SRP) when the action took place. - // Clicking on a tab in SRP applies a query filter automatically. - 6: optional SearchQueryFilterType queryFilterType -}(persisted='true', hasPersonalData='true') - -struct SearchTypeaheadInfo { - // search query string - 1: required string query(personalDataType = 'SearchQuery') - // 0-indexed position of item in list of typeahead drop-down - 2: optional i32 itemPosition -}(persisted='true', hasPersonalData='true') - -enum SearchQueryFilterType { - // filter to top ranked content for a query - TOP = 1 - // filter to latest content for a query - LATEST = 2 - // filter to user results for a query - PEOPLE = 3 - // filter to photo tweet results for a query - PHOTOS = 4 - // filter to video tweet results for a query - VIDEOS = 5 -} diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/unified_user_actions.docx b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/unified_user_actions.docx new file mode 100644 index 000000000..8f11ba524 Binary files /dev/null and b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/unified_user_actions.docx differ diff --git a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/unified_user_actions.thrift b/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/unified_user_actions.thrift deleted file mode 100644 index d1a073b03..000000000 --- a/unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/unified_user_actions.thrift +++ /dev/null @@ -1,37 +0,0 @@ -namespace java com.twitter.unified_user_actions.thriftjava -#@namespace scala com.twitter.unified_user_actions.thriftscala -#@namespace strato com.twitter.unified_user_actions - -include "com/twitter/unified_user_actions/action_info.thrift" -include "com/twitter/unified_user_actions/common.thrift" -include "com/twitter/unified_user_actions/item.thrift" -include "com/twitter/unified_user_actions/metadata.thrift" -include "com/twitter/unified_user_actions/product_surface_info.thrift" - -/* - * A Unified User Action (UUA) is essentially a tuple of - * (user, item, action type, some metadata) with more optional - * information unique to product surfaces when available. - * It represents a user (logged in / out) taking some action (e.g. engagement, - * impression) on an item (e.g. tweet, profile). - */ -struct UnifiedUserAction { - /* A user refers to either a logged in / logged out user */ - 1: required common.UserIdentifier userIdentifier - /* The item that received the action from the user */ - 2: required item.Item item - /* The type of action which took place */ - 3: required action_info.ActionType actionType - /* Useful for event level analysis and joins */ - 4: required metadata.EventMetadata eventMetadata - /* - * Product surface on which the action occurred. If None, - * it means we can not capture the product surface (e.g. for server-side events). - */ - 5: optional product_surface_info.ProductSurface productSurface - /* - * Product specific information like join keys. If None, - * it means we can not capture the product surface information. - */ - 6: optional product_surface_info.ProductSurfaceInfo productSurfaceInfo -}(persisted='true', hasPersonalData='true') diff --git a/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/BUILD.bazel b/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/BUILD.bazel deleted file mode 100644 index 5295c7ead..000000000 --- a/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -create_thrift_libraries( - org = "com.twitter", - base_name = "unified_user_actions_spec", - sources = ["*.thrift"], - tags = ["bazel-compatible"], - dependency_roots = [ - ], - generate_languages = [ - "java", - "scala", - "strato", - ], - provides_java_name = "unified_user_actions_spec-thrift-java", - provides_scala_name = "unified_user_actions_spec-thrift-scala", -) diff --git a/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/BUILD.docx b/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/BUILD.docx new file mode 100644 index 000000000..d19d993b1 Binary files /dev/null and b/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/BUILD.docx differ diff --git a/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/unified_user_actions.docx b/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/unified_user_actions.docx new file mode 100644 index 000000000..fbb414c93 Binary files /dev/null and b/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/unified_user_actions.docx differ diff --git a/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/unified_user_actions.thrift b/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/unified_user_actions.thrift deleted file mode 100644 index 5ab129aaf..000000000 --- a/unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions/unified_user_actions.thrift +++ /dev/null @@ -1,11 +0,0 @@ -namespace java com.twitter.unified_user_actions.thriftjava -#@namespace scala com.twitter.unified_user_actions.thriftscala -#@namespace strato com.twitter.unified_user_actions - -/* Useful for testing UnifiedUserAction-like schema in tests */ -struct UnifiedUserActionSpec { - /* A user refers to either a logged out / logged in user */ - 1: required i64 userId - /* Arbitrary payload */ - 2: optional string payload -}(hasPersonalData='false') diff --git a/user-signal-service/README.docx b/user-signal-service/README.docx new file mode 100644 index 000000000..fdcc89ad4 Binary files /dev/null and b/user-signal-service/README.docx differ diff --git a/user-signal-service/README.md b/user-signal-service/README.md deleted file mode 100644 index d30568cf4..000000000 --- a/user-signal-service/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# User Signal Service # - -**User Signal Service** (USS) is a centralized online platform that supplies comprehensive data on user actions and behaviors on Twitter. This information encompasses both explicit signals, such as favoriting, retweeting, and replying, as well as implicit signals, including tweet clicks, video views, profile visits, and more. - -To ensure consistency and accuracy, USS gathers these signals from various underlying datasets and online services, processing them into uniform formats. These standardized source signals are then utilized in candidate retrieval and machine learning features for ranking stages. \ No newline at end of file diff --git a/user-signal-service/server/BUILD b/user-signal-service/server/BUILD deleted file mode 100644 index 76ff96764..000000000 --- a/user-signal-service/server/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -jvm_binary( - name = "bin", - basename = "user-signal-service", - main = "com.twitter.usersignalservice.UserSignalServiceStratoFedServerMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "strato/src/main/scala/com/twitter/strato/logging/logback", - "user-signal-service/server/src/main/resources", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice", - ], -) - -# Aurora Workflows build phase convention requires a jvm_app named with ${project-name}-app -jvm_app( - name = "user-signal-service-app", - archive = "zip", - binary = ":bin", -) diff --git a/user-signal-service/server/BUILD.docx b/user-signal-service/server/BUILD.docx new file mode 100644 index 000000000..bce43ee27 Binary files /dev/null and b/user-signal-service/server/BUILD.docx differ diff --git a/user-signal-service/server/src/main/resources/BUILD b/user-signal-service/server/src/main/resources/BUILD deleted file mode 100644 index b35d9c9d4..000000000 --- a/user-signal-service/server/src/main/resources/BUILD +++ /dev/null @@ -1,7 +0,0 @@ -resources( - sources = [ - "*.xml", - "*.yml", - "config/*.yml", - ], -) diff --git a/user-signal-service/server/src/main/resources/BUILD.docx b/user-signal-service/server/src/main/resources/BUILD.docx new file mode 100644 index 000000000..8ff1ead3b Binary files /dev/null and b/user-signal-service/server/src/main/resources/BUILD.docx differ diff --git a/user-signal-service/server/src/main/resources/config/decider.docx b/user-signal-service/server/src/main/resources/config/decider.docx new file mode 100644 index 000000000..d56084f68 Binary files /dev/null and b/user-signal-service/server/src/main/resources/config/decider.docx differ diff --git a/user-signal-service/server/src/main/resources/config/decider.yml b/user-signal-service/server/src/main/resources/config/decider.yml deleted file mode 100644 index f22a9dc22..000000000 --- a/user-signal-service/server/src/main/resources/config/decider.yml +++ /dev/null @@ -1,6 +0,0 @@ -test_value: - comment: Test Value - default_availability: 10000 -dark_traffic_percent: - comment: Percentage of traffic to send to dark traffic destination - default_availability: 0 \ No newline at end of file diff --git a/user-signal-service/server/src/main/resources/logback.docx b/user-signal-service/server/src/main/resources/logback.docx new file mode 100644 index 000000000..851e05c5c Binary files /dev/null and b/user-signal-service/server/src/main/resources/logback.docx differ diff --git a/user-signal-service/server/src/main/resources/logback.xml b/user-signal-service/server/src/main/resources/logback.xml deleted file mode 100644 index 6511278df..000000000 --- a/user-signal-service/server/src/main/resources/logback.xml +++ /dev/null @@ -1,155 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - true - - - - - ${log.service.output} - - ${log.service.output}.%i - 1 - 10 - - - 50MB - - - %date %.-3level ${DEFAULT_SERVICE_PATTERN}%n - - - - - - ${log.strato_only.output} - - ${log.strato_only.output}.%i - 1 - 10 - - - 50MB - - - %date %.-3level ${DEFAULT_SERVICE_PATTERN}%n - - - - - - true - loglens - ${log.lens.index} - ${log.lens.tag}/service - - %msg%n - - - 500 - 50 - - - manhattan-client - .*InvalidRequest.* - - - - - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/BUILD b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/BUILD deleted file mode 100644 index 248fff64b..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config", - ], -) diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/BUILD.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/BUILD.docx new file mode 100644 index 000000000..7e36bbc6f Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/BUILD.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/UserSignalServiceStratoFedServerMain.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/UserSignalServiceStratoFedServerMain.docx new file mode 100644 index 000000000..09435bef1 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/UserSignalServiceStratoFedServerMain.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/UserSignalServiceStratoFedServerMain.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/UserSignalServiceStratoFedServerMain.scala deleted file mode 100644 index 878310abb..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/UserSignalServiceStratoFedServerMain.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.usersignalservice - -import com.google.inject.Module -import com.twitter.inject.thrift.modules.ThriftClientIdModule -import com.twitter.usersignalservice.columns.UserSignalServiceColumn -import com.twitter.strato.fed._ -import com.twitter.strato.fed.server._ -import com.twitter.usersignalservice.module.CacheModule -import com.twitter.usersignalservice.module.MHMtlsParamsModule -import com.twitter.usersignalservice.module.SocialGraphServiceClientModule -import com.twitter.usersignalservice.module.TimerModule - -object UserSignalServiceStratoFedServerMain extends UserSignalServiceStratoFedServer - -trait UserSignalServiceStratoFedServer extends StratoFedServer { - override def dest: String = "/s/user-signal-service/user-signal-service" - - override def columns: Seq[Class[_ <: StratoFed.Column]] = - Seq( - classOf[UserSignalServiceColumn] - ) - - override def modules: Seq[Module] = - Seq( - CacheModule, - MHMtlsParamsModule, - SocialGraphServiceClientModule, - ThriftClientIdModule, - TimerModule, - ) - -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/AggregatedSignalController.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/AggregatedSignalController.docx new file mode 100644 index 000000000..49cb2a5af Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/AggregatedSignalController.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/AggregatedSignalController.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/AggregatedSignalController.scala deleted file mode 100644 index fb698b01a..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/AggregatedSignalController.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.usersignalservice.base - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats -import com.twitter.storehaus.ReadableStore -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.base.BaseSignalFetcher.Timeout -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer - -case class AggregatedSignalController( - signalsAggregationInfo: Seq[SignalAggregatedInfo], - signalsWeightMapInfo: Map[SignalType, Double], - stats: StatsReceiver, - timer: Timer) - extends ReadableStore[Query, Seq[Signal]] { - - val name: String = this.getClass.getCanonicalName - val statsReceiver: StatsReceiver = stats.scope(name) - - override def get(query: Query): Future[Option[Seq[Signal]]] = { - Stats - .trackItems(statsReceiver) { - val allSignalsFut = - Future - .collect(signalsAggregationInfo.map(_.getSignals(query.userId))).map(_.flatten.flatten) - val aggregatedSignals = - allSignalsFut.map { allSignals => - allSignals - .groupBy(_.targetInternalId).collect { - case (Some(internalId), signals) => - val mostRecentEnagementTime = signals.map(_.timestamp).max - val totalWeight = - signals - .map(signal => signalsWeightMapInfo.getOrElse(signal.signalType, 0.0)).sum - (Signal(query.signalType, mostRecentEnagementTime, Some(internalId)), totalWeight) - }.toSeq.sortBy { case (signal, weight) => (-weight, -signal.timestamp) } - .map(_._1) - .take(query.maxResults.getOrElse(Int.MaxValue)) - } - aggregatedSignals.map(Some(_)) - }.raiseWithin(Timeout)(timer).handle { - case e => - statsReceiver.counter(e.getClass.getCanonicalName).incr() - Some(Seq.empty[Signal]) - } - } -} - -case class SignalAggregatedInfo( - signalType: SignalType, - signalFetcher: ReadableStore[Query, Seq[Signal]]) { - def getSignals(userId: UserId): Future[Option[Seq[Signal]]] = { - signalFetcher.get(Query(userId, signalType, None)) - } -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BUILD b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BUILD deleted file mode 100644 index 83bb0aa3e..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/src/jvm/com/twitter/storehaus:core", - "finagle/finagle-stats", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection", - "src/scala/com/twitter/storehaus_internal/manhattan", - "src/scala/com/twitter/twistly/common", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BUILD.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BUILD.docx new file mode 100644 index 000000000..510a2db3c Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BUILD.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BaseSignalFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BaseSignalFetcher.docx new file mode 100644 index 000000000..7ccaa478a Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BaseSignalFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BaseSignalFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BaseSignalFetcher.scala deleted file mode 100644 index 27646b9cc..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/BaseSignalFetcher.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.usersignalservice -package base - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus.ReadableStore -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.util.Future -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.frigate.common.base.Stats -import com.twitter.conversions.DurationOps._ -import com.twitter.usersignalservice.thriftscala.ClientIdentifier -import com.twitter.util.Duration -import com.twitter.util.Timer -import java.io.Serializable - -case class Query( - userId: UserId, - signalType: SignalType, - maxResults: Option[Int], - clientId: ClientIdentifier = ClientIdentifier.Unknown) - -/** - * A trait that defines a standard interface for the signal fetcher - * - * Extends this only when all other traits extending BaseSignalFetcher do not apply to - * your use case. - */ -trait BaseSignalFetcher extends ReadableStore[Query, Seq[Signal]] { - import BaseSignalFetcher._ - - /** - * This RawSignalType is the output type of `getRawSignals` and the input type of `process`. - * Override it as your own raw signal type to maintain meta data which can be used in the - * step of `process`. - * Note that the RawSignalType is an intermediate data type intended to be small to avoid - * big data chunks being passed over functions or being memcached. - */ - type RawSignalType <: Serializable - - def name: String - def statsReceiver: StatsReceiver - def timer: Timer - - /** - * This function is called by the top level class to fetch signals. It executes the pipeline to - * fetch raw signals, process and transform the signals. Exceptions and timeout control are - * handled here. - * @param query - * @return Future[Option[Seq[Signal]]] - */ - override def get(query: Query): Future[Option[Seq[Signal]]] = { - val clientStatsReceiver = statsReceiver.scope(query.clientId.name).scope(query.signalType.name) - Stats - .trackItems(clientStatsReceiver) { - val rawSignals = getRawSignals(query.userId) - val signals = process(query, rawSignals) - signals - }.raiseWithin(Timeout)(timer).handle { - case e => - clientStatsReceiver.scope("FetcherExceptions").counter(e.getClass.getCanonicalName).incr() - EmptyResponse - } - } - - /** - * Override this function to define how to fetch the raw signals from any store - * Note that the RawSignalType is an intermediate data type intended to be small to avoid - * big data chunks being passed over functions or being memcached. - * @param userId - * @return Future[Option[Seq[RawSignalType]]] - */ - def getRawSignals(userId: UserId): Future[Option[Seq[RawSignalType]]] - - /** - * Override this function to define how to process the raw signals and transform them to signals. - * @param query - * @param rawSignals - * @return Future[Option[Seq[Signal]]] - */ - def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] -} - -object BaseSignalFetcher { - val Timeout: Duration = 20.milliseconds - val EmptyResponse: Option[Seq[Signal]] = Some(Seq.empty[Signal]) -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/FilteredSignalFetcherController.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/FilteredSignalFetcherController.docx new file mode 100644 index 000000000..ae214e370 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/FilteredSignalFetcherController.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/FilteredSignalFetcherController.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/FilteredSignalFetcherController.scala deleted file mode 100644 index e2e0e96fe..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/FilteredSignalFetcherController.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.usersignalservice.base - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats -import com.twitter.storehaus.ReadableStore -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer - -/** - * Combine a BaseSignalFetcher with a map of negative signalFetchers. Filter out the negative - * signals from the signals from BaseSignalFetcher. - */ -case class FilteredSignalFetcherController( - backingSignalFetcher: BaseSignalFetcher, - originSignalType: SignalType, - stats: StatsReceiver, - timer: Timer, - filterSignalFetchers: Map[SignalType, BaseSignalFetcher] = - Map.empty[SignalType, BaseSignalFetcher]) - extends ReadableStore[Query, Seq[Signal]] { - val statsReceiver: StatsReceiver = stats.scope(this.getClass.getCanonicalName) - - override def get(query: Query): Future[Option[Seq[Signal]]] = { - val clientStatsReceiver = statsReceiver.scope(query.signalType.name).scope(query.clientId.name) - Stats - .trackItems(clientStatsReceiver) { - val backingSignals = - backingSignalFetcher.get(Query(query.userId, originSignalType, None, query.clientId)) - val filteredSignals = filter(query, backingSignals) - filteredSignals - }.raiseWithin(BaseSignalFetcher.Timeout)(timer).handle { - case e => - clientStatsReceiver.scope("FetcherExceptions").counter(e.getClass.getCanonicalName).incr() - BaseSignalFetcher.EmptyResponse - } - } - - def filter( - query: Query, - rawSignals: Future[Option[Seq[Signal]]] - ): Future[Option[Seq[Signal]]] = { - Stats - .trackItems(statsReceiver) { - val originSignals = rawSignals.map(_.getOrElse(Seq.empty[Signal])) - val filterSignals = - Future - .collect { - filterSignalFetchers.map { - case (signalType, signalFetcher) => - signalFetcher - .get(Query(query.userId, signalType, None, query.clientId)) - .map(_.getOrElse(Seq.empty)) - }.toSeq - }.map(_.flatten.toSet) - val filterSignalsSet = filterSignals - .map(_.flatMap(_.targetInternalId)) - - val originSignalsWithId = - originSignals.map(_.map(signal => (signal, signal.targetInternalId))) - Future.join(originSignalsWithId, filterSignalsSet).map { - case (originSignalsWithId, filterSignalsSet) => - Some( - originSignalsWithId - .collect { - case (signal, internalIdOpt) - if internalIdOpt.nonEmpty && !filterSignalsSet.contains(internalIdOpt.get) => - signal - }.take(query.maxResults.getOrElse(Int.MaxValue))) - } - } - } - -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/ManhattanSignalFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/ManhattanSignalFetcher.docx new file mode 100644 index 000000000..3cc17df73 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/ManhattanSignalFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/ManhattanSignalFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/ManhattanSignalFetcher.scala deleted file mode 100644 index d0918a165..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/ManhattanSignalFetcher.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.usersignalservice -package base - -import com.twitter.bijection.Codec -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.ManhattanCluster -import com.twitter.storehaus_internal.manhattan.ManhattanRO -import com.twitter.storehaus_internal.manhattan.ManhattanROConfig -import com.twitter.storehaus_internal.util.HDFSPath -import com.twitter.twistly.common.UserId -import com.twitter.util.Future -import com.twitter.storehaus_internal.util.ApplicationID -import com.twitter.storehaus_internal.util.DatasetName - -/** - * A Manhattan signal fetcher extending BaseSignalFetcher to provide an interface to fetch signals - * from a Manhattan dataset. - * - * Extends this when the underlying store is a single Manhattan dataset. - * @tparam ManhattanKeyType - * @tparam ManhattanValueType - */ -trait ManhattanSignalFetcher[ManhattanKeyType, ManhattanValueType] extends BaseSignalFetcher { - /* - Define the meta info of the Manhattan dataset - */ - protected def manhattanAppId: String - protected def manhattanDatasetName: String - protected def manhattanClusterId: ManhattanCluster - protected def manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams - - protected def manhattanKeyCodec: Codec[ManhattanKeyType] - protected def manhattanRawSignalCodec: Codec[ManhattanValueType] - - /** - * Adaptor to transform the userId to the ManhattanKey - * @param userId - * @return ManhattanKeyType - */ - protected def toManhattanKey(userId: UserId): ManhattanKeyType - - /** - * Adaptor to transform the ManhattanValue to the Seq of RawSignalType - * @param manhattanValue - * @return Seq[RawSignalType] - */ - protected def toRawSignals(manhattanValue: ManhattanValueType): Seq[RawSignalType] - - protected final lazy val underlyingStore: ReadableStore[UserId, Seq[RawSignalType]] = { - ManhattanRO - .getReadableStoreWithMtls[ManhattanKeyType, ManhattanValueType]( - ManhattanROConfig( - HDFSPath(""), - ApplicationID(manhattanAppId), - DatasetName(manhattanDatasetName), - manhattanClusterId), - manhattanKVClientMtlsParams - )(manhattanKeyCodec, manhattanRawSignalCodec) - .composeKeyMapping(userId => toManhattanKey(userId)) - .mapValues(manhattanRawSignal => toRawSignals(manhattanRawSignal)) - } - - override final def getRawSignals(userId: UserId): Future[Option[Seq[RawSignalType]]] = - underlyingStore.get(userId) -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/MemcachedSignalFetcherWrapper.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/MemcachedSignalFetcherWrapper.docx new file mode 100644 index 000000000..6d755ee4e Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/MemcachedSignalFetcherWrapper.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/MemcachedSignalFetcherWrapper.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/MemcachedSignalFetcherWrapper.scala deleted file mode 100644 index 4022d9021..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/MemcachedSignalFetcherWrapper.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.usersignalservice -package base - -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.injection.SeqObjectInjection -import com.twitter.storehaus.ReadableStore -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Timer - -/** - * Use this wrapper when the latency of the signal fetcher is too high (see BaseSignalFetcher.Timeout - * ) and the results from the signal fetcher don't change often (e.g. results are generated from a - * scalding job scheduled each day). - * @param memcachedClient - * @param baseSignalFetcher - * @param ttl - * @param stats - * @param timer - */ -case class MemcachedSignalFetcherWrapper( - memcachedClient: MemcachedClient, - baseSignalFetcher: BaseSignalFetcher, - ttl: Duration, - stats: StatsReceiver, - keyPrefix: String, - timer: Timer) - extends BaseSignalFetcher { - import MemcachedSignalFetcherWrapper._ - override type RawSignalType = baseSignalFetcher.RawSignalType - - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name).scope(baseSignalFetcher.name) - - val underlyingStore: ReadableStore[UserId, Seq[RawSignalType]] = { - val cacheUnderlyingStore = new ReadableStore[UserId, Seq[RawSignalType]] { - override def get(userId: UserId): Future[Option[Seq[RawSignalType]]] = - baseSignalFetcher.getRawSignals(userId) - } - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = cacheUnderlyingStore, - cacheClient = memcachedClient, - ttl = ttl)( - valueInjection = LZ4Injection.compose(SeqObjectInjection[RawSignalType]()), - statsReceiver = statsReceiver, - keyToString = { k: UserId => - s"$keyPrefix:${keyHasher.hashKey(k.toString.getBytes)}" - } - ) - } - - override def getRawSignals(userId: UserId): Future[Option[Seq[RawSignalType]]] = - underlyingStore.get(userId) - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = baseSignalFetcher.process(query, rawSignals) - -} - -object MemcachedSignalFetcherWrapper { - private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/StratoSignalFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/StratoSignalFetcher.docx new file mode 100644 index 000000000..129899dc8 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/StratoSignalFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/StratoSignalFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/StratoSignalFetcher.scala deleted file mode 100644 index 2d0de84b6..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base/StratoSignalFetcher.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.usersignalservice -package base -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.twistly.common.UserId -import com.twitter.util.Future - -/** - * A Strato signal fetcher extending BaseSignalFetcher to provide an interface to fetch signals from - * Strato Column. - * - * Extends this when the underlying store is a single Strato column. - * @tparam StratoKeyType - * @tparam StratoViewType - * @tparam StratoValueType - */ -trait StratoSignalFetcher[StratoKeyType, StratoViewType, StratoValueType] - extends BaseSignalFetcher { - /* - Define the meta info of the strato column - */ - def stratoClient: Client - def stratoColumnPath: String - def stratoView: StratoViewType - - /** - * Override these vals and remove the implicit key words. - * @return - */ - protected implicit def keyConv: Conv[StratoKeyType] - protected implicit def viewConv: Conv[StratoViewType] - protected implicit def valueConv: Conv[StratoValueType] - - /** - * Adapter to transform the userId to the StratoKeyType - * @param userId - * @return StratoKeyType - */ - protected def toStratoKey(userId: UserId): StratoKeyType - - /** - * Adapter to transform the StratoValueType to a Seq of RawSignalType - * @param stratoValue - * @return Seq[RawSignalType] - */ - protected def toRawSignals(stratoValue: StratoValueType): Seq[RawSignalType] - - protected final lazy val underlyingStore: ReadableStore[UserId, Seq[RawSignalType]] = - StratoFetchableStore - .withView[StratoKeyType, StratoViewType, StratoValueType]( - stratoClient, - stratoColumnPath, - stratoView) - .composeKeyMapping(toStratoKey) - .mapValues(toRawSignals) - - override final def getRawSignals(userId: UserId): Future[Option[Seq[RawSignalType]]] = - underlyingStore.get(userId) -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/BUILD b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/BUILD deleted file mode 100644 index 1cb85f732..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "src/scala/com/twitter/twistly/common", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/BUILD.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/BUILD.docx new file mode 100644 index 000000000..e499b9718 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/BUILD.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/UserSignalServiceColumn.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/UserSignalServiceColumn.docx new file mode 100644 index 000000000..c1cdd4ca3 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/UserSignalServiceColumn.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/UserSignalServiceColumn.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/UserSignalServiceColumn.scala deleted file mode 100644 index aea92ecd1..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/columns/UserSignalServiceColumn.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.usersignalservice.columns - -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.strato.catalog.OpMetadata -import com.twitter.strato.catalog.Ops -import com.twitter.strato.config.Policy -import com.twitter.strato.config.ReadWritePolicy -import com.twitter.strato.data.Conv -import com.twitter.strato.data.Description -import com.twitter.strato.data.Lifecycle -import com.twitter.strato.fed.StratoFed -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.usersignalservice.service.UserSignalService -import com.twitter.usersignalservice.thriftscala.BatchSignalRequest -import com.twitter.usersignalservice.thriftscala.BatchSignalResponse -import javax.inject.Inject - -class UserSignalServiceColumn @Inject() (userSignalService: UserSignalService) - extends StratoFed.Column(UserSignalServiceColumn.Path) - with StratoFed.Fetch.Stitch { - - override val metadata: OpMetadata = OpMetadata( - lifecycle = Some(Lifecycle.Production), - description = Some(Description.PlainText("User Signal Service Federated Column"))) - - override def ops: Ops = super.ops - - override type Key = BatchSignalRequest - override type View = Unit - override type Value = BatchSignalResponse - - override val keyConv: Conv[Key] = ScroogeConv.fromStruct[BatchSignalRequest] - override val viewConv: Conv[View] = Conv.ofType - override val valueConv: Conv[Value] = ScroogeConv.fromStruct[BatchSignalResponse] - - override def fetch(key: Key, view: View): Stitch[Result[Value]] = { - userSignalService - .userSignalServiceHandlerStoreStitch(key) - .map(result => found(result)) - .handle { - case NotFound => missing - } - } -} - -object UserSignalServiceColumn { - val Path = "recommendations/user-signal-service/signals" -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/BUILD b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/BUILD deleted file mode 100644 index cca1bf2e0..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/BUILD.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/BUILD.docx new file mode 100644 index 000000000..ceab161b5 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/BUILD.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/SignalFetcherConfig.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/SignalFetcherConfig.docx new file mode 100644 index 000000000..6c4f3f49a Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/SignalFetcherConfig.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/SignalFetcherConfig.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/SignalFetcherConfig.scala deleted file mode 100644 index f7238edcc..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config/SignalFetcherConfig.scala +++ /dev/null @@ -1,253 +0,0 @@ -package com.twitter.usersignalservice.config - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus.ReadableStore -import com.twitter.usersignalservice.base.BaseSignalFetcher -import com.twitter.usersignalservice.base.AggregatedSignalController -import com.twitter.usersignalservice.base.FilteredSignalFetcherController -import com.twitter.usersignalservice.base.MemcachedSignalFetcherWrapper -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.base.SignalAggregatedInfo -import com.twitter.usersignalservice.signals.AccountBlocksFetcher -import com.twitter.usersignalservice.signals.AccountFollowsFetcher -import com.twitter.usersignalservice.signals.AccountMutesFetcher -import com.twitter.usersignalservice.signals.NotificationOpenAndClickFetcher -import com.twitter.usersignalservice.signals.OriginalTweetsFetcher -import com.twitter.usersignalservice.signals.ProfileVisitsFetcher -import com.twitter.usersignalservice.signals.ProfileClickFetcher -import com.twitter.usersignalservice.signals.RealGraphOonFetcher -import com.twitter.usersignalservice.signals.ReplyTweetsFetcher -import com.twitter.usersignalservice.signals.RetweetsFetcher -import com.twitter.usersignalservice.signals.TweetClickFetcher -import com.twitter.usersignalservice.signals.TweetFavoritesFetcher -import com.twitter.usersignalservice.signals.TweetSharesFetcher -import com.twitter.usersignalservice.signals.VideoTweetsPlayback50Fetcher -import com.twitter.usersignalservice.signals.VideoTweetsQualityViewFetcher -import com.twitter.usersignalservice.signals.NegativeEngagedUserFetcher -import com.twitter.usersignalservice.signals.NegativeEngagedTweetFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SignalFetcherConfig @Inject() ( - notificationOpenAndClickFetcher: NotificationOpenAndClickFetcher, - accountFollowsFetcher: AccountFollowsFetcher, - profileVisitsFetcher: ProfileVisitsFetcher, - tweetFavoritesFetcher: TweetFavoritesFetcher, - retweetsFetcher: RetweetsFetcher, - replyTweetsFetcher: ReplyTweetsFetcher, - originalTweetsFetcher: OriginalTweetsFetcher, - tweetSharesFetcher: TweetSharesFetcher, - memcachedClient: MemcachedClient, - realGraphOonFetcher: RealGraphOonFetcher, - tweetClickFetcher: TweetClickFetcher, - videoTweetsPlayback50Fetcher: VideoTweetsPlayback50Fetcher, - videoTweetsQualityViewFetcher: VideoTweetsQualityViewFetcher, - accountMutesFetcher: AccountMutesFetcher, - accountBlocksFetcher: AccountBlocksFetcher, - profileClickFetcher: ProfileClickFetcher, - negativeEngagedTweetFetcher: NegativeEngagedTweetFetcher, - negativeEngagedUserFetcher: NegativeEngagedUserFetcher, - statsReceiver: StatsReceiver, - timer: Timer) { - - val MemcachedProfileVisitsFetcher: BaseSignalFetcher = - MemcachedSignalFetcherWrapper( - memcachedClient, - profileVisitsFetcher, - ttl = 8.hours, - statsReceiver, - keyPrefix = "uss:pv", - timer) - - val MemcachedAccountFollowsFetcher: BaseSignalFetcher = MemcachedSignalFetcherWrapper( - memcachedClient, - accountFollowsFetcher, - ttl = 5.minute, - statsReceiver, - keyPrefix = "uss:af", - timer) - - val GoodTweetClickDdgFetcher: SignalType => FilteredSignalFetcherController = signalType => - FilteredSignalFetcherController( - tweetClickFetcher, - signalType, - statsReceiver, - timer, - Map(SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher) - ) - - val GoodProfileClickDdgFetcher: SignalType => FilteredSignalFetcherController = signalType => - FilteredSignalFetcherController( - profileClickFetcher, - signalType, - statsReceiver, - timer, - Map(SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher) - ) - - val GoodProfileClickDdgFetcherWithBlocksMutes: SignalType => FilteredSignalFetcherController = - signalType => - FilteredSignalFetcherController( - profileClickFetcher, - signalType, - statsReceiver, - timer, - Map( - SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher, - SignalType.AccountMute -> accountMutesFetcher, - SignalType.AccountBlock -> accountBlocksFetcher - ) - ) - - val realGraphOonFilteredFetcher: FilteredSignalFetcherController = - FilteredSignalFetcherController( - realGraphOonFetcher, - SignalType.RealGraphOon, - statsReceiver, - timer, - Map( - SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher - ) - ) - - val videoTweetsQualityViewFilteredFetcher: FilteredSignalFetcherController = - FilteredSignalFetcherController( - videoTweetsQualityViewFetcher, - SignalType.VideoView90dQualityV1, - statsReceiver, - timer, - Map(SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher) - ) - - val videoTweetsPlayback50FilteredFetcher: FilteredSignalFetcherController = - FilteredSignalFetcherController( - videoTweetsPlayback50Fetcher, - SignalType.VideoView90dPlayback50V1, - statsReceiver, - timer, - Map(SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher) - ) - - val uniformTweetSignalInfo: Seq[SignalAggregatedInfo] = Seq( - SignalAggregatedInfo(SignalType.TweetFavorite, tweetFavoritesFetcher), - SignalAggregatedInfo(SignalType.Retweet, retweetsFetcher), - SignalAggregatedInfo(SignalType.Reply, replyTweetsFetcher), - SignalAggregatedInfo(SignalType.OriginalTweet, originalTweetsFetcher), - SignalAggregatedInfo(SignalType.TweetShareV1, tweetSharesFetcher), - SignalAggregatedInfo(SignalType.VideoView90dQualityV1, videoTweetsQualityViewFilteredFetcher), - ) - - val uniformProducerSignalInfo: Seq[SignalAggregatedInfo] = Seq( - SignalAggregatedInfo(SignalType.AccountFollow, MemcachedAccountFollowsFetcher), - SignalAggregatedInfo( - SignalType.RepeatedProfileVisit90dMinVisit6V1, - MemcachedProfileVisitsFetcher), - ) - - val memcachedAccountBlocksFetcher: MemcachedSignalFetcherWrapper = MemcachedSignalFetcherWrapper( - memcachedClient, - accountBlocksFetcher, - ttl = 5.minutes, - statsReceiver, - keyPrefix = "uss:ab", - timer) - - val memcachedAccountMutesFetcher: MemcachedSignalFetcherWrapper = MemcachedSignalFetcherWrapper( - memcachedClient, - accountMutesFetcher, - ttl = 5.minutes, - statsReceiver, - keyPrefix = "uss:am", - timer) - - val SignalFetcherMapper: Map[SignalType, ReadableStore[Query, Seq[Signal]]] = Map( - /* Raw Signals */ - SignalType.AccountFollow -> accountFollowsFetcher, - SignalType.AccountFollowWithDelay -> MemcachedAccountFollowsFetcher, - SignalType.GoodProfileClick -> GoodProfileClickDdgFetcher(SignalType.GoodProfileClick), - SignalType.GoodProfileClick20s -> GoodProfileClickDdgFetcher(SignalType.GoodProfileClick20s), - SignalType.GoodProfileClick30s -> GoodProfileClickDdgFetcher(SignalType.GoodProfileClick30s), - SignalType.GoodProfileClickFiltered -> GoodProfileClickDdgFetcherWithBlocksMutes( - SignalType.GoodProfileClick), - SignalType.GoodProfileClick20sFiltered -> GoodProfileClickDdgFetcherWithBlocksMutes( - SignalType.GoodProfileClick20s), - SignalType.GoodProfileClick30sFiltered -> GoodProfileClickDdgFetcherWithBlocksMutes( - SignalType.GoodProfileClick30s), - SignalType.GoodTweetClick -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick), - SignalType.GoodTweetClick5s -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick5s), - SignalType.GoodTweetClick10s -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick10s), - SignalType.GoodTweetClick30s -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick30s), - SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher, - SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher, - SignalType.NotificationOpenAndClickV1 -> notificationOpenAndClickFetcher, - SignalType.OriginalTweet -> originalTweetsFetcher, - SignalType.OriginalTweet90dV2 -> originalTweetsFetcher, - SignalType.RealGraphOon -> realGraphOonFilteredFetcher, - SignalType.RepeatedProfileVisit14dMinVisit2V1 -> MemcachedProfileVisitsFetcher, - SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative -> FilteredSignalFetcherController( - MemcachedProfileVisitsFetcher, - SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative, - statsReceiver, - timer, - Map( - SignalType.AccountMute -> accountMutesFetcher, - SignalType.AccountBlock -> accountBlocksFetcher) - ), - SignalType.RepeatedProfileVisit90dMinVisit6V1 -> MemcachedProfileVisitsFetcher, - SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative -> FilteredSignalFetcherController( - MemcachedProfileVisitsFetcher, - SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative, - statsReceiver, - timer, - Map( - SignalType.AccountMute -> accountMutesFetcher, - SignalType.AccountBlock -> accountBlocksFetcher), - ), - SignalType.RepeatedProfileVisit180dMinVisit6V1 -> MemcachedProfileVisitsFetcher, - SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative -> FilteredSignalFetcherController( - MemcachedProfileVisitsFetcher, - SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative, - statsReceiver, - timer, - Map( - SignalType.AccountMute -> accountMutesFetcher, - SignalType.AccountBlock -> accountBlocksFetcher), - ), - SignalType.Reply -> replyTweetsFetcher, - SignalType.Reply90dV2 -> replyTweetsFetcher, - SignalType.Retweet -> retweetsFetcher, - SignalType.Retweet90dV2 -> retweetsFetcher, - SignalType.TweetFavorite -> tweetFavoritesFetcher, - SignalType.TweetFavorite90dV2 -> tweetFavoritesFetcher, - SignalType.TweetShareV1 -> tweetSharesFetcher, - SignalType.VideoView90dQualityV1 -> videoTweetsQualityViewFilteredFetcher, - SignalType.VideoView90dPlayback50V1 -> videoTweetsPlayback50FilteredFetcher, - /* Aggregated Signals */ - SignalType.ProducerBasedUnifiedEngagementWeightedSignal -> AggregatedSignalController( - uniformProducerSignalInfo, - uniformProducerSignalEngagementAggregation, - statsReceiver, - timer - ), - SignalType.TweetBasedUnifiedEngagementWeightedSignal -> AggregatedSignalController( - uniformTweetSignalInfo, - uniformTweetSignalEngagementAggregation, - statsReceiver, - timer - ), - SignalType.AdFavorite -> tweetFavoritesFetcher, - /* Negative Signals */ - SignalType.AccountBlock -> memcachedAccountBlocksFetcher, - SignalType.AccountMute -> memcachedAccountMutesFetcher, - SignalType.TweetDontLike -> negativeEngagedTweetFetcher, - SignalType.TweetReport -> negativeEngagedTweetFetcher, - SignalType.TweetSeeFewer -> negativeEngagedTweetFetcher, - ) - -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/BUILD b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/BUILD deleted file mode 100644 index 96dbbeeaf..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "src/scala/com/twitter/twistly/common", - "src/scala/com/twitter/twistly/store", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/BUILD.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/BUILD.docx new file mode 100644 index 000000000..33311296e Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/BUILD.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/UserSignalHandler.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/UserSignalHandler.docx new file mode 100644 index 000000000..1343b4d98 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/UserSignalHandler.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/UserSignalHandler.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/UserSignalHandler.scala deleted file mode 100644 index 6fea51c4c..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler/UserSignalHandler.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.usersignalservice.handler - -import com.twitter.storehaus.ReadableStore -import com.twitter.usersignalservice.thriftscala.BatchSignalRequest -import com.twitter.usersignalservice.thriftscala.BatchSignalResponse -import com.twitter.util.Future -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.usersignalservice.config.SignalFetcherConfig -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.thriftscala.ClientIdentifier -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Duration -import com.twitter.util.Timer -import com.twitter.util.TimeoutException - -class UserSignalHandler( - signalFetcherConfig: SignalFetcherConfig, - timer: Timer, - stats: StatsReceiver) { - import UserSignalHandler._ - - val statsReceiver: StatsReceiver = stats.scope("user-signal-service/service") - - def getBatchSignalsResponse(request: BatchSignalRequest): Future[Option[BatchSignalResponse]] = { - StatsUtil.trackOptionStats(statsReceiver) { - val allSignals = request.signalRequest.map { signalRequest => - signalFetcherConfig - .SignalFetcherMapper(signalRequest.signalType) - .get(Query( - userId = request.userId, - signalType = signalRequest.signalType, - maxResults = signalRequest.maxResults.map(_.toInt), - clientId = request.clientId.getOrElse(ClientIdentifier.Unknown) - )) - .map(signalOpt => (signalRequest.signalType, signalOpt)) - } - - Future.collect(allSignals).map { signalsSeq => - val signalsMap = signalsSeq.map { - case (signalType: SignalType, signalsOpt) => - (signalType, signalsOpt.getOrElse(EmptySeq)) - }.toMap - Some(BatchSignalResponse(signalsMap)) - } - } - } - - def toReadableStore: ReadableStore[BatchSignalRequest, BatchSignalResponse] = { - new ReadableStore[BatchSignalRequest, BatchSignalResponse] { - override def get(request: BatchSignalRequest): Future[Option[BatchSignalResponse]] = { - getBatchSignalsResponse(request).raiseWithin(UserSignalServiceTimeout)(timer).rescue { - case _: TimeoutException => - statsReceiver.counter("endpointGetBatchSignals/failure/timeout").incr() - EmptyResponse - case e => - statsReceiver.counter("endpointGetBatchSignals/failure/" + e.getClass.getName).incr() - EmptyResponse - } - } - } - } -} - -object UserSignalHandler { - val UserSignalServiceTimeout: Duration = 25.milliseconds - - val EmptySeq: Seq[Nothing] = Seq.empty - val EmptyResponse: Future[Option[BatchSignalResponse]] = Future.value(Some(BatchSignalResponse())) -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/BUILD b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/BUILD deleted file mode 100644 index d8e1e6a49..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client", - "finagle/finagle-core/src/main", - "finagle/finagle-stats", - "finagle/finagle-thrift/src/main/scala", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/predicate/socialgraph", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection", - "servo/service/src/main/scala", - "src/scala/com/twitter/storehaus_internal/manhattan2", - "src/scala/com/twitter/storehaus_internal/memcache", - "src/scala/com/twitter/storehaus_internal/util", - "src/scala/com/twitter/twistly/common", - "src/scala/com/twitter/twistly/store", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-storehaus", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - "util/util-core:scala", - "util/util-stats/src/main/scala", - ], -) diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/BUILD.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/BUILD.docx new file mode 100644 index 000000000..bbd244eae Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/BUILD.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/CacheModule.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/CacheModule.docx new file mode 100644 index 000000000..febb6885e Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/CacheModule.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/CacheModule.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/CacheModule.scala deleted file mode 100644 index 38427b6ce..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/CacheModule.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.usersignalservice.module - -import com.google.inject.Provides -import javax.inject.Singleton -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.conversions.DurationOps._ -import com.twitter.storehaus_internal.memcache.MemcacheStore -import com.twitter.storehaus_internal.util.ZkEndPoint -import com.twitter.storehaus_internal.util.ClientName - -object CacheModule extends TwitterModule { - private val cacheDest = - flag[String](name = "cache_module.dest", help = "Path to memcache service") - private val timeout = - flag[Int](name = "memcache.timeout", help = "Memcache client timeout") - - @Singleton - @Provides - def providesCache( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver - ): Client = - MemcacheStore.memcachedClient( - name = ClientName("memcache_user_signal_service"), - dest = ZkEndPoint(cacheDest()), - timeout = timeout().milliseconds, - retries = 0, - statsReceiver = stats.scope("memcache"), - serviceIdentifier = serviceIdentifier - ) -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/MHMtlsParamsModule.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/MHMtlsParamsModule.docx new file mode 100644 index 000000000..6a895dd0c Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/MHMtlsParamsModule.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/MHMtlsParamsModule.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/MHMtlsParamsModule.scala deleted file mode 100644 index 1ff1a7c5d..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/MHMtlsParamsModule.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.usersignalservice.module - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.inject.TwitterModule -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.google.inject.Provides -import javax.inject.Singleton - -object MHMtlsParamsModule extends TwitterModule { - - @Singleton - @Provides - def providesManhattanMtlsParams( - serviceIdentifier: ServiceIdentifier - ): ManhattanKVClientMtlsParams = { - ManhattanKVClientMtlsParams(serviceIdentifier) - } -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/SocialGraphServiceClientModule.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/SocialGraphServiceClientModule.docx new file mode 100644 index 000000000..0ed48334b Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/SocialGraphServiceClientModule.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/SocialGraphServiceClientModule.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/SocialGraphServiceClientModule.scala deleted file mode 100644 index 194730261..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/SocialGraphServiceClientModule.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.usersignalservice.module - -import com.twitter.inject.Injector -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle._ -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.util.Duration -import com.twitter.util.Throw -import com.twitter.socialgraph.thriftscala.SocialGraphService - -object SocialGraphServiceClientModule - extends ThriftMethodBuilderClientModule[ - SocialGraphService.ServicePerEndpoint, - SocialGraphService.MethodPerEndpoint - ] - with MtlsClient { - override val label = "socialgraph" - override val dest = "/s/socialgraph/socialgraph" - override val requestTimeout: Duration = 30.milliseconds - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = { - super - .configureThriftMuxClient(injector, client) - .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) - .withSessionQualifier - .successRateFailureAccrual(successRate = 0.9, window = 30.seconds) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } - } - -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/TimerModule.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/TimerModule.docx new file mode 100644 index 000000000..2e80c4b29 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/TimerModule.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/TimerModule.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/TimerModule.scala deleted file mode 100644 index ffe26f8c4..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module/TimerModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.usersignalservice.module -import com.google.inject.Provides -import com.twitter.finagle.util.DefaultTimer -import com.twitter.inject.TwitterModule -import com.twitter.util.Timer -import javax.inject.Singleton - -object TimerModule extends TwitterModule { - @Singleton - @Provides - def providesTimer: Timer = DefaultTimer -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/BUILD b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/BUILD deleted file mode 100644 index d1cd4e3a3..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "stitch/stitch-storehaus", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/config", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/handler", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/module", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/BUILD.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/BUILD.docx new file mode 100644 index 000000000..1124ca31c Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/BUILD.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/UserSignalService.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/UserSignalService.docx new file mode 100644 index 000000000..b1ecdfa96 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/UserSignalService.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/UserSignalService.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/UserSignalService.scala deleted file mode 100644 index 92d956001..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/service/UserSignalService.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.usersignalservice -package service - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.storehaus.StitchOfReadableStore -import com.twitter.usersignalservice.config.SignalFetcherConfig -import com.twitter.usersignalservice.handler.UserSignalHandler -import com.twitter.usersignalservice.thriftscala.BatchSignalRequest -import com.twitter.usersignalservice.thriftscala.BatchSignalResponse -import com.twitter.util.Timer - -@Singleton -class UserSignalService @Inject() ( - signalFetcherConfig: SignalFetcherConfig, - timer: Timer, - stats: StatsReceiver) { - - private val userSignalHandler = - new UserSignalHandler(signalFetcherConfig, timer, stats) - - val userSignalServiceHandlerStoreStitch: BatchSignalRequest => com.twitter.stitch.Stitch[ - BatchSignalResponse - ] = StitchOfReadableStore(userSignalHandler.toReadableStore) -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountBlocksFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountBlocksFetcher.docx new file mode 100644 index 000000000..decc31ed0 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountBlocksFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountBlocksFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountBlocksFetcher.scala deleted file mode 100644 index a72348b7b..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountBlocksFetcher.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.base.BaseSignalFetcher -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.signals.common.SGSUtils -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class AccountBlocksFetcher @Inject() ( - sgsClient: SocialGraphService.MethodPerEndpoint, - timer: Timer, - stats: StatsReceiver) - extends BaseSignalFetcher { - - override type RawSignalType = Signal - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(this.name) - - override def getRawSignals( - userId: UserId - ): Future[Option[Seq[RawSignalType]]] = { - SGSUtils.getSGSRawSignals(userId, sgsClient, RelationshipType.Blocking, SignalType.AccountBlock) - } - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map(_.map(_.take(query.maxResults.getOrElse(Int.MaxValue)))) - } -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountFollowsFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountFollowsFetcher.docx new file mode 100644 index 000000000..7c6518208 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountFollowsFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountFollowsFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountFollowsFetcher.scala deleted file mode 100644 index 60cc2bbd7..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountFollowsFetcher.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.base.BaseSignalFetcher -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.signals.common.SGSUtils -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class AccountFollowsFetcher @Inject() ( - sgsClient: SocialGraphService.MethodPerEndpoint, - timer: Timer, - stats: StatsReceiver) - extends BaseSignalFetcher { - - override type RawSignalType = Signal - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(this.name) - - override def getRawSignals( - userId: UserId - ): Future[Option[Seq[RawSignalType]]] = { - SGSUtils.getSGSRawSignals( - userId, - sgsClient, - RelationshipType.Following, - SignalType.AccountFollow) - } - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map(_.map(_.take(query.maxResults.getOrElse(Int.MaxValue)))) - } -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountMutesFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountMutesFetcher.docx new file mode 100644 index 000000000..f742372ff Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountMutesFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountMutesFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountMutesFetcher.scala deleted file mode 100644 index 27eb0a36d..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/AccountMutesFetcher.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.base.BaseSignalFetcher -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.signals.common.SGSUtils -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class AccountMutesFetcher @Inject() ( - sgsClient: SocialGraphService.MethodPerEndpoint, - timer: Timer, - stats: StatsReceiver) - extends BaseSignalFetcher { - - override type RawSignalType = Signal - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(this.name) - - override def getRawSignals( - userId: UserId - ): Future[Option[Seq[RawSignalType]]] = { - SGSUtils.getSGSRawSignals(userId, sgsClient, RelationshipType.Muting, SignalType.AccountMute) - } - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map(_.map(_.take(query.maxResults.getOrElse(Int.MaxValue)))) - } -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/BUILD b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/BUILD deleted file mode 100644 index 50380a581..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/src/jvm/com/twitter/storehaus:core", - "discovery-ds/src/main/thrift/com/twitter/dds/jobs/repeated_profile_visits:profile_visit-scala", - "flock-client/src/main/thrift:thrift-scala", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/predicate/socialgraph", - "src/scala/com/twitter/scalding_internal/job", - "src/scala/com/twitter/simclusters_v2/common", - "src/scala/com/twitter/storehaus_internal/manhattan", - "src/scala/com/twitter/storehaus_internal/manhattan/config", - "src/scala/com/twitter/storehaus_internal/manhattan2", - "src/scala/com/twitter/storehaus_internal/offline", - "src/scala/com/twitter/storehaus_internal/util", - "src/scala/com/twitter/twistly/common", - "src/thrift/com/twitter/experiments/general_metrics:general_metrics-scala", - "src/thrift/com/twitter/frigate/data_pipeline:frigate-user-history-thrift-scala", - "src/thrift/com/twitter/onboarding/relevance/tweet_engagement:tweet_engagement-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/traffic_attribution:traffic_attribution-scala", - "strato/src/main/scala/com/twitter/strato/client", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/base", - "user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - "util/util-core:util-core-util", - "util/util-core/src/main/java/com/twitter/util", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/BUILD.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/BUILD.docx new file mode 100644 index 000000000..0eee381ae Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/BUILD.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedTweetFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedTweetFetcher.docx new file mode 100644 index 000000000..c0622385b Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedTweetFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedTweetFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedTweetFetcher.scala deleted file mode 100644 index 22c0b0852..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedTweetFetcher.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.twistly.thriftscala.RecentNegativeEngagedTweet -import com.twitter.twistly.thriftscala.TweetNegativeEngagementType -import com.twitter.twistly.thriftscala.UserRecentNegativeEngagedTweets -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.base.StratoSignalFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class NegativeEngagedTweetFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentNegativeEngagedTweets] { - - import NegativeEngagedTweetFetcher._ - - override type RawSignalType = RecentNegativeEngagedTweet - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = stratoPath - override val stratoView: Unit = None - - override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[UserRecentNegativeEngagedTweets] = - ScroogeConv.fromStruct[UserRecentNegativeEngagedTweets] - - override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, defaultVersion) - - override protected def toRawSignals( - stratoValue: UserRecentNegativeEngagedTweets - ): Seq[RecentNegativeEngagedTweet] = { - stratoValue.recentNegativeEngagedTweets - } - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RecentNegativeEngagedTweet]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { - _.map { signals => - signals - .filter(signal => negativeEngagedTweetTypeFilter(query.signalType, signal)) - .map { signal => - Signal( - query.signalType, - signal.engagedAt, - Some(InternalId.TweetId(signal.tweetId)) - ) - } - .groupBy(_.targetInternalId) // groupBy if there's duplicated authorIds - .mapValues(_.maxBy(_.timestamp)) - .values - .toSeq - .sortBy(-_.timestamp) - .take(query.maxResults.getOrElse(Int.MaxValue)) - } - } - } -} - -object NegativeEngagedTweetFetcher { - - val stratoPath = "recommendations/twistly/userRecentNegativeEngagedTweets" - private val defaultVersion = 0L - - private def negativeEngagedTweetTypeFilter( - signalType: SignalType, - signal: RecentNegativeEngagedTweet - ): Boolean = { - signalType match { - case SignalType.TweetDontLike => - signal.engagementType == TweetNegativeEngagementType.DontLike - case SignalType.TweetSeeFewer => - signal.engagementType == TweetNegativeEngagementType.SeeFewer - case SignalType.TweetReport => - signal.engagementType == TweetNegativeEngagementType.ReportClick - case SignalType.NegativeEngagedTweetId => true - case _ => false - } - } - -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedUserFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedUserFetcher.docx new file mode 100644 index 000000000..5321512a2 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedUserFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedUserFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedUserFetcher.scala deleted file mode 100644 index c07f61f91..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NegativeEngagedUserFetcher.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.twistly.thriftscala.RecentNegativeEngagedTweet -import com.twitter.twistly.thriftscala.UserRecentNegativeEngagedTweets -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.base.StratoSignalFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class NegativeEngagedUserFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentNegativeEngagedTweets] { - - import NegativeEngagedUserFetcher._ - - override type RawSignalType = RecentNegativeEngagedTweet - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = stratoPath - override val stratoView: Unit = None - - override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[UserRecentNegativeEngagedTweets] = - ScroogeConv.fromStruct[UserRecentNegativeEngagedTweets] - - override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, defaultVersion) - - override protected def toRawSignals( - stratoValue: UserRecentNegativeEngagedTweets - ): Seq[RecentNegativeEngagedTweet] = { - stratoValue.recentNegativeEngagedTweets - } - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RecentNegativeEngagedTweet]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { - _.map { signals => - signals - .map { e => - Signal( - defaultNegativeSignalType, - e.engagedAt, - Some(InternalId.UserId(e.authorId)) - ) - } - .groupBy(_.targetInternalId) // groupBy if there's duplicated authorIds - .mapValues(_.maxBy(_.timestamp)) - .values - .toSeq - .sortBy(-_.timestamp) - } - } - } -} - -object NegativeEngagedUserFetcher { - - val stratoPath = "recommendations/twistly/userRecentNegativeEngagedTweets" - private val defaultVersion = 0L - private val defaultNegativeSignalType = SignalType.NegativeEngagedUserId - -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NotificationOpenAndClickFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NotificationOpenAndClickFetcher.docx new file mode 100644 index 000000000..f951109e2 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NotificationOpenAndClickFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NotificationOpenAndClickFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NotificationOpenAndClickFetcher.scala deleted file mode 100644 index 5c40ec6a8..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/NotificationOpenAndClickFetcher.scala +++ /dev/null @@ -1,145 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.ClientEngagementEvent -import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.LatestEvents -import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.LatestNegativeEngagementEvents -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.Client -import com.twitter.twistly.common.TweetId -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.base.BaseSignalFetcher -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class NotificationOpenAndClickFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends BaseSignalFetcher { - import NotificationOpenAndClickFetcher._ - - override type RawSignalType = ClientEngagementEvent - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(this.name) - - private val latestEventsStore: ReadableStore[UserId, LatestEvents] = { - StratoFetchableStore - .withUnitView[UserId, LatestEvents](stratoClient, latestEventStoreColumn) - } - - private val notificationNegativeEngagementStore: ReadableStore[UserId, Seq[ - NotificationNegativeEngagement - ]] = { - StratoFetchableStore - .withUnitView[UserId, LatestNegativeEngagementEvents]( - stratoClient, - labeledPushRecsNegativeEngagementsColumn).mapValues(fromLatestNegativeEngagementEvents) - } - - override def getRawSignals( - userId: UserId - ): Future[Option[Seq[RawSignalType]]] = { - val notificationNegativeEngagementEventsFut = - notificationNegativeEngagementStore.get(userId) - val latestEventsFut = latestEventsStore.get(userId) - - Future - .join(latestEventsFut, notificationNegativeEngagementEventsFut).map { - case (latestEventsOpt, latestNegativeEngagementEventsOpt) => - latestEventsOpt.map { latestEvents => - // Negative Engagement Events Filter - filterNegativeEngagementEvents( - latestEvents.engagementEvents, - latestNegativeEngagementEventsOpt.getOrElse(Seq.empty), - statsReceiver.scope("filterNegativeEngagementEvents")) - } - } - } - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { - _.map { - _.take(query.maxResults.getOrElse(Int.MaxValue)).map { clientEngagementEvent => - Signal( - SignalType.NotificationOpenAndClickV1, - timestamp = clientEngagementEvent.timestampMillis, - targetInternalId = Some(InternalId.TweetId(clientEngagementEvent.tweetId)) - ) - } - } - } - } -} - -object NotificationOpenAndClickFetcher { - private val latestEventStoreColumn = "frigate/magicrecs/labeledPushRecsAggregated.User" - private val labeledPushRecsNegativeEngagementsColumn = - "frigate/magicrecs/labeledPushRecsNegativeEngagements.User" - - case class NotificationNegativeEngagement( - tweetId: TweetId, - timestampMillis: Long, - isNtabDisliked: Boolean, - isReportTweetClicked: Boolean, - isReportTweetDone: Boolean, - isReportUserClicked: Boolean, - isReportUserDone: Boolean) - - def fromLatestNegativeEngagementEvents( - latestNegativeEngagementEvents: LatestNegativeEngagementEvents - ): Seq[NotificationNegativeEngagement] = { - latestNegativeEngagementEvents.negativeEngagementEvents.map { event => - NotificationNegativeEngagement( - event.tweetId, - event.timestampMillis, - event.isNtabDisliked.getOrElse(false), - event.isReportTweetClicked.getOrElse(false), - event.isReportTweetDone.getOrElse(false), - event.isReportUserClicked.getOrElse(false), - event.isReportUserDone.getOrElse(false) - ) - } - } - - private def filterNegativeEngagementEvents( - engagementEvents: Seq[ClientEngagementEvent], - negativeEvents: Seq[NotificationNegativeEngagement], - statsReceiver: StatsReceiver - ): Seq[ClientEngagementEvent] = { - if (negativeEvents.nonEmpty) { - statsReceiver.counter("filterNegativeEngagementEvents").incr() - statsReceiver.stat("eventSizeBeforeFilter").add(engagementEvents.size) - - val negativeEngagementIdSet = - negativeEvents.collect { - case event - if event.isNtabDisliked || event.isReportTweetClicked || event.isReportTweetDone || event.isReportUserClicked || event.isReportUserDone => - event.tweetId - }.toSet - - // negative event size - statsReceiver.stat("negativeEventsSize").add(negativeEngagementIdSet.size) - - // filter out negative engagement sources - val result = engagementEvents.filterNot { event => - negativeEngagementIdSet.contains(event.tweetId) - } - - statsReceiver.stat("eventSizeAfterFilter").add(result.size) - - result - } else engagementEvents - } -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/OriginalTweetsFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/OriginalTweetsFetcher.docx new file mode 100644 index 000000000..af8dd993d Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/OriginalTweetsFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/OriginalTweetsFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/OriginalTweetsFetcher.scala deleted file mode 100644 index 46d5b8f9c..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/OriginalTweetsFetcher.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.twistly.common.TwistlyProfile -import com.twitter.twistly.thriftscala.EngagementMetadata.OriginalTweetMetadata -import com.twitter.twistly.thriftscala.RecentEngagedTweet -import com.twitter.twistly.thriftscala.UserRecentEngagedTweets -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.base.StratoSignalFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class OriginalTweetsFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentEngagedTweets] { - import OriginalTweetsFetcher._ - override type RawSignalType = RecentEngagedTweet - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = - TwistlyProfile.TwistlyProdProfile.userRecentEngagedStorePath - override val stratoView: Unit = None - - override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[UserRecentEngagedTweets] = - ScroogeConv.fromStruct[UserRecentEngagedTweets] - - override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, DefaultVersion) - - override protected def toRawSignals( - userRecentEngagedTweets: UserRecentEngagedTweets - ): Seq[RawSignalType] = - userRecentEngagedTweets.recentEngagedTweets - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { - _.map { signals => - val lookBackWindowFilteredSignals = - SignalFilter.lookBackWindow90DayFilter(signals, query.signalType) - lookBackWindowFilteredSignals - .collect { - case RecentEngagedTweet(tweetId, engagedAt, _: OriginalTweetMetadata, _) => - Signal(query.signalType, engagedAt, Some(InternalId.TweetId(tweetId))) - }.take(query.maxResults.getOrElse(Int.MaxValue)) - } - } - } - -} - -object OriginalTweetsFetcher { - // see com.twitter.twistly.store.UserRecentEngagedTweetsStore - private val DefaultVersion = 0 -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileClickFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileClickFetcher.docx new file mode 100644 index 000000000..3fe67416b Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileClickFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileClickFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileClickFetcher.scala deleted file mode 100644 index 1b93df59d..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileClickFetcher.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.twistly.thriftscala.RecentProfileClickImpressEvents -import com.twitter.twistly.thriftscala.ProfileClickImpressEvent -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.base.StratoSignalFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ProfileClickFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[(UserId, Long), Unit, RecentProfileClickImpressEvents] { - - import ProfileClickFetcher._ - - override type RawSignalType = ProfileClickImpressEvent - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = stratoPath - override val stratoView: Unit = None - - override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[RecentProfileClickImpressEvents] = - ScroogeConv.fromStruct[RecentProfileClickImpressEvents] - - override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, defaultVersion) - - override protected def toRawSignals( - stratoValue: RecentProfileClickImpressEvents - ): Seq[ProfileClickImpressEvent] = { - stratoValue.events - } - - override def process( - query: Query, - rawSignals: Future[Option[Seq[ProfileClickImpressEvent]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { events => - events - .map { clicks => - clicks - .filter(dwelltimeFilter(_, query.signalType)) - .map(signalFromProfileClick(_, query.signalType)) - .sortBy(-_.timestamp) - .take(query.maxResults.getOrElse(Int.MaxValue)) - } - } - } -} - -object ProfileClickFetcher { - - val stratoPath = "recommendations/twistly/userRecentProfileClickImpress" - private val defaultVersion = 0L - private val sec2millis: Int => Long = i => i * 1000L - private val minDwellTimeMap: Map[SignalType, Long] = Map( - SignalType.GoodProfileClick -> sec2millis(10), - SignalType.GoodProfileClick20s -> sec2millis(20), - SignalType.GoodProfileClick30s -> sec2millis(30), - SignalType.GoodProfileClickFiltered -> sec2millis(10), - SignalType.GoodProfileClick20sFiltered -> sec2millis(20), - SignalType.GoodProfileClick30sFiltered -> sec2millis(30), - ) - - def signalFromProfileClick( - profileClickImpressEvent: ProfileClickImpressEvent, - signalType: SignalType - ): Signal = { - Signal( - signalType, - profileClickImpressEvent.engagedAt, - Some(InternalId.UserId(profileClickImpressEvent.entityId)) - ) - } - - def dwelltimeFilter( - profileClickImpressEvent: ProfileClickImpressEvent, - signalType: SignalType - ): Boolean = { - val goodClickDwellTime = minDwellTimeMap(signalType) - profileClickImpressEvent.clickImpressEventMetadata.totalDwellTime >= goodClickDwellTime - } -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileVisitsFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileVisitsFetcher.docx new file mode 100644 index 000000000..f5daafea2 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileVisitsFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileVisitsFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileVisitsFetcher.scala deleted file mode 100644 index 1cb27261f..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ProfileVisitsFetcher.scala +++ /dev/null @@ -1,143 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.bijection.Codec -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.dds.jobs.repeated_profile_visits.thriftscala.ProfileVisitSet -import com.twitter.dds.jobs.repeated_profile_visits.thriftscala.ProfileVisitorInfo -import com.twitter.experiments.general_metrics.thriftscala.IdType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus_internal.manhattan.Apollo -import com.twitter.storehaus_internal.manhattan.ManhattanCluster -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.base.ManhattanSignalFetcher -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -case class ProfileVisitMetadata( - targetId: Option[Long], - totalTargetVisitsInLast14Days: Option[Int], - totalTargetVisitsInLast90Days: Option[Int], - totalTargetVisitsInLast180Days: Option[Int], - latestTargetVisitTimestampInLast90Days: Option[Long]) - -@Singleton -case class ProfileVisitsFetcher @Inject() ( - manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, - timer: Timer, - stats: StatsReceiver) - extends ManhattanSignalFetcher[ProfileVisitorInfo, ProfileVisitSet] { - import ProfileVisitsFetcher._ - - override type RawSignalType = ProfileVisitMetadata - - override val manhattanAppId: String = MHAppId - override val manhattanDatasetName: String = MHDatasetName - override val manhattanClusterId: ManhattanCluster = Apollo - override val manhattanKeyCodec: Codec[ProfileVisitorInfo] = BinaryScalaCodec(ProfileVisitorInfo) - override val manhattanRawSignalCodec: Codec[ProfileVisitSet] = BinaryScalaCodec(ProfileVisitSet) - - override protected def toManhattanKey(userId: UserId): ProfileVisitorInfo = - ProfileVisitorInfo(userId, IdType.User) - - override protected def toRawSignals(manhattanValue: ProfileVisitSet): Seq[ProfileVisitMetadata] = - manhattanValue.profileVisitSet - .map { - _.collect { - // only keep the Non-NSFW and not-following profile visits - case profileVisit - if profileVisit.targetId.nonEmpty - // The below check covers 180 days, not only 90 days as the name implies. - // See comment on [[ProfileVisit.latestTargetVisitTimestampInLast90Days]] thrift. - && profileVisit.latestTargetVisitTimestampInLast90Days.nonEmpty - && !profileVisit.isTargetNSFW.getOrElse(false) - && !profileVisit.doesSourceIdFollowTargetId.getOrElse(false) => - ProfileVisitMetadata( - targetId = profileVisit.targetId, - totalTargetVisitsInLast14Days = profileVisit.totalTargetVisitsInLast14Days, - totalTargetVisitsInLast90Days = profileVisit.totalTargetVisitsInLast90Days, - totalTargetVisitsInLast180Days = profileVisit.totalTargetVisitsInLast180Days, - latestTargetVisitTimestampInLast90Days = - profileVisit.latestTargetVisitTimestampInLast90Days - ) - }.toSeq - }.getOrElse(Seq.empty) - - override val name: String = this.getClass.getCanonicalName - - override val statsReceiver: StatsReceiver = stats.scope(name) - - override def process( - query: Query, - rawSignals: Future[Option[Seq[ProfileVisitMetadata]]] - ): Future[Option[Seq[Signal]]] = rawSignals.map { profiles => - profiles - .map { - _.filter(profileVisitMetadata => visitCountFilter(profileVisitMetadata, query.signalType)) - .sortBy(profileVisitMetadata => - -visitCountMap(query.signalType)(profileVisitMetadata).getOrElse(0)) - .map(profileVisitMetadata => - signalFromProfileVisit(profileVisitMetadata, query.signalType)) - .take(query.maxResults.getOrElse(Int.MaxValue)) - } - } -} - -object ProfileVisitsFetcher { - private val MHAppId = "repeated_profile_visits_aggregated" - private val MHDatasetName = "repeated_profile_visits_aggregated" - - private val minVisitCountMap: Map[SignalType, Int] = Map( - SignalType.RepeatedProfileVisit14dMinVisit2V1 -> 2, - SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative -> 2, - SignalType.RepeatedProfileVisit90dMinVisit6V1 -> 6, - SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative -> 6, - SignalType.RepeatedProfileVisit180dMinVisit6V1 -> 6, - SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative -> 6 - ) - - private val visitCountMap: Map[SignalType, ProfileVisitMetadata => Option[Int]] = Map( - SignalType.RepeatedProfileVisit14dMinVisit2V1 -> - ((profileVisitMetadata: ProfileVisitMetadata) => - profileVisitMetadata.totalTargetVisitsInLast14Days), - SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative -> - ((profileVisitMetadata: ProfileVisitMetadata) => - profileVisitMetadata.totalTargetVisitsInLast14Days), - SignalType.RepeatedProfileVisit90dMinVisit6V1 -> - ((profileVisitMetadata: ProfileVisitMetadata) => - profileVisitMetadata.totalTargetVisitsInLast90Days), - SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative -> - ((profileVisitMetadata: ProfileVisitMetadata) => - profileVisitMetadata.totalTargetVisitsInLast90Days), - SignalType.RepeatedProfileVisit180dMinVisit6V1 -> - ((profileVisitMetadata: ProfileVisitMetadata) => - profileVisitMetadata.totalTargetVisitsInLast180Days), - SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative -> - ((profileVisitMetadata: ProfileVisitMetadata) => - profileVisitMetadata.totalTargetVisitsInLast180Days) - ) - - def signalFromProfileVisit( - profileVisitMetadata: ProfileVisitMetadata, - signalType: SignalType - ): Signal = { - Signal( - signalType, - profileVisitMetadata.latestTargetVisitTimestampInLast90Days.get, - profileVisitMetadata.targetId.map(targetId => InternalId.UserId(targetId)) - ) - } - - def visitCountFilter( - profileVisitMetadata: ProfileVisitMetadata, - signalType: SignalType - ): Boolean = { - visitCountMap(signalType)(profileVisitMetadata).exists(_ >= minVisitCountMap(signalType)) - } -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RealGraphOonFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RealGraphOonFetcher.docx new file mode 100644 index 000000000..8674c01a6 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RealGraphOonFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RealGraphOonFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RealGraphOonFetcher.scala deleted file mode 100644 index ad5cc4f4b..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RealGraphOonFetcher.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.usersignalservice.base.Query -import com.twitter.wtf.candidate.thriftscala.CandidateSeq -import com.twitter.wtf.candidate.thriftscala.Candidate -import com.twitter.usersignalservice.base.StratoSignalFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class RealGraphOonFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[UserId, Unit, CandidateSeq] { - import RealGraphOonFetcher._ - override type RawSignalType = Candidate - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = RealGraphOonFetcher.stratoColumnPath - override val stratoView: Unit = None - - override protected val keyConv: Conv[UserId] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[CandidateSeq] = - ScroogeConv.fromStruct[CandidateSeq] - - override protected def toStratoKey(userId: UserId): UserId = userId - - override protected def toRawSignals( - realGraphOonCandidates: CandidateSeq - ): Seq[RawSignalType] = realGraphOonCandidates.candidates - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals - .map { - _.map( - _.sortBy(-_.score) - .collect { - case c if c.score >= MinRgScore => - Signal( - SignalType.RealGraphOon, - RealGraphOonFetcher.DefaultTimestamp, - Some(InternalId.UserId(c.userId))) - }.take(query.maxResults.getOrElse(Int.MaxValue))) - } - } -} - -object RealGraphOonFetcher { - val stratoColumnPath = "recommendations/real_graph/realGraphScoresOon.User" - // quality threshold for real graph score - private val MinRgScore = 0.0 - // no timestamp for RealGraph Candidates, set default as 0L - private val DefaultTimestamp = 0L -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ReplyTweetsFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ReplyTweetsFetcher.docx new file mode 100644 index 000000000..b3b0d5a14 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ReplyTweetsFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ReplyTweetsFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ReplyTweetsFetcher.scala deleted file mode 100644 index 7f84f41c9..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/ReplyTweetsFetcher.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.twistly.common.TwistlyProfile -import com.twitter.twistly.thriftscala.EngagementMetadata.ReplyTweetMetadata -import com.twitter.twistly.thriftscala.RecentEngagedTweet -import com.twitter.twistly.thriftscala.UserRecentEngagedTweets -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.base.StratoSignalFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ReplyTweetsFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentEngagedTweets] { - import ReplyTweetsFetcher._ - override type RawSignalType = RecentEngagedTweet - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = - TwistlyProfile.TwistlyProdProfile.userRecentEngagedStorePath - override val stratoView: Unit = None - - override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[UserRecentEngagedTweets] = - ScroogeConv.fromStruct[UserRecentEngagedTweets] - - override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, DefaultVersion) - - override protected def toRawSignals( - userRecentEngagedTweets: UserRecentEngagedTweets - ): Seq[RawSignalType] = - userRecentEngagedTweets.recentEngagedTweets - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { - _.map { signals => - val lookBackWindowFilteredSignals = - SignalFilter.lookBackWindow90DayFilter(signals, query.signalType) - lookBackWindowFilteredSignals - .collect { - case RecentEngagedTweet(tweetId, engagedAt, _: ReplyTweetMetadata, _) => - Signal(query.signalType, engagedAt, Some(InternalId.TweetId(tweetId))) - }.take(query.maxResults.getOrElse(Int.MaxValue)) - } - } - } - -} - -object ReplyTweetsFetcher { - // see com.twitter.twistly.store.UserRecentEngagedTweetsStore - private val DefaultVersion = 0 -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RetweetsFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RetweetsFetcher.docx new file mode 100644 index 000000000..c3ef712a0 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RetweetsFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RetweetsFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RetweetsFetcher.scala deleted file mode 100644 index 4b81c8d0b..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/RetweetsFetcher.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.twistly.common.TwistlyProfile -import com.twitter.twistly.thriftscala.EngagementMetadata.RetweetMetadata -import com.twitter.twistly.thriftscala.RecentEngagedTweet -import com.twitter.twistly.thriftscala.UserRecentEngagedTweets -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.base.StratoSignalFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class RetweetsFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentEngagedTweets] { - import RetweetsFetcher._ - override type RawSignalType = RecentEngagedTweet - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = - TwistlyProfile.TwistlyProdProfile.userRecentEngagedStorePath - override val stratoView: Unit = None - - override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[UserRecentEngagedTweets] = - ScroogeConv.fromStruct[UserRecentEngagedTweets] - - override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, DefaultVersion) - - override protected def toRawSignals( - userRecentEngagedTweets: UserRecentEngagedTweets - ): Seq[RawSignalType] = - userRecentEngagedTweets.recentEngagedTweets - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { - _.map { signals => - val lookBackWindowFilteredSignals = - SignalFilter.lookBackWindow90DayFilter(signals, query.signalType) - lookBackWindowFilteredSignals - .filter { recentEngagedTweet => - recentEngagedTweet.features.statusCounts - .flatMap(_.favoriteCount).exists(_ >= MinFavCount) - }.collect { - case RecentEngagedTweet(tweetId, engagedAt, _: RetweetMetadata, _) => - Signal(query.signalType, engagedAt, Some(InternalId.TweetId(tweetId))) - }.take(query.maxResults.getOrElse(Int.MaxValue)) - } - } - } - -} - -object RetweetsFetcher { - private val MinFavCount = 10 - // see com.twitter.twistly.store.UserRecentEngagedTweetsStore - private val DefaultVersion = 0 -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/SignalFilter.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/SignalFilter.docx new file mode 100644 index 000000000..9a1f8ec17 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/SignalFilter.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/SignalFilter.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/SignalFilter.scala deleted file mode 100644 index 01be88a26..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/SignalFilter.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.twistly.thriftscala.EngagementMetadata.FavoriteMetadata -import com.twitter.twistly.thriftscala.RecentEngagedTweet -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Time - -// Shared Logic for filtering signal across different signal types -object SignalFilter { - - final val LookBackWindow90DayFilterEnabledSignalTypes: Set[SignalType] = Set( - SignalType.TweetFavorite90dV2, - SignalType.Retweet90dV2, - SignalType.OriginalTweet90dV2, - SignalType.Reply90dV2) - - /* Raw Signal Filter for TweetFavorite, Retweet, Original Tweet and Reply - * Filter out all raw signal if the most recent {Tweet Favorite + Retweet + Original Tweet + Reply} - * is older than 90 days. - * The filter is shared across 4 signal types as they are stored in the same physical store - * thus sharing the same TTL - * */ - def lookBackWindow90DayFilter( - signals: Seq[RecentEngagedTweet], - querySignalType: SignalType - ): Seq[RecentEngagedTweet] = { - if (LookBackWindow90DayFilterEnabledSignalTypes.contains( - querySignalType) && !isMostRecentSignalWithin90Days(signals.head)) { - Seq.empty - } else signals - } - - private def isMostRecentSignalWithin90Days( - signal: RecentEngagedTweet - ): Boolean = { - val diff = Time.now - Time.fromMilliseconds(signal.engagedAt) - diff.inDays <= 90 - } - - def isPromotedTweet(signal: RecentEngagedTweet): Boolean = { - signal match { - case RecentEngagedTweet(_, _, metadata: FavoriteMetadata, _) => - metadata.favoriteMetadata.isAd.getOrElse(false) - case _ => false - } - } - -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetClickFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetClickFetcher.docx new file mode 100644 index 000000000..b23d68fa7 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetClickFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetClickFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetClickFetcher.scala deleted file mode 100644 index 19462a4e2..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetClickFetcher.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.twistly.thriftscala.RecentTweetClickImpressEvents -import com.twitter.twistly.thriftscala.TweetClickImpressEvent -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.base.StratoSignalFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class TweetClickFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[(UserId, Long), Unit, RecentTweetClickImpressEvents] { - - import TweetClickFetcher._ - - override type RawSignalType = TweetClickImpressEvent - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = stratoPath - override val stratoView: Unit = None - - override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[RecentTweetClickImpressEvents] = - ScroogeConv.fromStruct[RecentTweetClickImpressEvents] - - override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, defaultVersion) - - override protected def toRawSignals( - stratoValue: RecentTweetClickImpressEvents - ): Seq[TweetClickImpressEvent] = { - stratoValue.events - } - - override def process( - query: Query, - rawSignals: Future[Option[Seq[TweetClickImpressEvent]]] - ): Future[Option[Seq[Signal]]] = - rawSignals.map { events => - events.map { clicks => - clicks - .filter(dwelltimeFilter(_, query.signalType)) - .map(signalFromTweetClick(_, query.signalType)) - .sortBy(-_.timestamp) - .take(query.maxResults.getOrElse(Int.MaxValue)) - } - } -} - -object TweetClickFetcher { - - val stratoPath = "recommendations/twistly/userRecentTweetClickImpress" - private val defaultVersion = 0L - - private val minDwellTimeMap: Map[SignalType, Long] = Map( - SignalType.GoodTweetClick -> 2 * 1000L, - SignalType.GoodTweetClick5s -> 5 * 1000L, - SignalType.GoodTweetClick10s -> 10 * 1000L, - SignalType.GoodTweetClick30s -> 30 * 1000L, - ) - - def signalFromTweetClick( - tweetClickImpressEvent: TweetClickImpressEvent, - signalType: SignalType - ): Signal = { - Signal( - signalType, - tweetClickImpressEvent.engagedAt, - Some(InternalId.TweetId(tweetClickImpressEvent.entityId)) - ) - } - - def dwelltimeFilter( - tweetClickImpressEvent: TweetClickImpressEvent, - signalType: SignalType - ): Boolean = { - val goodClickDwellTime = minDwellTimeMap(signalType) - tweetClickImpressEvent.clickImpressEventMetadata.totalDwellTime >= goodClickDwellTime - } -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetFavoritesFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetFavoritesFetcher.docx new file mode 100644 index 000000000..8d17c6d6f Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetFavoritesFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetFavoritesFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetFavoritesFetcher.scala deleted file mode 100644 index b427f722f..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetFavoritesFetcher.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.twistly.common.TwistlyProfile -import com.twitter.twistly.thriftscala.EngagementMetadata.FavoriteMetadata -import com.twitter.twistly.thriftscala.RecentEngagedTweet -import com.twitter.twistly.thriftscala.UserRecentEngagedTweets -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.base.StratoSignalFetcher -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class TweetFavoritesFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[(UserId, Long), Unit, UserRecentEngagedTweets] { - import TweetFavoritesFetcher._ - override type RawSignalType = RecentEngagedTweet - override val name: String = this.getClass.getCanonicalName - override val statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = - TwistlyProfile.TwistlyProdProfile.userRecentEngagedStorePath - override val stratoView: Unit = None - - override protected val keyConv: Conv[(UserId, Long)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[UserRecentEngagedTweets] = - ScroogeConv.fromStruct[UserRecentEngagedTweets] - - override protected def toStratoKey(userId: UserId): (UserId, Long) = (userId, DefaultVersion) - - override protected def toRawSignals( - userRecentEngagedTweets: UserRecentEngagedTweets - ): Seq[RawSignalType] = - userRecentEngagedTweets.recentEngagedTweets - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RawSignalType]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { - _.map { signals => - val lookBackWindowFilteredSignals = - SignalFilter.lookBackWindow90DayFilter(signals, query.signalType) - lookBackWindowFilteredSignals - .filter { recentEngagedTweet => - recentEngagedTweet.features.statusCounts - .flatMap(_.favoriteCount).exists(_ >= MinFavCount) - }.filter { recentEngagedTweet => - applySignalTweetTypeFilter(query.signalType, recentEngagedTweet) - }.collect { - case RecentEngagedTweet(tweetId, engagedAt, _: FavoriteMetadata, _) => - Signal(query.signalType, engagedAt, Some(InternalId.TweetId(tweetId))) - }.take(query.maxResults.getOrElse(Int.MaxValue)) - } - } - } - private def applySignalTweetTypeFilter( - signal: SignalType, - recentEngagedTweet: RecentEngagedTweet - ): Boolean = { - // Perform specific filters for particular signal types. - signal match { - case SignalType.AdFavorite => SignalFilter.isPromotedTweet(recentEngagedTweet) - case _ => true - } - } -} - -object TweetFavoritesFetcher { - private val MinFavCount = 10 - // see com.twitter.twistly.store.UserRecentEngagedTweetsStore - private val DefaultVersion = 0 -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetSharesFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetSharesFetcher.docx new file mode 100644 index 000000000..1e6a5310d Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetSharesFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetSharesFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetSharesFetcher.scala deleted file mode 100644 index 6205e1bc3..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/TweetSharesFetcher.scala +++ /dev/null @@ -1,77 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.bijection.Codec -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.onboarding.relevance.tweet_engagement.thriftscala.EngagementIdentifier -import com.twitter.onboarding.relevance.tweet_engagement.thriftscala.TweetEngagement -import com.twitter.onboarding.relevance.tweet_engagement.thriftscala.TweetEngagements -import com.twitter.scalding_internal.multiformat.format.keyval.KeyValInjection.Long2BigEndian -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus_internal.manhattan.Apollo -import com.twitter.storehaus_internal.manhattan.ManhattanCluster -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.base.ManhattanSignalFetcher -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Future -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class TweetSharesFetcher @Inject() ( - manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, - timer: Timer, - stats: StatsReceiver) - extends ManhattanSignalFetcher[Long, TweetEngagements] { - - import TweetSharesFetcher._ - - override type RawSignalType = TweetEngagement - - override def name: String = this.getClass.getCanonicalName - - override def statsReceiver: StatsReceiver = stats.scope(name) - - override protected def manhattanAppId: String = MHAppId - - override protected def manhattanDatasetName: String = MHDatasetName - - override protected def manhattanClusterId: ManhattanCluster = Apollo - - override protected def manhattanKeyCodec: Codec[Long] = Long2BigEndian - - override protected def manhattanRawSignalCodec: Codec[TweetEngagements] = BinaryScalaCodec( - TweetEngagements) - - override protected def toManhattanKey(userId: UserId): Long = userId - - override protected def toRawSignals( - manhattanValue: TweetEngagements - ): Seq[TweetEngagement] = manhattanValue.tweetEngagements - - override def process( - query: Query, - rawSignals: Future[Option[Seq[TweetEngagement]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { - _.map { - _.collect { - case tweetEngagement if (tweetEngagement.engagementType == EngagementIdentifier.Share) => - Signal( - SignalType.TweetShareV1, - tweetEngagement.timestampMs, - Some(InternalId.TweetId(tweetEngagement.tweetId))) - }.sortBy(-_.timestamp).take(query.maxResults.getOrElse(Int.MaxValue)) - } - } - } -} - -object TweetSharesFetcher { - private val MHAppId = "uss_prod_apollo" - private val MHDatasetName = "tweet_share_engagements" -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsPlayback50Fetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsPlayback50Fetcher.docx new file mode 100644 index 000000000..7b78b25cc Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsPlayback50Fetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsPlayback50Fetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsPlayback50Fetcher.scala deleted file mode 100644 index 1577b2e99..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsPlayback50Fetcher.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.twistly.common.UserId -import com.twitter.twistly.thriftscala.UserRecentVideoViewTweets -import com.twitter.twistly.thriftscala.VideoViewEngagementType -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.util.Future -import com.twitter.util.Timer -import com.twitter.twistly.thriftscala.RecentVideoViewTweet -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.usersignalservice.base.StratoSignalFetcher -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class VideoTweetsPlayback50Fetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[ - (UserId, VideoViewEngagementType), - Unit, - UserRecentVideoViewTweets - ] { - import VideoTweetsPlayback50Fetcher._ - - override type RawSignalType = RecentVideoViewTweet - override def name: String = this.getClass.getCanonicalName - override def statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = StratoColumn - override val stratoView: Unit = None - override protected val keyConv: Conv[(UserId, VideoViewEngagementType)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[UserRecentVideoViewTweets] = - ScroogeConv.fromStruct[UserRecentVideoViewTweets] - - override protected def toStratoKey(userId: UserId): (UserId, VideoViewEngagementType) = - (userId, VideoViewEngagementType.VideoPlayback50) - - override protected def toRawSignals( - stratoValue: UserRecentVideoViewTweets - ): Seq[RecentVideoViewTweet] = stratoValue.recentEngagedTweets - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RecentVideoViewTweet]]] - ): Future[Option[Seq[Signal]]] = rawSignals.map { - _.map { - _.filter(videoView => - !videoView.isPromotedTweet && videoView.videoDurationSeconds >= MinVideoDurationSeconds) - .map { rawSignal => - Signal( - SignalType.VideoView90dPlayback50V1, - rawSignal.engagedAt, - Some(InternalId.TweetId(rawSignal.tweetId))) - }.take(query.maxResults.getOrElse(Int.MaxValue)) - } - } - -} - -object VideoTweetsPlayback50Fetcher { - private val StratoColumn = "recommendations/twistly/userRecentVideoViewTweetEngagements" - private val MinVideoDurationSeconds = 10 -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsQualityViewFetcher.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsQualityViewFetcher.docx new file mode 100644 index 000000000..9fd9d1684 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsQualityViewFetcher.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsQualityViewFetcher.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsQualityViewFetcher.scala deleted file mode 100644 index d513b978c..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/VideoTweetsQualityViewFetcher.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.usersignalservice.signals - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.twistly.common.UserId -import com.twitter.twistly.thriftscala.UserRecentVideoViewTweets -import com.twitter.twistly.thriftscala.VideoViewEngagementType -import com.twitter.usersignalservice.base.Query -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.util.Future -import com.twitter.util.Timer -import com.twitter.twistly.thriftscala.RecentVideoViewTweet -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.strato.client.Client -import com.twitter.strato.data.Conv -import com.twitter.strato.thrift.ScroogeConv -import com.twitter.usersignalservice.base.StratoSignalFetcher -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class VideoTweetsQualityViewFetcher @Inject() ( - stratoClient: Client, - timer: Timer, - stats: StatsReceiver) - extends StratoSignalFetcher[ - (UserId, VideoViewEngagementType), - Unit, - UserRecentVideoViewTweets - ] { - import VideoTweetsQualityViewFetcher._ - override type RawSignalType = RecentVideoViewTweet - override def name: String = this.getClass.getCanonicalName - override def statsReceiver: StatsReceiver = stats.scope(name) - - override val stratoColumnPath: String = StratoColumn - override val stratoView: Unit = None - override protected val keyConv: Conv[(UserId, VideoViewEngagementType)] = Conv.ofType - override protected val viewConv: Conv[Unit] = Conv.ofType - override protected val valueConv: Conv[UserRecentVideoViewTweets] = - ScroogeConv.fromStruct[UserRecentVideoViewTweets] - - override protected def toStratoKey(userId: UserId): (UserId, VideoViewEngagementType) = - (userId, VideoViewEngagementType.VideoQualityView) - - override protected def toRawSignals( - stratoValue: UserRecentVideoViewTweets - ): Seq[RecentVideoViewTweet] = stratoValue.recentEngagedTweets - - override def process( - query: Query, - rawSignals: Future[Option[Seq[RecentVideoViewTweet]]] - ): Future[Option[Seq[Signal]]] = { - rawSignals.map { - _.map { - _.filter(videoView => - !videoView.isPromotedTweet && videoView.videoDurationSeconds >= MinVideoDurationSeconds) - .map { rawSignal => - Signal( - SignalType.VideoView90dQualityV1, - rawSignal.engagedAt, - Some(InternalId.TweetId(rawSignal.tweetId))) - }.take(query.maxResults.getOrElse(Int.MaxValue)) - } - } - } -} - -object VideoTweetsQualityViewFetcher { - private val StratoColumn = "recommendations/twistly/userRecentVideoViewTweetEngagements" - private val MinVideoDurationSeconds = 10 -} diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/BUILD b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/BUILD deleted file mode 100644 index baca538b0..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "hermit/hermit-core/src/main/scala/com/twitter/hermit/predicate/socialgraph", - "src/scala/com/twitter/simclusters_v2/common", - "src/scala/com/twitter/twistly/common", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - "util/util-core:util-core-util", - "util/util-core/src/main/java/com/twitter/util", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/BUILD.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/BUILD.docx new file mode 100644 index 000000000..fa11086b0 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/BUILD.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/SGSUtils.docx b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/SGSUtils.docx new file mode 100644 index 000000000..d1bd13e47 Binary files /dev/null and b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/SGSUtils.docx differ diff --git a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/SGSUtils.scala b/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/SGSUtils.scala deleted file mode 100644 index 01fbd8f38..000000000 --- a/user-signal-service/server/src/main/scala/com/twitter/usersignalservice/signals/common/SGSUtils.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.usersignalservice.signals -package common - -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.socialgraph.thriftscala.EdgesRequest -import com.twitter.socialgraph.thriftscala.EdgesResult -import com.twitter.socialgraph.thriftscala.PageRequest -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.socialgraph.thriftscala.SrcRelationship -import com.twitter.twistly.common.UserId -import com.twitter.usersignalservice.thriftscala.Signal -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time - -object SGSUtils { - val MaxNumSocialGraphSignals = 200 - val MaxAge: Duration = Duration.fromDays(90) - - def getSGSRawSignals( - userId: UserId, - sgsClient: SocialGraphService.MethodPerEndpoint, - relationshipType: RelationshipType, - signalType: SignalType, - ): Future[Option[Seq[Signal]]] = { - val edgeRequest = EdgesRequest( - relationship = SrcRelationship(userId, relationshipType), - pageRequest = Some(PageRequest(count = None)) - ) - val now = Time.now.inMilliseconds - - sgsClient - .edges(Seq(edgeRequest)) - .map { sgsEdges => - sgsEdges.flatMap { - case EdgesResult(edges, _, _) => - edges.collect { - case edge if edge.createdAt >= now - MaxAge.inMilliseconds => - Signal( - signalType, - timestamp = edge.createdAt, - targetInternalId = Some(InternalId.UserId(edge.target))) - } - } - } - .map { signals => - signals - .take(MaxNumSocialGraphSignals) - .groupBy(_.targetInternalId) - .mapValues(_.maxBy(_.timestamp)) - .values - .toSeq - .sortBy(-_.timestamp) - } - .map(Some(_)) - } -} diff --git a/user-signal-service/thrift/src/main/thrift/BUILD b/user-signal-service/thrift/src/main/thrift/BUILD deleted file mode 100644 index faab4af7e..000000000 --- a/user-signal-service/thrift/src/main/thrift/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -create_thrift_libraries( - base_name = "thrift", - sources = [ - "client_identifier.thrift", - "service.thrift", - "signal.thrift", - ], - platform = "java8", - tags = ["bazel-compatible"], - dependency_roots = [ - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift", - ], - generate_languages = [ - "java", - "scala", - "strato", - ], - provides_java_name = "uss-thrift-java", - provides_scala_name = "uss-thrift-scala", -) diff --git a/user-signal-service/thrift/src/main/thrift/BUILD.docx b/user-signal-service/thrift/src/main/thrift/BUILD.docx new file mode 100644 index 000000000..5371c7a42 Binary files /dev/null and b/user-signal-service/thrift/src/main/thrift/BUILD.docx differ diff --git a/user-signal-service/thrift/src/main/thrift/client_identifier.docx b/user-signal-service/thrift/src/main/thrift/client_identifier.docx new file mode 100644 index 000000000..d6b925a96 Binary files /dev/null and b/user-signal-service/thrift/src/main/thrift/client_identifier.docx differ diff --git a/user-signal-service/thrift/src/main/thrift/client_identifier.thrift b/user-signal-service/thrift/src/main/thrift/client_identifier.thrift deleted file mode 100644 index c953e6b8f..000000000 --- a/user-signal-service/thrift/src/main/thrift/client_identifier.thrift +++ /dev/null @@ -1,22 +0,0 @@ -namespace java com.twitter.usersignalservice.thriftjava -namespace py gen.twitter.usersignalservice.service -#@namespace scala com.twitter.usersignalservice.thriftscala -#@namespace strato com.twitter.usersignalservice.strato - -# ClientIdentifier should be defined as ServiceId_Product -enum ClientIdentifier { - # reserve 1-10 for CrMixer - CrMixer_Home = 1 - CrMixer_Notifications = 2 - CrMixer_Email = 3 - # reserve 11-20 for RSX - RepresentationScorer_Home = 11 - RepresentationScorer_Notifications = 12 - - # reserve 21-30 for Explore - ExploreRanker = 21 - - # We will throw an exception after we make sure all clients are sending the - # ClientIdentifier in their request. - Unknown = 9999 -} diff --git a/user-signal-service/thrift/src/main/thrift/service.docx b/user-signal-service/thrift/src/main/thrift/service.docx new file mode 100644 index 000000000..cf8b60c22 Binary files /dev/null and b/user-signal-service/thrift/src/main/thrift/service.docx differ diff --git a/user-signal-service/thrift/src/main/thrift/service.thrift b/user-signal-service/thrift/src/main/thrift/service.thrift deleted file mode 100644 index a10959ea8..000000000 --- a/user-signal-service/thrift/src/main/thrift/service.thrift +++ /dev/null @@ -1,23 +0,0 @@ -namespace java com.twitter.usersignalservice.thriftjava -namespace py gen.twitter.usersignalservice.service -#@namespace scala com.twitter.usersignalservice.thriftscala -#@namespace strato com.twitter.usersignalservice.strato - -include "signal.thrift" -include "client_identifier.thrift" - -struct SignalRequest { - 1: optional i64 maxResults - 2: required signal.SignalType signalType -} - -struct BatchSignalRequest { - 1: required i64 userId(personalDataType = "UserId") - 2: required list signalRequest - # make sure to populate the clientId, otherwise the service would throw exceptions - 3: optional client_identifier.ClientIdentifier clientId -}(hasPersonalData='true') - -struct BatchSignalResponse { - 1: required map> signalResponse -} diff --git a/user-signal-service/thrift/src/main/thrift/signal.docx b/user-signal-service/thrift/src/main/thrift/signal.docx new file mode 100644 index 000000000..72c85bae8 Binary files /dev/null and b/user-signal-service/thrift/src/main/thrift/signal.docx differ diff --git a/user-signal-service/thrift/src/main/thrift/signal.thrift b/user-signal-service/thrift/src/main/thrift/signal.thrift deleted file mode 100644 index e32947be8..000000000 --- a/user-signal-service/thrift/src/main/thrift/signal.thrift +++ /dev/null @@ -1,113 +0,0 @@ -namespace java com.twitter.usersignalservice.thriftjava -namespace py gen.twitter.usersignalservice.signal -#@namespace scala com.twitter.usersignalservice.thriftscala -#@namespace strato com.twitter.usersignalservice.strato - -include "com/twitter/simclusters_v2/identifier.thrift" - - -enum SignalType { - /** - Please maintain the key space rule to avoid compatibility issue for the downstream production job - * Prod Key space: 0-1000 - * Devel Key space: 1000+ - **/ - - - /* tweet based signals */ - TweetFavorite = 0, // 540 Days Looback window - Retweet = 1, // 540 Days Lookback window - TrafficAttribution = 2, - OriginalTweet = 3, // 540 Days Looback window - Reply = 4, // 540 Days Looback window - /* Tweets that the user shared (sharer side) - * V1: successful shares (click share icon -> click in-app, or off-platform share option - * or copying link) - * */ - TweetShare_V1 = 5, // 14 Days Lookback window - - TweetFavorite_90D_V2 = 6, // 90 Days Lookback window : tweet fav from user with recent engagement in the past 90 days - Retweet_90D_V2 = 7, // 90 Days Lookback window : retweet from user with recent engagement in the past 90 days - OriginalTweet_90D_V2 = 8, // 90 Days Lookback window : original tweet from user with recent engagement in the past 90 days - Reply_90D_V2 = 9,// 90 Days Lookback window : reply from user with recent engagement in the past 90 days - GoodTweetClick = 10,// GoodTweetCilick Signal : Dwell Time Threshold >=2s - - // video tweets that were watched (10s OR 95%) in the past 90 days, are not ads, and have >=10s video - VideoView_90D_Quality_V1 = 11 // 90 Days Lookback window - // video tweets that were watched 50% in the past 90 days, are not ads, and have >=10s video - VideoView_90D_Playback50_V1 = 12 // 90 Days Lookback window - - /* user based signals */ - AccountFollow = 100, // infinite lookback window - RepeatedProfileVisit_14D_MinVisit2_V1 = 101, - RepeatedProfileVisit_90D_MinVisit6_V1 = 102, - RepeatedProfileVisit_180D_MinVisit6_V1 = 109, - RepeatedProfileVisit_14D_MinVisit2_V1_No_Negative = 110, - RepeatedProfileVisit_90D_MinVisit6_V1_No_Negative = 111, - RepeatedProfileVisit_180D_MinVisit6_V1_No_Negative = 112, - RealGraphOon = 104, - TrafficAttributionProfile_30D_LastVisit = 105, - TrafficAttributionProfile_30D_DecayedVisit = 106, - TrafficAttributionProfile_30D_WeightedEventDecayedVisit = 107, - TrafficAttributionProfile_30D_DecayedVisit_WithoutAgathaFilter = 108, - GoodProfileClick = 120, // GoodTweetCilick Signal : Dwell Time Threshold >=10s - AdFavorite = 121, // Favorites filtered to ads TweetFavorite has both organic and ads Favs - - // AccountFollowWithDelay should only be used by high-traffic clients and has 1 min delay - AccountFollowWithDelay = 122, - - - /* notifications based signals */ - /* V1: notification clicks from past 90 days with negative events (reports, dislikes) being filtered */ - NotificationOpenAndClick_V1 = 200, - - /* - negative signals for filter - */ - NegativeEngagedTweetId = 901 // tweetId for all negative engagements - NegativeEngagedUserId = 902 // userId for all negative engagements - AccountBlock = 903, - AccountMute = 904, - // skip 905 - 906 for Account report abuse / report spam - // User clicked dont like from past 90 Days - TweetDontLike = 907 - // User clicked see fewer on the recommended tweet from past 90 Days - TweetSeeFewer = 908 - // User clicked on the "report tweet" option in the tweet caret dropdown menu from past 90 days - TweetReport = 909 - - /* - devel signals - use the num > 1000 to test out signals under development/ddg - put it back to the correct corresponding Key space (0-1000) before ship - */ - GoodTweetClick_5s = 1001,// GoodTweetCilick Signal : Dwell Time Threshold >=5s - GoodTweetClick_10s = 1002,// GoodTweetCilick Signal : Dwell Time Threshold >=10s - GoodTweetClick_30s = 1003,// GoodTweetCilick Signal : Dwell Time Threshold >=30s - - GoodProfileClick_20s = 1004,// GoodProfileClick Signal : Dwell Time Threshold >=20s - GoodProfileClick_30s = 1005,// GoodProfileClick Signal : Dwell Time Threshold >=30s - - GoodProfileClick_Filtered = 1006, // GoodProfileClick Signal filtered by blocks and mutes. - GoodProfileClick_20s_Filtered = 1007// GoodProfileClick Signal : Dwell Time Threshold >=20s, filtered byblocks and mutes. - GoodProfileClick_30s_Filtered = 1008,// GoodProfileClick Signal : Dwell Time Threshold >=30s, filtered by blocks and mutes. - - /* - Unified Signals - These signals are aimed to unify multiple signal fetches into a single response. - This might be a healthier way for our retrievals layer to run inference on. - */ - TweetBasedUnifiedUniformSignal = 1300 - TweetBasedUnifiedEngagementWeightedSignal = 1301 - TweetBasedUnifiedQualityWeightedSignal = 1302 - ProducerBasedUnifiedUniformSignal = 1303 - ProducerBasedUnifiedEngagementWeightedSignal = 1304 - ProducerBasedUnifiedQualityWeightedSignal = 1305 - -} - -struct Signal { - 1: required SignalType signalType - 2: required i64 timestamp - 3: optional identifier.InternalId targetInternalId -} diff --git a/visibilitylib/BUILD b/visibilitylib/BUILD deleted file mode 100644 index 76ea1a659..000000000 --- a/visibilitylib/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -target( - dependencies = [ - "visibility/lib/src/main/scala/com/twitter/visibility", - ], -) - -target( - name = "conversations", - dependencies = [ - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/tweets", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/conversations", - ], -) - -target( - name = "tweets", - dependencies = [ - "visibility/lib/src/main/scala/com/twitter/visibility/generators", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/tweets", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/tweets", - ], -) - -target( - name = "users", - dependencies = [ - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/users", - ], -) diff --git a/visibilitylib/BUILD.docx b/visibilitylib/BUILD.docx new file mode 100644 index 000000000..eadfbe4e0 Binary files /dev/null and b/visibilitylib/BUILD.docx differ diff --git a/visibilitylib/README.docx b/visibilitylib/README.docx new file mode 100644 index 000000000..3e18fdd6e Binary files /dev/null and b/visibilitylib/README.docx differ diff --git a/visibilitylib/README.md b/visibilitylib/README.md deleted file mode 100644 index 28c7af03f..000000000 --- a/visibilitylib/README.md +++ /dev/null @@ -1,51 +0,0 @@ -Overview -======== - -Visibility Filtering is a centralized rule engine that instructs clients how to alter the display of certain Twitter content on read time. The Visibility Filtering library is responsible for filtering Twitter content to support legal compliance, improve product quality, increase user trust, protect revenue through the use of hard-filtering, visible product treatments, and coarse-grained downranking. - -Notice -====== - -Visibility Filtering library is currently being reviewed and rebuilt, and part of the code has been removed and is not ready to be shared yet. The remaining part of the code needs further review and will be shared once it’s ready. Also code comments have been sanitized. - -SafetyLevel -=========== - -Represents the product context in which the Viewer is requesting to view the Content (e.g. Timeline, Profile). - -Features -======== - -Include safety labels and other metadata of a Tweet, flags set on a User (including the Viewer), relationships between Users (e.g. block, follow), User settings, relationships between Users and Content (e.g. reported for spam). - -Action -====== - -The way the Visibility Framework instructs the client to respond to the Viewer’s request for Content, and can include hard filtering (e.g. Drop), soft filtering (e.g. Labels and Interstitials), ranking clues, etc. - -Condition -========= - -Returns a boolean when given map of Features. Conditions can be combined to determine if a Rule should return an Action or the default (Allow). - -Policy -====== - -Rules are expressed as a sequence in priority order to create a Visibility Policy. The library has one policy -per SafetyLevel. - -RuleEngine -=========== - -Evaluates the Action for a Request. - -SafetyLabel -=========== - -A primary labeling mechanism for Safety. A labeled entity associates with tweet, user, Direct Message, media, space etc. Safety labels power different ways of remediations (e.g. applying a SafetyLabel that results in tweet interstitial or notice). - -SafetyLabelType -=============== - -Describes a particular policy violation for a given noun instance, and usually leads to reduced visibility of the -labeled entity in product surfaces. There are many deprecated, and experimental safety label types. Labels with these safety label types have no effect on VF. Additionally, some safety label types are not used, and not designed for VF. diff --git a/visibilitylib/src/main/resources/config/BUILD b/visibilitylib/src/main/resources/config/BUILD deleted file mode 100644 index b45c857ae..000000000 --- a/visibilitylib/src/main/resources/config/BUILD +++ /dev/null @@ -1,6 +0,0 @@ -resources( - sources = [ - "com/twitter/visibility/*.csv", - "com/twitter/visibility/*.yml", - ], -) diff --git a/visibilitylib/src/main/resources/config/BUILD.docx b/visibilitylib/src/main/resources/config/BUILD.docx new file mode 100644 index 000000000..93295bad9 Binary files /dev/null and b/visibilitylib/src/main/resources/config/BUILD.docx differ diff --git a/visibilitylib/src/main/resources/config/com/twitter/visibility/decider.docx b/visibilitylib/src/main/resources/config/com/twitter/visibility/decider.docx new file mode 100644 index 000000000..26ee4b698 Binary files /dev/null and b/visibilitylib/src/main/resources/config/com/twitter/visibility/decider.docx differ diff --git a/visibilitylib/src/main/resources/config/com/twitter/visibility/decider.yml b/visibilitylib/src/main/resources/config/com/twitter/visibility/decider.yml deleted file mode 100644 index 54b5edcba..000000000 --- a/visibilitylib/src/main/resources/config/com/twitter/visibility/decider.yml +++ /dev/null @@ -1,909 +0,0 @@ - -visibility_library_enable_all_subscribed_lists_safety_level: - default_availability: 10000 - -visibility_library_enable_ads_business_settings_safety_level: - default_availability: 10000 - -visibility_library_enable_ads_campaign_safety_level: - default_availability: 10000 - -visibility_library_enable_ads_manager_safety_level: - default_availability: 10000 - -visibility_library_enable_appeals_safety_level: - default_availability: 10000 - -visibility_library_enable_article_tweet_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_birdwatch_note_author_safety_level: - default_availability: 10000 - -visibility_library_enable_birdwatch_note_tweets_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_birdwatch_needs_your_help_notifications_safety_level: - default_availability: 10000 - -visibility_library_enable_block_mute_users_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_brand_safety_safety_level: - default_availability: 10000 - -visibility_library_enable_card_poll_voting_safety_level: - default_availability: 10000 - -visibility_library_enable_cards_service_safety_level: - default_availability: 10000 - -visibility_library_enable_communities_safety_level: - default_availability: 10000 - -visibility_library_enable_conversation_focal_prehydration_safety_level: - default_availability: 10000 - -visibility_library_enable_conversation_focal_tweet_safety_level: - default_availability: 10000 - -visibility_library_enable_conversation_injected_tweet_safety_level: - default_availability: 10000 - -visibility_library_enable_conversation_reply_safety_level: - default_availability: 10000 - -visibility_library_curated_trends_representative_tweet: - default_availability: 10000 - -visibility_library_curation_policy_violations: - default_availability: 10000 - -visibility_library_enable_deprecated_safety_level_safety_level: - default_availability: 10000 - -visibility_library_enable_dev_platform_get_list_tweets_safety_level: - default_availability: 10000 - -visibility_library_enable_des_following_and_followers_user_list_safety_level: - default_availability: 10000 - -visibility_library_enable_des_home_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_des_quote_tweet_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_des_realtime_safety_level: - default_availability: 10000 - -visibility_library_enable_des_realtime_spam_enrichment_safety_level: - default_availability: 10000 - -visibility_library_enable_des_realtime_tweet_filter_safety_level: - default_availability: 10000 - -visibility_library_enable_des_retweeting_users_safety_level: - default_availability: 10000 - -visibility_library_enable_des_tweet_detail_safety_level: - default_availability: 10000 - -visibility_library_enable_des_tweet_liking_users_safety_level: - default_availability: 10000 - -visibility_library_enable_des_user_bookmarks_safety_level: - default_availability: 10000 - -visibility_library_enable_des_user_liked_tweets_safety_level: - default_availability: 10000 - -visibility_library_enable_des_user_mentions_safety_level: - default_availability: 10000 - -visibility_library_enable_des_user_tweets_safety_level: - default_availability: 10000 - -visibility_library_enable_dev_platform_compliance_stream_safety_level: - default_availability: 10000 - -visibility_library_enable_direct_messages_safety_level: - default_availability: 10000 - -visibility_library_enable_direct_messages_conversation_list_safety_level: - default_availability: 10000 - -visibility_library_enable_direct_messages_conversation_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_direct_messages_inbox_safety_level: - default_availability: 10000 - -visibility_library_enable_direct_messages_muted_users_safety_level: - default_availability: 10000 - -visibility_library_enable_direct_messages_pinned_safety_level: - default_availability: 10000 - -visibility_library_enable_elevated_quote_tweet_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_direct_messages_search_safety_level: - default_availability: 10000 - -visibility_library_enable_embedded_tweet_safety_level: - default_availability: 10000 - -visibility_library_enable_embeds_public_interest_notice_safety_level: - default_availability: 10000 - -visibility_library_enable_embed_tweet_markup_safety_level: - default_availability: 10000 - -visibility_library_enable_write_path_limited_actions_enforcement_safety_level: - default_availability: 10000 - -visibility_library_enable_filter_all_safety_level: - default_availability: 10000 - -visibility_library_enable_filter_all_placeholder_safety_level: - default_availability: 10000 - -visibility_library_enable_filter_default_safety_level: - default_availability: 10000 - -visibility_library_enable_filter_none_safety_level: - default_availability: 10000 - -visibility_library_enable_followed_topics_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_follower_connections_safety_level: - default_availability: 10000 - -visibility_library_enable_following_and_followers_user_list_safety_level: - default_availability: 10000 - -visibility_library_enable_for_development_only_safety_level: - default_availability: 10000 - -visibility_library_enable_friends_following_list_safety_level: - default_availability: 10000 - -visibility_library_enable_graphql_default_safety_level: - default_availability: 10000 - -visibility_library_enable_gryphon_decks_and_columns_safety_level: - default_availability: 10000 - -visibility_library_enable_humanization_nudge_safety_level: - default_availability: 10000 - -visibility_library_enable_kitchen_sink_development_safety_level: - default_availability: 10000 - -visibility_library_enable_list_header_safety_level: - default_availability: 10000 - -visibility_library_enable_list_memberships_safety_level: - default_availability: 10000 - -visibility_library_enable_list_ownerships_safety_level: - default_availability: 10000 - -visibility_library_enable_list_recommendations_safety_level: - default_availability: 10000 - -visibility_library_enable_list_search_safety_level: - default_availability: 10000 - -visibility_library_enable_list_subscriptions_safety_level: - default_availability: 10000 - -visibility_library_enable_live_pipeline_engagement_counts_safety_level: - default_availability: 10000 - -visibility_library_enable_live_video_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_magic_recs_safety_level: - default_availability: 10000 - -visibility_library_enable_magic_recs_aggressive_safety_level: - default_availability: 10000 - -visibility_library_enable_magic_recs_aggressive_v2_safety_level: - default_availability: 10000 - -visibility_library_enable_magic_recs_v2_safety_level: - default_availability: 10000 - -visibility_library_enable_minimal_safety_level: - default_availability: 10000 - -visibility_library_enable_moderated_tweets_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_moments_safety_level: - default_availability: 10000 - -visibility_library_enable_nearby_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_new_user_experience_safety_level: - default_availability: 10000 - -visibility_library_enable_notifications_ibis_safety_level: - default_availability: 10000 - -visibility_library_enable_notifications_platform_safety_level: - default_availability: 10000 - -visibility_library_enable_notifications_platform_push_safety_level: - default_availability: 10000 - -visibility_library_enable_notifications_read_safety_level: - default_availability: 10000 - -visibility_library_enable_notifications_timeline_device_follow_safety_level: - default_availability: 10000 - -visibility_library_enable_notifications_write_safety_level: - default_availability: 10000 - -visibility_library_enable_notification_writer_v2_safety_level: - default_availability: 10000 - -visibility_library_enable_notifications_writer_tweet_hydrator_safety_level: - default_availability: 10000 - -visibility_library_enable_quick_promote_tweet_eligibility_safety_level: - default_availability: 10000 - -visibility_library_enable_quote_tweet_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_quoted_tweet_rules_safety_level: - default_availability: 10000 - -visibility_library_enable_recommendations_safety_level: - default_availability: 10000 - -visibility_library_enable_recos_video_safety_level: - default_availability: 10000 - -visibility_library_enable_recos_write_path_safety_level: - default_availability: 10000 - -visibility_library_enable_replies_grouping_safety_level: - default_availability: 10000 - -visibility_library_enable_report_center_safety_level: - default_availability: 10000 - -visibility_library_enable_returning_user_experience_safety_level: - default_availability: 10000 - -visibility_library_enable_returning_user_experience_focal_tweet_safety_level: - default_availability: 10000 - -visibility_library_enable_revenue_safety_level: - default_availability: 10000 - -visibility_library_enable_rito_actioned_tweet_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_safe_search_minimal_safety_level: - default_availability: 10000 - -visibility_library_enable_safe_search_strict_safety_level: - default_availability: 10000 - -visibility_library_enable_search_hydration_safety_level: - default_availability: 10000 - -visibility_library_enable_search_latest_safety_level: - default_availability: 10000 - -visibility_library_enable_search_mixer_srp_minimal_safety_level: - default_availability: 10000 - -visibility_library_enable_search_mixer_srp_strict_safety_level: - default_availability: 10000 - -visibility_library_enable_user_search_srp_safety_level: - default_availability: 10000 - -visibility_library_enable_user_search_typeahead_safety_level: - default_availability: 10000 - -visibility_library_enable_search_people_srp_safety_level: - default_availability: 10000 - -visibility_library_enable_search_people_typeahead_safety_level: - default_availability: 10000 - -visibility_library_enable_search_photo_safety_level: - default_availability: 10000 - -visibility_library_enable_search_top_safety_level: - default_availability: 10000 - -visibility_library_enable_search_trend_takeover_promoted_tweet_safety_level: - default_availability: 10000 - -visibility_library_enable_search_video_safety_level: - default_availability: 10000 - -visibility_library_enable_search_latest_user_rules_safety_level: - default_availability: 10000 - -visibility_library_enable_shopping_manager_spy_mode_safety_level: - default_availability: 10000 - -visibility_library_enable_signals_reactions_safety_level: - default_availability: 10000 - -visibility_library_enable_signals_tweet_reacting_users_safety_level: - default_availability: 10000 - -visibility_library_enable_social_proof_safety_level: - default_availability: 10000 - -visibility_library_enable_soft_intervention_pivot_safety_level: - default_availability: 10000 - -visibility_library_enable_space_fleetline_safety_level: - default_availability: 10000 - -visibility_library_enable_space_home_timeline_upranking_safety_level: - default_availability: 10000 - -visibility_library_enable_space_join_screen_safety_level: - default_availability: 10000 - -visibility_library_enable_space_notifications_safety_level: - default_availability: 10000 - -visibility_library_enable_spaces_safety_level: - default_availability: 10000 - -visibility_library_enable_spaces_participants_safety_level: - default_availability: 0 - -visibility_library_enable_spaces_seller_application_status_safety_level: - default_availability: 10000 - -visibility_library_enable_spaces_sharing_safety_level: - default_availability: 10000 - -visibility_library_enable_space_tweet_avatar_home_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_stickers_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_strato_ext_limited_engagements_safety_level: - default_availability: 10000 - -visibility_library_enable_stream_services_safety_level: - default_availability: 10000 - -visibility_library_enable_test_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_bookmark_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_content_controls_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_conversations_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_conversations_downranking_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_conversations_downranking_minimal_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_favorites_safety_level: - default_availability: 10000 - -visibility_library_enable_self_view_timeline_favorites_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_focal_tweet_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_following_activity_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_home_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_home_communities_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_home_hydration_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_home_latest_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_home_recommendations_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_home_topic_follow_recommendations_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_scorer_safety_level: - default_availability: 10000 - -visibility_library_enable_topics_landing_page_topic_recommendations_safety_level: - default_availability: 10000 - -visibility_library_enable_explore_recommendations_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_moderated_tweets_hydration_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_injection_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_liked_by_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_lists_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_media_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_mentions_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_profile_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_profile_all_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_profile_spaces_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_profile_super_follows_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_reactive_blending_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_retweeted_by_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_super_liked_by_safety_level: - default_availability: 10000 - -visibility_library_enable_tombstoning_safety_level: - default_availability: 10000 - -visibility_library_enable_trends_representative_tweet_safety_level: - default_availability: 10000 - -visibility_library_enable_trusted_friends_user_list_safety_level: - default_availability: 10000 - -visibility_library_enable_twitter_delegate_user_list_safety_level: - default_availability: 10000 - -visibility_library_enable_tweet_detail_safety_level: - default_availability: 10000 - -visibility_library_enable_tweet_detail_non_too_safety_level: - default_availability: 10000 - -visibility_library_enable_tweet_detail_with_injections_hydration_safety_level: - default_availability: 10000 - -visibility_library_enable_tweet_engagers_safety_level: - default_availability: 10000 - -visibility_library_enable_tweet_reply_nudge_safety_level: - default_availability: 10000 - -visibility_library_enable_tweet_scoped_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_tweet_writes_api_safety_level: - default_availability: 10000 - -visibility_library_enable_twitter_article_compose_safety_level: - default_availability: 10000 - -visibility_library_enable_twitter_article_profile_tab_safety_level: - default_availability: 10000 - -visibility_library_enable_twitter_article_read_safety_level: - default_availability: 10000 - -visibility_library_enable_user_profile_header_safety_level: - default_availability: 10000 - -visibility_library_enable_user_milestone_recommendation_safety_level: - default_availability: 10000 - -visibility_library_enable_user_scoped_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_user_settings_safety_level: - default_availability: 10000 - -visibility_library_enable_video_ads_safety_level: - default_availability: 10000 - -visibility_library_enable_timeline_home_promoted_hydration_safety_level: - default_availability: 10000 - -visibility_library_enable_super_follower_connnections_safety_level: - default_availability: 10000 - -visibility_library_enable_super_like_safety_level: - default_availability: 10000 - -visibility_library_enable_topic_recommendations_safety_level: - default_availability: 10000 - -visibility_library_enable_ads_reporting_dashboard_safety_level: - default_availability: 10000 - -visibility_library_enable_search_top_qig_safety_level: - default_availability: 10000 - -visibility_library_enable_content_control_tool_install_safety_level: - default_availability: 10000 - -visibility_library_enable_conversation_control_rules: - default_availability: 10000 - -visibility_library_enable_community_tweets_rules: - default_availability: 10000 - -visibility_library_enable_drop_community_tweet_with_undefined_community_rule: - default_availability: 10000 - -visibility_library_enable_p_spammy_tweet_downrank_convos_low_quality: - default_availability: 10000 - -visibility_library_enable_high_p_spammy_tweet_score_search_tweet_label_drop_rule: - default_availability: 10000 - -visibility_library_enable_rito_actioned_tweet_downrank_convos_low_quality: - default_availability: 10000 - -visibility_library_enable_toxic_reply_filter_conversation: - default_availability: 10000 - -visibility_library_enable_toxic_reply_filter_notifications: - default_availability: 10000 - -visibility_library_enable_new_sensitive_media_settings_interstitial_rules_home_timeline: - default_availability: 10000 - -visibility_library_enable_legacy_sensitive_media_rules_home_timeline: - default_availability: 10000 - -visibility_library_enable_new_sensitive_media_settings_interstitial_rules_conversation: - default_availability: 10000 - -visibility_library_enable_legacy_sensitive_media_rules_conversation: - default_availability: 10000 - -visibility_library_enable_new_sensitive_media_settings_interstitials_rules_profile_timeline: - default_availability: 10000 - -visibility_library_enable_legacy_sensitive_media_rules_profile_timeline: - default_availability: 10000 - -visibility_library_enable_new_sensitive_media_settings_interstitials_rules_tweet_detail: - default_availability: 10000 - -visibility_library_enable_legacy_sensitive_media_rules_tweet_detail: - default_availability: 10000 - -visibility_library_enable_legacy_sensitive_media_rules_direct_messages: - default_availability: 10000 - -visibility_library_enable_smyte_spam_tweet_rule: - default_availability: 10000 - -visibility_library_enable_high_spammy_tweet_content_score_search_latest_tweet_label_drop_rule: - default_availability: 10000 - -visibility_library_enable_high_spammy_tweet_content_score_search_top_tweet_label_drop_rule: - default_availability: 10000 - -visibility_library_enable_high_spammy_tweet_content_score_convo_downrank_abusive_quality_rule: - default_availability: 10000 - -visibility_library_enable_high_cryptospam_score_convo_downrank_abusive_quality_rule: - default_availability: 10000 - -visibility_library_enable_high_spammy_tweet_content_score_trends_top_tweet_label_drop_rule: - default_availability: 10000 - -visibility_library_enable_high_spammy_tweet_content_score_trends_latest_tweet_label_drop_rule: - default_availability: 10000 - -visibility_library_enable_gore_and_violence_topic_high_recall_tweet_label_rule: - default_availability: 10000 - -visibility_library_enable_limit_replies_followers_conversation_rule: - default_availability: 10000 - -visibility_library_enable_blink_bad_downranking_rule: - default_availability: 10000 - -visibility_library_enable_blink_worst_downranking_rule: - default_availability: 10000 - -visibility_library_enable_copypasta_spam_downrank_convos_abusive_quality_rule: - default_availability: 10000 - -visibility_library_enable_copypasta_spam_search_drop_rule: - default_availability: 10000 - -visibility_library_enable_spammy_user_model_high_precision_drop_tweet_rule: - default_availability: 10000 - -visibility_library_enable_avoid_nsfw_rules: - default_availability: 10000 - -visibility_library_enable_reported_tweet_interstitial_rule: - default_availability: 10000 - -visibility_library_enable_reported_tweet_interstitial_search_rule: - default_availability: 10000 - -visibility_library_enable_drop_exclusive_tweet_content_rule: - default_availability: 10000 - -visibility_library_enable_drop_exclusive_tweet_content_rule_fail_closed: - default_availability: 10000 - -visibility_library_enable_drop_all_exclusive_tweets_rule: - default_availability: 10000 - -visibility_library_enable_drop_all_exclusive_tweets_rule_fail_closed: - default_availability: 10000 - -visibility_library_enable_tombstone_exclusive_quoted_tweet_content_rule: - default_availability: 10000 - -visibility_library_enable_downrank_spam_reply_sectioning_rule: - default_availability: 10000 - -visibility_library_enable_nsfw_text_sectioning_rule: - default_availability: 10000 - -visibility_library_enable_search_ipi_safe_search_without_user_in_query_drop_rule: - default_availability: 10000 - -visibility_library_enable_timeline_home_promoted_tweet_health_enforcement_rules: - default_availability: 10000 - -visibility_library_enable_muted_keyword_filtering_space_title_notifications_rule: - default_availability: 10000 - -visibility_library_enable_drop_tweets_with_georestricted_media_rule: - default_availability: 10000 - -visibility_library_enable_drop_all_trusted_friends_tweets_rule: - default_availability: 10000 - -visibility_library_enable_drop_all_trusted_friends_tweet_content_rule: - default_availability: 10000 - -visibility_library_enable_drop_all_collab_invitation_tweets_rule: - default_availability: 10000 - -visibility_library_enable_fetch_tweet_reported_perspective: - default_availability: 10000 - -visibility_library_enable_fetch_tweet_media_metadata: - default_availability: 10000 - -visibility_library_enable_follow_check_in_mutedkeyword: - default_availability: 10000 - -visibility_library_enable_media_interstitial_composition: - default_availability: 10000 - -visibility_library_enable_verdict_scribing_from_tweet_visibility_library: - default_availability: 0 - -visibility_library_enable_verdict_logger_event_publisher_instantiation_from_tweet_visibility_library: - default_availability: 10000 - -visibility_library_enable_verdict_scribing_from_timeline_conversations_visibility_library: - default_availability: 0 - -visibility_library_enable_verdict_logger_event_publisher_instantiation_from_timeline_conversations_visibility_library: - default_availability: 10000 - -visibility_library_enable_verdict_scribing_from_blender_visibility_library: - default_availability: 0 - -visibility_library_enable_verdict_logger_event_publisher_instantiation_from_blender_visibility_library: - default_availability: 10000 - -visibility_library_enable_verdict_scribing_from_search_visibility_library: - default_availability: 0 - -visibility_library_enable_verdict_logger_event_publisher_instantiation_from_search_visibility_library: - default_availability: 0 - -visibility_library_enable_localized_tombstones_on_visibility_results: - default_availability: 10000 - -visibility_library_enable_short_circuiting_from_tweet_visibility_library: - default_availability: 10000 - -visibility_library_enable_card_visibility_library_card_uri_parsing: - default_availability: 10000 - -visibility_library_enable_short_circuiting_from_timeline_conversations_visibility_library: - default_availability: 10000 - -visibility_library_enable_short_circuiting_from_blender_visibility_library: - default_availability: 10000 - -visibility_library_enable_short_circuiting_from_search_visibility_library: - default_availability: 0 - -visibility_library_enable_nsfw_text_high_precision_drop_rule: - default_availability: 10000 - -visibility_library_enable_spammy_tweet_rule_verdict_logging: - default_availability: 0 - -visibility_library_enable_downlevel_rule_verdict_logging: - default_availability: 0 - -visibility_library_enable_likely_likely_ivs_user_label_drop_rule: - default_availability: 10000 - -visibility_library_enable_card_uri_root_domain_deny_list_rule: - default_availability: 10000 - -visibility_library_enable_community_non_member_poll_card_rule: - default_availability: 10000 - -visibility_library_enable_community_non_member_poll_card_rule_fail_closed: - default_availability: 10000 - -visibility_library_enable_experimental_nudge_label_rule: - default_availability: 10000 - -visibility_library_enable_user_self_view_only_safety_level: - default_availability: 10000 - -visibility_library_nsfw_high_precision_user_label_avoid_tweet_rule_enabled: - default_availability: 10000 - -visibility_library_enable_new_ad_avoidance_rules: - default_availability: 10000 - -visibility_library_enable_nsfa_high_recall_ad_avoidance_rules: - default_availability: 0 - -visibility_library_enable_nsfa_keywords_high_precision_ad_avoidance_rules: - default_availability: 0 - -visibility_library_enable_stale_tweet_drop_rule: - default_availability: 10000 - -visibility_library_enable_stale_tweet_drop_rule_fail_closed: - default_availability: 10000 - -visibility_library_enable_edit_history_timeline_safety_level: - default_availability: 10000 - -visibility_library_enable_delete_state_tweet_rules: - default_availability: 10000 - -visibility_library_enable_spaces_sharing_nsfw_drop_rule: - default_availability: 10000 - -visibility_library_enable_viewer_is_soft_user_drop_rule: - default_availability: 10000 - -visibility_library_enable_backend_limited_actions: - default_availability: 10000 - -visibility_library_enable_base_qig_safety_level: - default_availability: 10000 - -visibility_library_enable_notifications_qig_safety_level: - default_availability: 10000 - -visibility_library_enable_access_internal_promoted_content_safety_level: - default_availability: 10000 - -visibility_library_enable_pdna_quoted_tweet_tombstone_rule: - default_availability: 10000 - -visibility_library_enable_spam_quoted_tweet_tombstone_rule: - default_availability: 10000 - -visibility_library_enable_nsfw_hp_quoted_tweet_drop_experiment_rule: - default_availability: 10000 - -visibility_library_enable_nsfw_hp_quoted_tweet_tombstone_experiment_rule: - default_availability: 10000 - -visibility_library_enable_inner_quoted_tweet_viewer_blocks_author_interstitial_rule: - default_availability: 10000 - -visibility_library_enable_inner_quoted_tweet_viewer_mutes_author_interstitial_rule: - default_availability: 10000 - -visibility_library_enable_experimental_rule_engine: - default_availability: 10000 - -visibility_library_enable_fosnr_rules: - default_availability: 0 - -visibility_library_enable_localized_interstitial_generator: - default_availability: 10000 - -visibility_library_convos_enable_legacy_interstitial: - default_availability: 10000 - -visibility_library_convos_enable_localized_interstitial: - default_availability: 10000 - -visibility_library_enable_profile_mixer_media_safety_level: - default_availability: 10000 - -visibility_library_enable_profile_mixer_favorites_safety_level: - default_availability: 10000 - -visibility_library_enable_zipbird_consumer_archives_safety_level: - default_availability: 10000 - -visibility_library_enable_tweet_award_safety_level: - default_availability: 10000 - -visibility_library_disable_legacy_interstitial_filtered_reason: - default_availability: 10000 - -visibility_library_enable_search_basic_block_mute_rules: - default_availability: 10000 - -visibility_library_enable_localized_interstitial_user_state_lib: - default_availability: 10000 - -visibility_library_enable_abusive_behavior_drop_rule: - default_availability: 10000 - -visibility_library_enable_abusive_behavior_interstitial_rule: - default_availability: 10000 - -visibility_library_enable_abusive_behavior_limited_engagements_rule: - default_availability: 10000 - -visibility_library_enable_not_graduated_downrank_convos_abusive_quality_rule: - default_availability: 0 - -visibility_library_enable_not_graduated_search_drop_rule: - default_availability: 0 - -visibility_library_enable_not_graduated_drop_rule: - default_availability: 0 - -visibility_library_enable_memoize_safety_level_params: - default_availability: 0 - -visibility_library_enable_author_blocks_viewer_drop_rule: - default_availability: 0 diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/BUILD deleted file mode 100644 index 47501c7d4..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "abdecider/src/main/scala", - "configapi/configapi-core", - "decider/src/main/scala", - "featureswitches/featureswitches-core/src/main/scala", - "servo/decider/src/main/scala", - "servo/util/src/main/scala", - "stitch/stitch-core", - "util/util-logging/src/main/scala", - "util/util-stats/src/main/scala", - "visibility/common/src/main/scala/com/twitter/visibility/common/actions", - "visibility/common/src/main/scala/com/twitter/visibility/common/stitch", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility/builder", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/configs", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/params", - "visibility/lib/src/main/scala/com/twitter/visibility/engine", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/generators", - "visibility/lib/src/main/scala/com/twitter/visibility/models", - "visibility/lib/src/main/scala/com/twitter/visibility/rules", - "visibility/lib/src/main/scala/com/twitter/visibility/rules/generators", - "visibility/lib/src/main/scala/com/twitter/visibility/rules/providers", - "visibility/lib/src/main/scala/com/twitter/visibility/util", - ], - exports = [ - "configapi/configapi-core", - "visibility/common/src/main/scala/com/twitter/visibility/common/actions", - "visibility/lib/src/main/scala/com/twitter/visibility/builder", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/configs", - "visibility/lib/src/main/scala/com/twitter/visibility/models", - "visibility/lib/src/main/scala/com/twitter/visibility/rules", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/BUILD.docx new file mode 100644 index 000000000..55db11d84 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/VisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/VisibilityLibrary.docx new file mode 100644 index 000000000..ffd985003 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/VisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/VisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/VisibilityLibrary.scala deleted file mode 100644 index 1e89c8818..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/VisibilityLibrary.scala +++ /dev/null @@ -1,387 +0,0 @@ -package com.twitter.visibility - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.abdecider.NullABDecider -import com.twitter.decider.Decider -import com.twitter.decider.NullDecider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.featureswitches.v2.NullFeatureSwitches -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.logging.NullLogger -import com.twitter.servo.util.Gate -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params -import com.twitter.util.Try -import com.twitter.visibility.builder._ -import com.twitter.visibility.common.stitch.StitchHelpers -import com.twitter.visibility.configapi.VisibilityParams -import com.twitter.visibility.configapi.configs.VisibilityDeciderGates -import com.twitter.visibility.engine.DeciderableVisibilityRuleEngine -import com.twitter.visibility.engine.VisibilityResultsMetricRecorder -import com.twitter.visibility.engine.VisibilityRuleEngine -import com.twitter.visibility.engine.VisibilityRulePreprocessor -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.models.ContentId -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext -import com.twitter.visibility.rules.EvaluationContext -import com.twitter.visibility.rules.Rule -import com.twitter.visibility.rules.generators.TweetRuleGenerator -import com.twitter.visibility.rules.providers.InjectedPolicyProvider -import com.twitter.visibility.util.DeciderUtil -import com.twitter.visibility.util.FeatureSwitchUtil -import com.twitter.visibility.util.LoggingUtil - -object VisibilityLibrary { - - object Builder { - - def apply(log: Logger, statsReceiver: StatsReceiver): Builder = new Builder( - log, - new MemoizingStatsReceiver(statsReceiver) - ) - } - - case class Builder( - log: Logger, - statsReceiver: StatsReceiver, - decider: Option[Decider] = None, - abDecider: Option[LoggingABDecider] = None, - featureSwitches: Option[FeatureSwitches] = None, - enableStitchProfiling: Gate[Unit] = Gate.False, - captureDebugStats: Gate[Unit] = Gate.False, - enableComposableActions: Gate[Unit] = Gate.False, - enableFailClosed: Gate[Unit] = Gate.False, - enableShortCircuiting: Gate[Unit] = Gate.True, - memoizeSafetyLevelParams: Gate[Unit] = Gate.False) { - - def withDecider(decider: Decider): Builder = copy(decider = Some(decider)) - - @deprecated("use .withDecider and pass in a decider that is properly configured per DC") - def withDefaultDecider(isLocal: Boolean, useLocalOverrides: Boolean = false): Builder = { - if (isLocal) { - withLocalDecider - } else { - withDecider( - DeciderUtil.mkDecider( - useLocalDeciderOverrides = useLocalOverrides, - )) - } - } - - def withLocalDecider(): Builder = withDecider(DeciderUtil.mkLocalDecider) - - def withNullDecider(): Builder = - withDecider(new NullDecider(isAvail = true, availabilityDefined = true)) - - def withABDecider(abDecider: LoggingABDecider, featureSwitches: FeatureSwitches): Builder = - abDecider match { - case abd: NullABDecider => - copy(abDecider = Some(abd), featureSwitches = Some(NullFeatureSwitches)) - case _ => - copy( - abDecider = Some(abDecider), - featureSwitches = Some(featureSwitches) - ) - } - - def withABDecider(abDecider: LoggingABDecider): Builder = abDecider match { - case abd: NullABDecider => - withABDecider(abDecider = abd, featureSwitches = NullFeatureSwitches) - case _ => - withABDecider( - abDecider = abDecider, - featureSwitches = - FeatureSwitchUtil.mkVisibilityLibraryFeatureSwitches(abDecider, statsReceiver) - ) - } - - def withClientEventsLogger(clientEventsLogger: Logger): Builder = - withABDecider(DeciderUtil.mkABDecider(Some(clientEventsLogger))) - - def withDefaultABDecider(isLocal: Boolean): Builder = - if (isLocal) { - withABDecider(NullABDecider) - } else { - withClientEventsLogger(LoggingUtil.mkDefaultLogger(statsReceiver)) - } - - def withNullABDecider(): Builder = withABDecider(NullABDecider) - - def withEnableStitchProfiling(gate: Gate[Unit]): Builder = - copy(enableStitchProfiling = gate) - - def withCaptureDebugStats(gate: Gate[Unit]): Builder = - copy(captureDebugStats = gate) - - def withEnableComposableActions(gate: Gate[Unit]): Builder = - copy(enableComposableActions = gate) - - def withEnableComposableActions(gateBoolean: Boolean): Builder = { - val gate = Gate.const(gateBoolean) - copy(enableComposableActions = gate) - } - - def withEnableFailClosed(gate: Gate[Unit]): Builder = - copy(enableFailClosed = gate) - - def withEnableFailClosed(gateBoolean: Boolean): Builder = { - val gate = Gate.const(gateBoolean) - copy(enableFailClosed = gate) - } - - def withEnableShortCircuiting(gate: Gate[Unit]): Builder = - copy(enableShortCircuiting = gate) - - def withEnableShortCircuiting(gateBoolean: Boolean): Builder = { - val gate = Gate.const(gateBoolean) - copy(enableShortCircuiting = gate) - } - - def memoizeSafetyLevelParams(gate: Gate[Unit]): Builder = - copy(memoizeSafetyLevelParams = gate) - - def memoizeSafetyLevelParams(gateBoolean: Boolean): Builder = { - val gate = Gate.const(gateBoolean) - copy(memoizeSafetyLevelParams = gate) - } - - def build(): VisibilityLibrary = { - - (decider, abDecider, featureSwitches) match { - case (None, _, _) => - throw new IllegalStateException( - "Decider is unset! If intentional, please call .withNullDecider()." - ) - - case (_, None, _) => - throw new IllegalStateException( - "ABDecider is unset! If intentional, please call .withNullABDecider()." - ) - - case (_, _, None) => - throw new IllegalStateException( - "FeatureSwitches is unset! This is a bug." - ) - - case (Some(d), Some(abd), Some(fs)) => - new VisibilityLibrary( - statsReceiver, - d, - abd, - VisibilityParams(log, statsReceiver, d, abd, fs), - enableStitchProfiling = enableStitchProfiling, - captureDebugStats = captureDebugStats, - enableComposableActions = enableComposableActions, - enableFailClosed = enableFailClosed, - enableShortCircuiting = enableShortCircuiting, - memoizeSafetyLevelParams = memoizeSafetyLevelParams) - } - } - } - - val nullDecider = new NullDecider(true, true) - - lazy val NullLibrary: VisibilityLibrary = new VisibilityLibrary( - NullStatsReceiver, - nullDecider, - NullABDecider, - VisibilityParams( - NullLogger, - NullStatsReceiver, - nullDecider, - NullABDecider, - NullFeatureSwitches), - enableStitchProfiling = Gate.False, - captureDebugStats = Gate.False, - enableComposableActions = Gate.False, - enableFailClosed = Gate.False, - enableShortCircuiting = Gate.True, - memoizeSafetyLevelParams = Gate.False - ) -} - -class VisibilityLibrary private[VisibilityLibrary] ( - baseStatsReceiver: StatsReceiver, - decider: Decider, - abDecider: LoggingABDecider, - visibilityParams: VisibilityParams, - enableStitchProfiling: Gate[Unit], - captureDebugStats: Gate[Unit], - enableComposableActions: Gate[Unit], - enableFailClosed: Gate[Unit], - enableShortCircuiting: Gate[Unit], - memoizeSafetyLevelParams: Gate[Unit]) { - - val statsReceiver: StatsReceiver = - new MemoizingStatsReceiver(baseStatsReceiver.scope("visibility_library")) - - val metricsRecorder = VisibilityResultsMetricRecorder(statsReceiver, captureDebugStats) - - val visParams: VisibilityParams = visibilityParams - - val visibilityDeciderGates = VisibilityDeciderGates(decider) - - val profileStats: MemoizingStatsReceiver = new MemoizingStatsReceiver( - statsReceiver.scope("profiling")) - - val perSafetyLevelProfileStats: StatsReceiver = profileStats.scope("for_safety_level") - - val featureMapBuilder: FeatureMapBuilder.Build = - FeatureMapBuilder(statsReceiver, enableStitchProfiling) - - private lazy val tweetRuleGenerator = new TweetRuleGenerator() - lazy val policyProvider = new InjectedPolicyProvider( - visibilityDeciderGates = visibilityDeciderGates, - tweetRuleGenerator = tweetRuleGenerator) - - val candidateVisibilityRulePreprocessor: VisibilityRulePreprocessor = VisibilityRulePreprocessor( - metricsRecorder, - policyProviderOpt = Some(policyProvider) - ) - - val fallbackVisibilityRulePreprocessor: VisibilityRulePreprocessor = VisibilityRulePreprocessor( - metricsRecorder) - - lazy val candidateVisibilityRuleEngine: VisibilityRuleEngine = VisibilityRuleEngine( - Some(candidateVisibilityRulePreprocessor), - metricsRecorder, - enableComposableActions, - enableFailClosed, - policyProviderOpt = Some(policyProvider) - ) - - lazy val fallbackVisibilityRuleEngine: VisibilityRuleEngine = VisibilityRuleEngine( - Some(fallbackVisibilityRulePreprocessor), - metricsRecorder, - enableComposableActions, - enableFailClosed) - - val ruleEngineVersionStatsReceiver = statsReceiver.scope("rule_engine_version") - def isReleaseCandidateEnabled: Boolean = visibilityDeciderGates.enableExperimentalRuleEngine() - - private def visibilityRuleEngine: DeciderableVisibilityRuleEngine = { - if (isReleaseCandidateEnabled) { - ruleEngineVersionStatsReceiver.counter("release_candidate").incr() - candidateVisibilityRuleEngine - } else { - ruleEngineVersionStatsReceiver.counter("fallback").incr() - fallbackVisibilityRuleEngine - } - } - - private def profileStitch[A](result: Stitch[A], safetyLevelName: String): Stitch[A] = - if (enableStitchProfiling()) { - StitchHelpers.profileStitch( - result, - Seq(profileStats, perSafetyLevelProfileStats.scope(safetyLevelName)) - ) - } else { - result - } - - def getParams(viewerContext: ViewerContext, safetyLevel: SafetyLevel): Params = { - if (memoizeSafetyLevelParams()) { - visibilityParams.memoized(viewerContext, safetyLevel) - } else { - visibilityParams(viewerContext, safetyLevel) - } - } - - def evaluationContextBuilder(viewerContext: ViewerContext): EvaluationContext.Builder = - EvaluationContext - .Builder(statsReceiver, visibilityParams, viewerContext) - .withMemoizedParams(memoizeSafetyLevelParams) - - def runRuleEngine( - contentId: ContentId, - featureMap: FeatureMap, - evaluationContextBuilder: EvaluationContext.Builder, - safetyLevel: SafetyLevel - ): Stitch[VisibilityResult] = - profileStitch( - visibilityRuleEngine( - evaluationContextBuilder.build(safetyLevel), - safetyLevel, - new VisibilityResultBuilder(contentId, featureMap), - enableShortCircuiting - ), - safetyLevel.name - ) - - def runRuleEngine( - contentId: ContentId, - featureMap: FeatureMap, - viewerContext: ViewerContext, - safetyLevel: SafetyLevel - ): Stitch[VisibilityResult] = - profileStitch( - visibilityRuleEngine( - EvaluationContext(safetyLevel, getParams(viewerContext, safetyLevel), statsReceiver), - safetyLevel, - new VisibilityResultBuilder(contentId, featureMap), - enableShortCircuiting - ), - safetyLevel.name - ) - - def runRuleEngine( - viewerContext: ViewerContext, - safetyLevel: SafetyLevel, - preprocessedResultBuilder: VisibilityResultBuilder, - preprocessedRules: Seq[Rule] - ): Stitch[VisibilityResult] = - profileStitch( - visibilityRuleEngine( - EvaluationContext(safetyLevel, getParams(viewerContext, safetyLevel), statsReceiver), - safetyLevel, - preprocessedResultBuilder, - enableShortCircuiting, - Some(preprocessedRules) - ), - safetyLevel.name - ) - - def runRuleEngineBatch( - contentIds: Seq[ContentId], - featureMapProvider: (ContentId, SafetyLevel) => FeatureMap, - viewerContext: ViewerContext, - safetyLevel: SafetyLevel, - ): Stitch[Seq[Try[VisibilityResult]]] = { - val params = getParams(viewerContext, safetyLevel) - profileStitch( - Stitch.traverse(contentIds) { contentId => - visibilityRuleEngine( - EvaluationContext(safetyLevel, params, NullStatsReceiver), - safetyLevel, - new VisibilityResultBuilder(contentId, featureMapProvider(contentId, safetyLevel)), - enableShortCircuiting - ).liftToTry - }, - safetyLevel.name - ) - } - - def runRuleEngineBatch( - contentIds: Seq[ContentId], - featureMapProvider: (ContentId, SafetyLevel) => FeatureMap, - evaluationContextBuilder: EvaluationContext.Builder, - safetyLevel: SafetyLevel - ): Stitch[Seq[Try[VisibilityResult]]] = { - val evaluationContext = evaluationContextBuilder.build(safetyLevel) - profileStitch( - Stitch.traverse(contentIds) { contentId => - visibilityRuleEngine( - evaluationContext, - safetyLevel, - new VisibilityResultBuilder(contentId, featureMapProvider(contentId, safetyLevel)), - enableShortCircuiting - ).liftToTry - }, - safetyLevel.name - ) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/builder/BUILD deleted file mode 100644 index 46f9c1147..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/src/java/com/twitter/logpipeline/client:logpipeline-event-publisher-thin", - "configapi/configapi-core", - "decider/src/main/scala", - "servo/util/src/main/scala", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "stitch/stitch-core", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - "visibility/common/src/main/scala/com/twitter/visibility/common/actions", - "visibility/common/src/main/scala/com/twitter/visibility/common/actions/converter/scala", - "visibility/common/src/main/scala/com/twitter/visibility/common/stitch", - "visibility/common/src/main/thrift/com/twitter/visibility:action-scala", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/configs", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/models", - "visibility/lib/src/main/scala/com/twitter/visibility/rules", - "visibility/lib/src/main/scala/com/twitter/visibility/util", - "visibility/lib/src/main/thrift/com/twitter/visibility/logging:vf-logging-scala", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/BUILD.docx new file mode 100644 index 000000000..d43061a9a Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/FeatureMapBuilder.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/FeatureMapBuilder.docx new file mode 100644 index 000000000..eb8d01b88 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/FeatureMapBuilder.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/FeatureMapBuilder.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/FeatureMapBuilder.scala deleted file mode 100644 index 946879c5e..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/FeatureMapBuilder.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.visibility.builder - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.Gate -import com.twitter.stitch.Stitch -import com.twitter.visibility.features._ -import com.twitter.visibility.common.stitch.StitchHelpers -import scala.collection.mutable - -object FeatureMapBuilder { - type Build = Seq[FeatureMapBuilder => FeatureMapBuilder] => FeatureMap - - def apply( - statsReceiver: StatsReceiver = NullStatsReceiver, - enableStitchProfiling: Gate[Unit] = Gate.False - ): Build = - fns => - Function - .chain(fns).apply( - new FeatureMapBuilder(statsReceiver, enableStitchProfiling) - ).build -} - -class FeatureMapBuilder private[builder] ( - statsReceiver: StatsReceiver, - enableStitchProfiling: Gate[Unit] = Gate.False) { - - private[this] val hydratedScope = - statsReceiver.scope("visibility_result_builder").scope("hydrated") - - val mapBuilder: mutable.Builder[(Feature[_], Stitch[_]), Map[Feature[_], Stitch[_]]] = - Map.newBuilder[Feature[_], Stitch[_]] - - val constantMapBuilder: mutable.Builder[(Feature[_], Any), Map[Feature[_], Any]] = - Map.newBuilder[Feature[_], Any] - - def build: FeatureMap = new FeatureMap(mapBuilder.result(), constantMapBuilder.result()) - - def withConstantFeature[T](feature: Feature[T], value: T): FeatureMapBuilder = { - val anyValue: Any = value.asInstanceOf[Any] - constantMapBuilder += (feature -> anyValue) - this - } - - def withFeature[T](feature: Feature[T], stitch: Stitch[T]): FeatureMapBuilder = { - val profiledStitch = if (enableStitchProfiling()) { - val featureScope = hydratedScope.scope(feature.name) - StitchHelpers.profileStitch(stitch, Seq(hydratedScope, featureScope)) - } else { - stitch - } - - val featureStitchRef = Stitch.ref(profiledStitch) - - mapBuilder += FeatureMap.rescueFeatureTuple(feature -> featureStitchRef) - - this - } - - def withConstantFeature[T](feature: Feature[T], option: Option[T]): FeatureMapBuilder = { - option.map(withConstantFeature(feature, _)).getOrElse(this) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/VerdictLogger.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/VerdictLogger.docx new file mode 100644 index 000000000..f1bf91279 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/VerdictLogger.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/VerdictLogger.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/VerdictLogger.scala deleted file mode 100644 index a47a35fa2..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/VerdictLogger.scala +++ /dev/null @@ -1,187 +0,0 @@ -package com.twitter.visibility.builder - -import com.twitter.datatools.entityservice.entities.thriftscala.FleetInterstitial -import com.twitter.decider.Decider -import com.twitter.decider.Decider.NullDecider -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.logpipeline.client.EventPublisherManager -import com.twitter.logpipeline.client.serializers.EventLogMsgThriftStructSerializer -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.visibility.builder.VerdictLogger.FailureCounterName -import com.twitter.visibility.builder.VerdictLogger.SuccessCounterName -import com.twitter.visibility.features.Feature -import com.twitter.visibility.logging.thriftscala.ActionSource -import com.twitter.visibility.logging.thriftscala.EntityId -import com.twitter.visibility.logging.thriftscala.EntityIdType -import com.twitter.visibility.logging.thriftscala.EntityIdValue -import com.twitter.visibility.logging.thriftscala.HealthActionType -import com.twitter.visibility.logging.thriftscala.MisinfoPolicyCategory -import com.twitter.visibility.logging.thriftscala.VFLibType -import com.twitter.visibility.logging.thriftscala.VFVerdictLogEntry -import com.twitter.visibility.models.ContentId -import com.twitter.visibility.rules._ - -object VerdictLogger { - - private val BaseStatsNamespace = "vf_verdict_logger" - private val FailureCounterName = "failures" - private val SuccessCounterName = "successes" - val LogCategoryName: String = "visibility_filtering_verdicts" - - val Empty: VerdictLogger = new VerdictLogger(NullStatsReceiver, NullDecider, None) - - def apply( - statsReceiver: StatsReceiver, - decider: Decider - ): VerdictLogger = { - val eventPublisher: EventPublisher[VFVerdictLogEntry] = - EventPublisherManager - .newScribePublisherBuilder( - LogCategoryName, - EventLogMsgThriftStructSerializer.getNewSerializer[VFVerdictLogEntry]()).build() - new VerdictLogger(statsReceiver.scope(BaseStatsNamespace), decider, Some(eventPublisher)) - } -} - -class VerdictLogger( - statsReceiver: StatsReceiver, - decider: Decider, - publisherOpt: Option[EventPublisher[VFVerdictLogEntry]]) { - - def log( - verdictLogEntry: VFVerdictLogEntry, - publisher: EventPublisher[VFVerdictLogEntry] - ): Unit = { - publisher - .publish(verdictLogEntry) - .onSuccess(_ => statsReceiver.counter(SuccessCounterName).incr()) - .onFailure { e => - statsReceiver.counter(FailureCounterName).incr() - statsReceiver.scope(FailureCounterName).counter(e.getClass.getName).incr() - } - } - - private def toEntityId(contentId: ContentId): Option[EntityId] = { - contentId match { - case ContentId.TweetId(id) => Some(EntityId(EntityIdType.TweetId, EntityIdValue.EntityId(id))) - case ContentId.UserId(id) => Some(EntityId(EntityIdType.UserId, EntityIdValue.EntityId(id))) - case ContentId.QuotedTweetRelationship(outerTweetId, _) => - Some(EntityId(EntityIdType.TweetId, EntityIdValue.EntityId(outerTweetId))) - case ContentId.NotificationId(Some(id)) => - Some(EntityId(EntityIdType.NotificationId, EntityIdValue.EntityId(id))) - case ContentId.DmId(id) => Some(EntityId(EntityIdType.DmId, EntityIdValue.EntityId(id))) - case ContentId.BlenderTweetId(id) => - Some(EntityId(EntityIdType.TweetId, EntityIdValue.EntityId(id))) - case ContentId.SpacePlusUserId(_) => - } - } - - private def getLogEntryData( - actingRule: Option[Rule], - secondaryActingRules: Seq[Rule], - verdict: Action, - secondaryVerdicts: Seq[Action], - resolvedFeatureMap: Map[Feature[_], Any] - ): (Seq[String], Seq[ActionSource], Seq[HealthActionType], Option[FleetInterstitial]) = { - actingRule - .filter { - case decideredRule: DoesLogVerdictDecidered => - decider.isAvailable(decideredRule.verdictLogDeciderKey.toString) - case rule: DoesLogVerdict => true - case _ => false - } - .map { primaryRule => - val secondaryRulesAndVerdicts = secondaryActingRules zip secondaryVerdicts - var actingRules: Seq[Rule] = Seq(primaryRule) - var actingRuleNames: Seq[String] = Seq(primaryRule.name) - var actionSources: Seq[ActionSource] = Seq() - var healthActionTypes: Seq[HealthActionType] = Seq(verdict.toHealthActionTypeThrift.get) - - val misinfoPolicyCategory: Option[FleetInterstitial] = { - verdict match { - case softIntervention: SoftIntervention => - softIntervention.fleetInterstitial - case tweetInterstitial: TweetInterstitial => - tweetInterstitial.softIntervention.flatMap(_.fleetInterstitial) - case _ => None - } - } - - secondaryRulesAndVerdicts.foreach(ruleAndVerdict => { - if (ruleAndVerdict._1.isInstanceOf[DoesLogVerdict]) { - actingRules = actingRules :+ ruleAndVerdict._1 - actingRuleNames = actingRuleNames :+ ruleAndVerdict._1.name - healthActionTypes = healthActionTypes :+ ruleAndVerdict._2.toHealthActionTypeThrift.get - } - }) - - actingRules.foreach(rule => { - rule.actionSourceBuilder - .flatMap(_.build(resolvedFeatureMap, verdict)) - .map(actionSource => { - actionSources = actionSources :+ actionSource - }) - }) - (actingRuleNames, actionSources, healthActionTypes, misinfoPolicyCategory) - } - .getOrElse((Seq.empty[String], Seq.empty[ActionSource], Seq.empty[HealthActionType], None)) - } - - def scribeVerdict( - visibilityResult: VisibilityResult, - safetyLevel: SafetyLevel, - vfLibType: VFLibType, - viewerId: Option[Long] = None - ): Unit = { - publisherOpt.foreach { publisher => - toEntityId(visibilityResult.contentId).foreach { entityId => - visibilityResult.verdict.toHealthActionTypeThrift.foreach { healthActionType => - val (actioningRules, actionSources, healthActionTypes, misinfoPolicyCategory) = - getLogEntryData( - actingRule = visibilityResult.actingRule, - secondaryActingRules = visibilityResult.secondaryActingRules, - verdict = visibilityResult.verdict, - secondaryVerdicts = visibilityResult.secondaryVerdicts, - resolvedFeatureMap = visibilityResult.resolvedFeatureMap - ) - - if (actioningRules.nonEmpty) { - log( - VFVerdictLogEntry( - entityId = entityId, - viewerId = viewerId, - timestampMsec = System.currentTimeMillis(), - vfLibType = vfLibType, - healthActionType = healthActionType, - safetyLevel = safetyLevel, - actioningRules = actioningRules, - actionSources = actionSources, - healthActionTypes = healthActionTypes, - misinfoPolicyCategory = - fleetInterstitialToMisinfoPolicyCategory(misinfoPolicyCategory) - ), - publisher - ) - } - } - } - } - } - - def fleetInterstitialToMisinfoPolicyCategory( - fleetInterstitialOption: Option[FleetInterstitial] - ): Option[MisinfoPolicyCategory] = { - fleetInterstitialOption.map { - case FleetInterstitial.Generic => - MisinfoPolicyCategory.Generic - case FleetInterstitial.Samm => - MisinfoPolicyCategory.Samm - case FleetInterstitial.CivicIntegrity => - MisinfoPolicyCategory.CivicIntegrity - case _ => MisinfoPolicyCategory.Unknown - } - } - -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResult.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResult.docx new file mode 100644 index 000000000..13ed7d62c Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResult.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResult.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResult.scala deleted file mode 100644 index bdf8764eb..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResult.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.visibility.builder - -import com.twitter.spam.rtf.thriftscala.SafetyResult -import com.twitter.visibility.common.actions.converter.scala.DropReasonConverter -import com.twitter.visibility.rules.ComposableActions._ -import com.twitter.visibility.features.Feature -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.models.ContentId -import com.twitter.visibility.rules._ -import com.twitter.visibility.{thriftscala => t} - -case class VisibilityResult( - contentId: ContentId, - featureMap: FeatureMap = FeatureMap.empty, - ruleResultMap: Map[Rule, RuleResult] = Map.empty, - verdict: Action = Allow, - finished: Boolean = false, - actingRule: Option[Rule] = None, - secondaryActingRules: Seq[Rule] = Seq(), - secondaryVerdicts: Seq[Action] = Seq(), - resolvedFeatureMap: Map[Feature[_], Any] = Map.empty) { - - def getSafetyResult: SafetyResult = - verdict match { - case InterstitialLimitedEngagements(reason: Reason, _, _, _) - if PublicInterest.Reasons - .contains(reason) => - SafetyResult( - Some(PublicInterest.ReasonToSafetyResultReason(reason)), - verdict.toActionThrift() - ) - case ComposableActionsWithInterstitialLimitedEngagements(tweetInterstitial) - if PublicInterest.Reasons.contains(tweetInterstitial.reason) => - SafetyResult( - Some(PublicInterest.ReasonToSafetyResultReason(tweetInterstitial.reason)), - verdict.toActionThrift() - ) - case FreedomOfSpeechNotReachReason(appealableReason) => - SafetyResult( - Some(FreedomOfSpeechNotReach.reasonToSafetyResultReason(appealableReason)), - verdict.toActionThrift() - ) - case _ => SafetyResult(None, verdict.toActionThrift()) - } - - def getUserVisibilityResult: Option[t.UserVisibilityResult] = - (verdict match { - case Drop(reason, _) => - Some( - t.UserAction.Drop(t.Drop(Reason.toDropReason(reason).map(DropReasonConverter.toThrift)))) - case _ => None - }).map(userAction => t.UserVisibilityResult(Some(userAction))) -} - -object VisibilityResult { - class Builder { - var featureMap: FeatureMap = FeatureMap.empty - var ruleResultMap: Map[Rule, RuleResult] = Map.empty - var verdict: Action = Allow - var finished: Boolean = false - var actingRule: Option[Rule] = None - var secondaryActingRules: Seq[Rule] = Seq() - var secondaryVerdicts: Seq[Action] = Seq() - var resolvedFeatureMap: Map[Feature[_], Any] = Map.empty - - def withFeatureMap(featureMapBld: FeatureMap) = { - featureMap = featureMapBld - this - } - - def withRuleResultMap(ruleResultMapBld: Map[Rule, RuleResult]) = { - ruleResultMap = ruleResultMapBld - this - } - - def withVerdict(verdictBld: Action) = { - verdict = verdictBld - this - } - - def withFinished(finishedBld: Boolean) = { - finished = finishedBld - this - } - - def withActingRule(actingRuleBld: Option[Rule]) = { - actingRule = actingRuleBld - this - } - - def withSecondaryActingRules(secondaryActingRulesBld: Seq[Rule]) = { - secondaryActingRules = secondaryActingRulesBld - this - } - - def withSecondaryVerdicts(secondaryVerdictsBld: Seq[Action]) = { - secondaryVerdicts = secondaryVerdictsBld - this - } - - def build(contentId: ContentId) = VisibilityResult( - contentId, - featureMap, - ruleResultMap, - verdict, - finished, - actingRule, - secondaryActingRules, - secondaryVerdicts, - resolvedFeatureMap) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResultBuilder.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResultBuilder.docx new file mode 100644 index 000000000..5ece443f7 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResultBuilder.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResultBuilder.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResultBuilder.scala deleted file mode 100644 index 83731eb88..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/VisibilityResultBuilder.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.visibility.builder - -import com.twitter.visibility.features.Feature -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.models.ContentId -import com.twitter.visibility.rules.Action -import com.twitter.visibility.rules.Allow -import com.twitter.visibility.rules.EvaluationContext -import com.twitter.visibility.rules.FailClosedException -import com.twitter.visibility.rules.FeaturesFailedException -import com.twitter.visibility.rules.MissingFeaturesException -import com.twitter.visibility.rules.Rule -import com.twitter.visibility.rules.RuleFailedException -import com.twitter.visibility.rules.RuleResult -import com.twitter.visibility.rules.State.FeatureFailed -import com.twitter.visibility.rules.State.MissingFeature -import com.twitter.visibility.rules.State.RuleFailed - -class VisibilityResultBuilder( - val contentId: ContentId, - val featureMap: FeatureMap = FeatureMap.empty, - private var ruleResultMap: Map[Rule, RuleResult] = Map.empty) { - private var mapBuilder = Map.newBuilder[Rule, RuleResult] - mapBuilder ++= ruleResultMap - var verdict: Action = Allow - var finished: Boolean = false - var features: FeatureMap = featureMap - var actingRule: Option[Rule] = None - var secondaryVerdicts: Seq[Action] = Seq() - var secondaryActingRules: Seq[Rule] = Seq() - var resolvedFeatureMap: Map[Feature[_], Any] = Map.empty - - def ruleResults: Map[Rule, RuleResult] = mapBuilder.result() - - def withFeatureMap(featureMap: FeatureMap): VisibilityResultBuilder = { - this.features = featureMap - this - } - - def withRuleResultMap(ruleResultMap: Map[Rule, RuleResult]): VisibilityResultBuilder = { - this.ruleResultMap = ruleResultMap - mapBuilder = Map.newBuilder[Rule, RuleResult] - mapBuilder ++= ruleResultMap - this - } - - def withRuleResult(rule: Rule, result: RuleResult): VisibilityResultBuilder = { - mapBuilder += ((rule, result)) - this - } - - def withVerdict(verdict: Action, ruleOpt: Option[Rule] = None): VisibilityResultBuilder = { - this.verdict = verdict - this.actingRule = ruleOpt - this - } - - def withSecondaryVerdict(verdict: Action, rule: Rule): VisibilityResultBuilder = { - this.secondaryVerdicts = this.secondaryVerdicts :+ verdict - this.secondaryActingRules = this.secondaryActingRules :+ rule - this - } - - def withFinished(finished: Boolean): VisibilityResultBuilder = { - this.finished = finished - this - } - - def withResolvedFeatureMap( - resolvedFeatureMap: Map[Feature[_], Any] - ): VisibilityResultBuilder = { - this.resolvedFeatureMap = resolvedFeatureMap - this - } - - def isVerdictComposable(): Boolean = this.verdict.isComposable - - def failClosedException(evaluationContext: EvaluationContext): Option[FailClosedException] = { - mapBuilder - .result().collect { - case (r: Rule, RuleResult(_, MissingFeature(mf))) - if r.shouldFailClosed(evaluationContext.params) => - Some(MissingFeaturesException(r.name, mf)) - case (r: Rule, RuleResult(_, FeatureFailed(ff))) - if r.shouldFailClosed(evaluationContext.params) => - Some(FeaturesFailedException(r.name, ff)) - case (r: Rule, RuleResult(_, RuleFailed(t))) - if r.shouldFailClosed(evaluationContext.params) => - Some(RuleFailedException(r.name, t)) - }.toList.foldLeft(None: Option[FailClosedException]) { (acc, arg) => - (acc, arg) match { - case (None, Some(_)) => arg - case (Some(FeaturesFailedException(_, _)), Some(MissingFeaturesException(_, _))) => arg - case (Some(RuleFailedException(_, _)), Some(MissingFeaturesException(_, _))) => arg - case (Some(RuleFailedException(_, _)), Some(FeaturesFailedException(_, _))) => arg - case _ => acc - } - } - } - - def build: VisibilityResult = { - VisibilityResult( - contentId = contentId, - featureMap = features, - ruleResultMap = mapBuilder.result(), - verdict = verdict, - finished = finished, - actingRule = actingRule, - secondaryActingRules = secondaryActingRules, - secondaryVerdicts = secondaryVerdicts, - resolvedFeatureMap = resolvedFeatureMap - ) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/BUILD deleted file mode 100644 index 5d44ccfd1..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "communities/thrift/src/main/thrift/com/twitter/communities:thrift-scala", - "communities/thrift/src/main/thrift/com/twitter/communities/moderation:thrift-scala", - "escherbird/src/thrift/com/twitter/escherbird/softintervention:softintervention_thrift-scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/thrift/com/twitter/context:twitter-context-scala", - "src/thrift/com/twitter/escherbird/common:common-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "tweetypie/src/scala/com/twitter/tweetypie/additionalfields", - "twitter-context/src/main/scala", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/common/src/main/scala/com/twitter/visibility/common/stitch", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/blender", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/tweets", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/BUILD.docx new file mode 100644 index 000000000..9fe78ab77 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/MutedKeywordFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/MutedKeywordFeatures.docx new file mode 100644 index 000000000..e8c3ff726 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/MutedKeywordFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/MutedKeywordFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/MutedKeywordFeatures.scala deleted file mode 100644 index eb0a21663..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/common/MutedKeywordFeatures.scala +++ /dev/null @@ -1,228 +0,0 @@ -package com.twitter.visibility.builder.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.MuteOption -import com.twitter.gizmoduck.thriftscala.MuteSurface -import com.twitter.gizmoduck.thriftscala.{MutedKeyword => GdMutedKeyword} -import com.twitter.servo.util.Gate -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common._ -import com.twitter.visibility.features._ -import com.twitter.visibility.models.{MutedKeyword => VfMutedKeyword} -import java.util.Locale - -class MutedKeywordFeatures( - userSource: UserSource, - userRelationshipSource: UserRelationshipSource, - keywordMatcher: KeywordMatcher.Matcher = KeywordMatcher.TestMatcher, - statsReceiver: StatsReceiver, - enableFollowCheckInMutedKeyword: Gate[Unit] = Gate.False) { - - private[this] val scopedStatsReceiver: StatsReceiver = - statsReceiver.scope("muted_keyword_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val viewerMutesKeywordInTweetForHomeTimeline = - scopedStatsReceiver.scope(ViewerMutesKeywordInTweetForHomeTimeline.name).counter("requests") - private[this] val viewerMutesKeywordInTweetForTweetReplies = - scopedStatsReceiver.scope(ViewerMutesKeywordInTweetForTweetReplies.name).counter("requests") - private[this] val viewerMutesKeywordInTweetForNotifications = - scopedStatsReceiver.scope(ViewerMutesKeywordInTweetForNotifications.name).counter("requests") - private[this] val excludeFollowingForMutedKeywordsRequests = - scopedStatsReceiver.scope("exclude_following").counter("requests") - private[this] val viewerMutesKeywordInTweetForAllSurfaces = - scopedStatsReceiver.scope(ViewerMutesKeywordInTweetForAllSurfaces.name).counter("requests") - - def forTweet( - tweet: Tweet, - viewerId: Option[Long], - authorId: Long - ): FeatureMapBuilder => FeatureMapBuilder = { featureMapBuilder => - requests.incr() - viewerMutesKeywordInTweetForHomeTimeline.incr() - viewerMutesKeywordInTweetForTweetReplies.incr() - viewerMutesKeywordInTweetForNotifications.incr() - viewerMutesKeywordInTweetForAllSurfaces.incr() - - val keywordsBySurface = allMutedKeywords(viewerId) - - val keywordsWithoutDefinedSurface = allMutedKeywordsWithoutDefinedSurface(viewerId) - - featureMapBuilder - .withFeature( - ViewerMutesKeywordInTweetForHomeTimeline, - tweetContainsMutedKeyword( - tweet, - keywordsBySurface, - MuteSurface.HomeTimeline, - viewerId, - authorId - ) - ) - .withFeature( - ViewerMutesKeywordInTweetForTweetReplies, - tweetContainsMutedKeyword( - tweet, - keywordsBySurface, - MuteSurface.TweetReplies, - viewerId, - authorId - ) - ) - .withFeature( - ViewerMutesKeywordInTweetForNotifications, - tweetContainsMutedKeyword( - tweet, - keywordsBySurface, - MuteSurface.Notifications, - viewerId, - authorId - ) - ) - .withFeature( - ViewerMutesKeywordInTweetForAllSurfaces, - tweetContainsMutedKeywordWithoutDefinedSurface( - tweet, - keywordsWithoutDefinedSurface, - viewerId, - authorId - ) - ) - } - - def allMutedKeywords(viewerId: Option[Long]): Stitch[Map[MuteSurface, Seq[GdMutedKeyword]]] = - viewerId - .map { id => userSource.getAllMutedKeywords(id) }.getOrElse(Stitch.value(Map.empty)) - - def allMutedKeywordsWithoutDefinedSurface(viewerId: Option[Long]): Stitch[Seq[GdMutedKeyword]] = - viewerId - .map { id => userSource.getAllMutedKeywordsWithoutDefinedSurface(id) }.getOrElse( - Stitch.value(Seq.empty)) - - private def mutingKeywordsText( - mutedKeywords: Seq[GdMutedKeyword], - muteSurface: MuteSurface, - viewerIdOpt: Option[Long], - authorId: Long - ): Stitch[Option[String]] = { - if (muteSurface == MuteSurface.HomeTimeline && mutedKeywords.nonEmpty) { - Stitch.value(Some(mutedKeywords.map(_.keyword).mkString(","))) - } else { - mutedKeywords.partition(kw => - kw.muteOptions.contains(MuteOption.ExcludeFollowingAccounts)) match { - case (_, mutedKeywordsFromAnyone) if mutedKeywordsFromAnyone.nonEmpty => - Stitch.value(Some(mutedKeywordsFromAnyone.map(_.keyword).mkString(","))) - case (mutedKeywordsExcludeFollowing, _) - if mutedKeywordsExcludeFollowing.nonEmpty && enableFollowCheckInMutedKeyword() => - excludeFollowingForMutedKeywordsRequests.incr() - viewerIdOpt match { - case Some(viewerId) => - userRelationshipSource.follows(viewerId, authorId).map { - case true => - case false => Some(mutedKeywordsExcludeFollowing.map(_.keyword).mkString(",")) - } - case _ => Stitch.None - } - case (_, _) => Stitch.None - } - } - } - - private def mutingKeywordsTextWithoutDefinedSurface( - mutedKeywords: Seq[GdMutedKeyword], - viewerIdOpt: Option[Long], - authorId: Long - ): Stitch[Option[String]] = { - mutedKeywords.partition(kw => - kw.muteOptions.contains(MuteOption.ExcludeFollowingAccounts)) match { - case (_, mutedKeywordsFromAnyone) if mutedKeywordsFromAnyone.nonEmpty => - Stitch.value(Some(mutedKeywordsFromAnyone.map(_.keyword).mkString(","))) - case (mutedKeywordsExcludeFollowing, _) - if mutedKeywordsExcludeFollowing.nonEmpty && enableFollowCheckInMutedKeyword() => - excludeFollowingForMutedKeywordsRequests.incr() - viewerIdOpt match { - case Some(viewerId) => - userRelationshipSource.follows(viewerId, authorId).map { - case true => - case false => Some(mutedKeywordsExcludeFollowing.map(_.keyword).mkString(",")) - } - case _ => Stitch.None - } - case (_, _) => Stitch.None - } - } - - def tweetContainsMutedKeyword( - tweet: Tweet, - mutedKeywordMap: Stitch[Map[MuteSurface, Seq[GdMutedKeyword]]], - muteSurface: MuteSurface, - viewerIdOpt: Option[Long], - authorId: Long - ): Stitch[VfMutedKeyword] = { - mutedKeywordMap.flatMap { keywordMap => - if (keywordMap.isEmpty) { - Stitch.value(VfMutedKeyword(None)) - } else { - val mutedKeywords = keywordMap.getOrElse(muteSurface, Nil) - val matchTweetFn: KeywordMatcher.MatchTweet = keywordMatcher(mutedKeywords) - val locale = tweet.language.map(l => Locale.forLanguageTag(l.language)) - val text = tweet.coreData.get.text - - matchTweetFn(locale, text).flatMap { results => - mutingKeywordsText(results, muteSurface, viewerIdOpt, authorId).map(VfMutedKeyword) - } - } - } - } - - def tweetContainsMutedKeywordWithoutDefinedSurface( - tweet: Tweet, - mutedKeywordSeq: Stitch[Seq[GdMutedKeyword]], - viewerIdOpt: Option[Long], - authorId: Long - ): Stitch[VfMutedKeyword] = { - mutedKeywordSeq.flatMap { mutedKeyword => - if (mutedKeyword.isEmpty) { - Stitch.value(VfMutedKeyword(None)) - } else { - val matchTweetFn: KeywordMatcher.MatchTweet = keywordMatcher(mutedKeyword) - val locale = tweet.language.map(l => Locale.forLanguageTag(l.language)) - val text = tweet.coreData.get.text - - matchTweetFn(locale, text).flatMap { results => - mutingKeywordsTextWithoutDefinedSurface(results, viewerIdOpt, authorId).map( - VfMutedKeyword - ) - } - } - } - } - def spaceTitleContainsMutedKeyword( - spaceTitle: String, - spaceLanguageOpt: Option[String], - mutedKeywordMap: Stitch[Map[MuteSurface, Seq[GdMutedKeyword]]], - muteSurface: MuteSurface, - ): Stitch[VfMutedKeyword] = { - mutedKeywordMap.flatMap { keywordMap => - if (keywordMap.isEmpty) { - Stitch.value(VfMutedKeyword(None)) - } else { - val mutedKeywords = keywordMap.getOrElse(muteSurface, Nil) - val matchTweetFn: KeywordMatcher.MatchTweet = keywordMatcher(mutedKeywords) - - val locale = spaceLanguageOpt.map(l => Locale.forLanguageTag(l)) - matchTweetFn(locale, spaceTitle).flatMap { results => - if (results.nonEmpty) { - Stitch.value(Some(results.map(_.keyword).mkString(","))).map(VfMutedKeyword) - } else { - Stitch.None.map(VfMutedKeyword) - } - } - } - } - } - -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/BUILD deleted file mode 100644 index 3c769bb73..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/thrift/com/twitter/convosvc:convosvc-scala", - "src/thrift/com/twitter/convosvc/internal:internal-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "stitch/stitch-core", - "stitch/stitch-core/src/main/scala/com/twitter/stitch", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/common/src/main/scala/com/twitter/visibility/common/dm_sources", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/util", - "visibility/lib/src/main/thrift/com/twitter/visibility/safety_label_store:safety-label-store-scala", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/BUILD.docx new file mode 100644 index 000000000..61aed49ba Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmConversationFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmConversationFeatures.docx new file mode 100644 index 000000000..fc4843625 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmConversationFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmConversationFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmConversationFeatures.scala deleted file mode 100644 index ad21e40ad..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmConversationFeatures.scala +++ /dev/null @@ -1,196 +0,0 @@ -package com.twitter.visibility.builder.dms - -import com.twitter.convosvc.thriftscala.ConversationQuery -import com.twitter.convosvc.thriftscala.ConversationQueryOptions -import com.twitter.convosvc.thriftscala.ConversationType -import com.twitter.convosvc.thriftscala.TimelineLookupState -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.common.DmConversationId -import com.twitter.visibility.common.UserId -import com.twitter.visibility.common.dm_sources.DmConversationSource -import com.twitter.visibility.features._ - -case class InvalidDmConversationFeatureException(message: String) extends Exception(message) - -class DmConversationFeatures( - dmConversationSource: DmConversationSource, - authorFeatures: AuthorFeatures) { - - def forDmConversationId( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): FeatureMapBuilder => FeatureMapBuilder = - _.withFeature( - DmConversationIsOneToOneConversation, - dmConversationIsOneToOneConversation(dmConversationId, viewerIdOpt)) - .withFeature( - DmConversationHasEmptyTimeline, - dmConversationHasEmptyTimeline(dmConversationId, viewerIdOpt)) - .withFeature( - DmConversationHasValidLastReadableEventId, - dmConversationHasValidLastReadableEventId(dmConversationId, viewerIdOpt)) - .withFeature( - DmConversationInfoExists, - dmConversationInfoExists(dmConversationId, viewerIdOpt)) - .withFeature( - DmConversationTimelineExists, - dmConversationTimelineExists(dmConversationId, viewerIdOpt)) - .withFeature( - AuthorIsSuspended, - dmConversationHasSuspendedParticipant(dmConversationId, viewerIdOpt)) - .withFeature( - AuthorIsDeactivated, - dmConversationHasDeactivatedParticipant(dmConversationId, viewerIdOpt)) - .withFeature( - AuthorIsErased, - dmConversationHasErasedParticipant(dmConversationId, viewerIdOpt)) - .withFeature( - ViewerIsDmConversationParticipant, - viewerIsDmConversationParticipant(dmConversationId, viewerIdOpt)) - - def dmConversationIsOneToOneConversation( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - viewerIdOpt match { - case Some(viewerId) => - dmConversationSource.getConversationType(dmConversationId, viewerId).flatMap { - case Some(ConversationType.OneToOneDm | ConversationType.SecretOneToOneDm) => - Stitch.True - case None => - Stitch.exception(InvalidDmConversationFeatureException("Conversation type not found")) - case _ => Stitch.False - } - case _ => Stitch.exception(InvalidDmConversationFeatureException("Viewer id missing")) - } - - private[dms] def dmConversationHasEmptyTimeline( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - dmConversationSource - .getConversationTimelineEntries( - dmConversationId, - ConversationQuery( - conversationId = Some(dmConversationId), - options = Some( - ConversationQueryOptions( - perspectivalUserId = viewerIdOpt, - hydrateEvents = Some(false), - supportsReactions = Some(true) - ) - ), - maxCount = 10 - ) - ).map(_.forall(entries => entries.isEmpty)) - - private[dms] def dmConversationHasValidLastReadableEventId( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - viewerIdOpt match { - case Some(viewerId) => - dmConversationSource - .getConversationLastReadableEventId(dmConversationId, viewerId).map(_.exists(id => - id > 0L)) - case _ => Stitch.exception(InvalidDmConversationFeatureException("Viewer id missing")) - } - - private[dms] def dmConversationInfoExists( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - viewerIdOpt match { - case Some(viewerId) => - dmConversationSource - .getDmConversationInfo(dmConversationId, viewerId).map(_.isDefined) - case _ => Stitch.exception(InvalidDmConversationFeatureException("Viewer id missing")) - } - - private[dms] def dmConversationTimelineExists( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - dmConversationSource - .getConversationTimelineState( - dmConversationId, - ConversationQuery( - conversationId = Some(dmConversationId), - options = Some( - ConversationQueryOptions( - perspectivalUserId = viewerIdOpt, - hydrateEvents = Some(false), - supportsReactions = Some(true) - ) - ), - maxCount = 1 - ) - ).map { - case Some(TimelineLookupState.NotFound) | None => false - case _ => true - } - - private[dms] def anyConversationParticipantMatchesCondition( - condition: UserId => Stitch[Boolean], - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - viewerIdOpt match { - case Some(viewerId) => - dmConversationSource - .getConversationParticipantIds(dmConversationId, viewerId).flatMap { - case Some(participants) => - Stitch - .collect(participants.map(condition)).map(_.contains(true)).rescue { - case NotFound => - Stitch.exception(InvalidDmConversationFeatureException("User not found")) - } - case _ => Stitch.False - } - case _ => Stitch.exception(InvalidDmConversationFeatureException("Viewer id missing")) - } - - def dmConversationHasSuspendedParticipant( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - anyConversationParticipantMatchesCondition( - participant => authorFeatures.authorIsSuspended(participant), - dmConversationId, - viewerIdOpt) - - def dmConversationHasDeactivatedParticipant( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - anyConversationParticipantMatchesCondition( - participant => authorFeatures.authorIsDeactivated(participant), - dmConversationId, - viewerIdOpt) - - def dmConversationHasErasedParticipant( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - anyConversationParticipantMatchesCondition( - participant => authorFeatures.authorIsErased(participant), - dmConversationId, - viewerIdOpt) - - def viewerIsDmConversationParticipant( - dmConversationId: DmConversationId, - viewerIdOpt: Option[UserId] - ): Stitch[Boolean] = - viewerIdOpt match { - case Some(viewerId) => - dmConversationSource - .getConversationParticipantIds(dmConversationId, viewerId).map { - case Some(participants) => participants.contains(viewerId) - case _ => false - } - case _ => Stitch.exception(InvalidDmConversationFeatureException("Viewer id missing")) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmEventFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmEventFeatures.docx new file mode 100644 index 000000000..0241fa878 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmEventFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmEventFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmEventFeatures.scala deleted file mode 100644 index 797a52268..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/dms/DmEventFeatures.scala +++ /dev/null @@ -1,341 +0,0 @@ -package com.twitter.visibility.builder.dms - -import com.twitter.convosvc.thriftscala.Event -import com.twitter.convosvc.thriftscala.StoredDelete -import com.twitter.convosvc.thriftscala.StoredPerspectivalMessageInfo -import com.twitter.convosvc.thriftscala.PerspectivalSpamState -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.common.DmEventId -import com.twitter.visibility.common.dm_sources.DmEventSource -import com.twitter.visibility.common.UserId -import com.twitter.convosvc.thriftscala.EventType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.NotFound -import com.twitter.visibility.common.dm_sources.DmConversationSource -import com.twitter.visibility.features._ - -case class InvalidDmEventFeatureException(message: String) extends Exception(message) - -class DmEventFeatures( - dmEventSource: DmEventSource, - dmConversationSource: DmConversationSource, - authorFeatures: AuthorFeatures, - dmConversationFeatures: DmConversationFeatures, - statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("dm_event_features") - private[this] val requests = scopedStatsReceiver.counter("requests") - - def forDmEventId( - dmEventId: DmEventId, - viewerId: UserId - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - val dmEventStitchRef: Stitch[Option[Event]] = - Stitch.ref(dmEventSource.getDmEvent(dmEventId, viewerId)) - - _.withFeature( - DmEventIsMessageCreateEvent, - isDmEventType(dmEventStitchRef, EventType.MessageCreate)) - .withFeature( - AuthorIsSuspended, - messageCreateEventHasInactiveInitiatingUser( - dmEventStitchRef, - initiatingUser => authorFeatures.authorIsSuspended(initiatingUser)) - ) - .withFeature( - AuthorIsDeactivated, - messageCreateEventHasInactiveInitiatingUser( - dmEventStitchRef, - initiatingUser => authorFeatures.authorIsDeactivated(initiatingUser)) - ) - .withFeature( - AuthorIsErased, - messageCreateEventHasInactiveInitiatingUser( - dmEventStitchRef, - initiatingUser => authorFeatures.authorIsErased(initiatingUser)) - ) - .withFeature( - DmEventOccurredBeforeLastClearedEvent, - dmEventOccurredBeforeLastClearedEvent(dmEventStitchRef, dmEventId, viewerId) - ) - .withFeature( - DmEventOccurredBeforeJoinConversationEvent, - dmEventOccurredBeforeJoinConversationEvent(dmEventStitchRef, dmEventId, viewerId) - ) - .withFeature( - ViewerIsDmConversationParticipant, - dmEventViewerIsDmConversationParticipant(dmEventStitchRef, viewerId) - ) - .withFeature( - DmEventIsDeleted, - dmEventIsDeleted(dmEventStitchRef, dmEventId) - ) - .withFeature( - DmEventIsHidden, - dmEventIsHidden(dmEventStitchRef, dmEventId) - ) - .withFeature( - ViewerIsDmEventInitiatingUser, - viewerIsDmEventInitiatingUser(dmEventStitchRef, viewerId) - ) - .withFeature( - DmEventInOneToOneConversationWithUnavailableUser, - dmEventInOneToOneConversationWithUnavailableUser(dmEventStitchRef, viewerId) - ) - .withFeature( - DmEventIsLastMessageReadUpdateEvent, - isDmEventType(dmEventStitchRef, EventType.LastMessageReadUpdate) - ) - .withFeature( - DmEventIsJoinConversationEvent, - isDmEventType(dmEventStitchRef, EventType.JoinConversation) - ) - .withFeature( - DmEventIsWelcomeMessageCreateEvent, - isDmEventType(dmEventStitchRef, EventType.WelcomeMessageCreate) - ) - .withFeature( - DmEventIsTrustConversationEvent, - isDmEventType(dmEventStitchRef, EventType.TrustConversation) - ) - .withFeature( - DmEventIsCsFeedbackSubmitted, - isDmEventType(dmEventStitchRef, EventType.CsFeedbackSubmitted) - ) - .withFeature( - DmEventIsCsFeedbackDismissed, - isDmEventType(dmEventStitchRef, EventType.CsFeedbackDismissed) - ) - .withFeature( - DmEventIsConversationCreateEvent, - isDmEventType(dmEventStitchRef, EventType.ConversationCreate) - ) - .withFeature( - DmEventInOneToOneConversation, - dmEventInOneToOneConversation(dmEventStitchRef, viewerId) - ) - .withFeature( - DmEventIsPerspectivalJoinConversationEvent, - dmEventIsPerspectivalJoinConversationEvent(dmEventStitchRef, dmEventId, viewerId)) - - } - - private def isDmEventType( - dmEventOptStitch: Stitch[Option[Event]], - eventType: EventType - ): Stitch[Boolean] = - dmEventSource.getEventType(dmEventOptStitch).flatMap { - case Some(_: eventType.type) => - Stitch.True - case None => - Stitch.exception(InvalidDmEventFeatureException(s"$eventType event type not found")) - case _ => - Stitch.False - } - - private def dmEventIsPerspectivalJoinConversationEvent( - dmEventOptStitch: Stitch[Option[Event]], - dmEventId: DmEventId, - viewerId: UserId - ): Stitch[Boolean] = - Stitch - .join( - dmEventSource.getEventType(dmEventOptStitch), - dmEventSource.getConversationId(dmEventOptStitch)).flatMap { - case (Some(EventType.JoinConversation), conversationIdOpt) => - conversationIdOpt match { - case Some(conversationId) => - dmConversationSource - .getParticipantJoinConversationEventId(conversationId, viewerId, viewerId) - .flatMap { - case Some(joinConversationEventId) => - Stitch.value(joinConversationEventId == dmEventId) - case _ => Stitch.False - } - case _ => - Stitch.exception(InvalidDmEventFeatureException("Conversation id not found")) - } - case (None, _) => - Stitch.exception(InvalidDmEventFeatureException("Event type not found")) - case _ => Stitch.False - } - - private def messageCreateEventHasInactiveInitiatingUser( - dmEventOptStitch: Stitch[Option[Event]], - condition: UserId => Stitch[Boolean], - ): Stitch[Boolean] = - Stitch - .join( - dmEventSource.getEventType(dmEventOptStitch), - dmEventSource.getInitiatingUserId(dmEventOptStitch)).flatMap { - case (Some(EventType.MessageCreate), Some(userId)) => - condition(userId).rescue { - case NotFound => - Stitch.exception(InvalidDmEventFeatureException("initiating user not found")) - } - case (None, _) => - Stitch.exception(InvalidDmEventFeatureException("DmEvent type is missing")) - case (Some(EventType.MessageCreate), _) => - Stitch.exception(InvalidDmEventFeatureException("initiating user id is missing")) - case _ => Stitch.False - } - - private def dmEventOccurredBeforeLastClearedEvent( - dmEventOptStitch: Stitch[Option[Event]], - dmEventId: DmEventId, - viewerId: UserId - ): Stitch[Boolean] = { - dmEventSource.getConversationId(dmEventOptStitch).flatMap { - case Some(convoId) => - val lastClearedEventIdStitch = - dmConversationSource.getParticipantLastClearedEventId(convoId, viewerId, viewerId) - lastClearedEventIdStitch.flatMap { - case Some(lastClearedEventId) => Stitch(dmEventId <= lastClearedEventId) - case _ => - Stitch.False - } - case _ => Stitch.False - } - } - - private def dmEventOccurredBeforeJoinConversationEvent( - dmEventOptStitch: Stitch[Option[Event]], - dmEventId: DmEventId, - viewerId: UserId - ): Stitch[Boolean] = { - dmEventSource.getConversationId(dmEventOptStitch).flatMap { - case Some(convoId) => - val joinConversationEventIdStitch = - dmConversationSource - .getParticipantJoinConversationEventId(convoId, viewerId, viewerId) - joinConversationEventIdStitch.flatMap { - case Some(joinConversationEventId) => Stitch(dmEventId < joinConversationEventId) - case _ => Stitch.False - } - case _ => Stitch.False - } - } - - private def dmEventViewerIsDmConversationParticipant( - dmEventOptStitch: Stitch[Option[Event]], - viewerId: UserId - ): Stitch[Boolean] = { - dmEventSource.getConversationId(dmEventOptStitch).flatMap { - case Some(convoId) => - dmConversationFeatures.viewerIsDmConversationParticipant(convoId, Some(viewerId)) - case _ => Stitch.True - } - } - - private def dmEventIsDeleted( - dmEventOptStitch: Stitch[Option[Event]], - dmEventId: DmEventId - ): Stitch[Boolean] = - dmEventSource.getConversationId(dmEventOptStitch).flatMap { - case Some(convoId) => - dmConversationSource - .getDeleteInfo(convoId, dmEventId).rescue { - case e: java.lang.IllegalArgumentException => - Stitch.exception(InvalidDmEventFeatureException("Invalid conversation id")) - }.flatMap { - case Some(StoredDelete(None)) => Stitch.True - case _ => Stitch.False - } - case _ => Stitch.False - } - - private def dmEventIsHidden( - dmEventOptStitch: Stitch[Option[Event]], - dmEventId: DmEventId - ): Stitch[Boolean] = - dmEventSource.getConversationId(dmEventOptStitch).flatMap { - case Some(convoId) => - dmConversationSource - .getPerspectivalMessageInfo(convoId, dmEventId).rescue { - case e: java.lang.IllegalArgumentException => - Stitch.exception(InvalidDmEventFeatureException("Invalid conversation id")) - }.flatMap { - case Some(StoredPerspectivalMessageInfo(Some(hidden), _)) if hidden => - Stitch.True - case Some(StoredPerspectivalMessageInfo(_, Some(spamState))) - if spamState == PerspectivalSpamState.Spam => - Stitch.True - case _ => Stitch.False - } - case _ => Stitch.False - } - - private def viewerIsDmEventInitiatingUser( - dmEventOptStitch: Stitch[Option[Event]], - viewerId: UserId - ): Stitch[Boolean] = - Stitch - .join( - dmEventSource.getEventType(dmEventOptStitch), - dmEventSource.getInitiatingUserId(dmEventOptStitch)).flatMap { - case ( - Some( - EventType.TrustConversation | EventType.CsFeedbackSubmitted | - EventType.CsFeedbackDismissed | EventType.WelcomeMessageCreate | - EventType.JoinConversation), - Some(userId)) => - Stitch(viewerId == userId) - case ( - Some( - EventType.TrustConversation | EventType.CsFeedbackSubmitted | - EventType.CsFeedbackDismissed | EventType.WelcomeMessageCreate | - EventType.JoinConversation), - None) => - Stitch.exception(InvalidDmEventFeatureException("Initiating user id is missing")) - case (None, _) => - Stitch.exception(InvalidDmEventFeatureException("DmEvent type is missing")) - case _ => Stitch.True - } - - private def dmEventInOneToOneConversationWithUnavailableUser( - dmEventOptStitch: Stitch[Option[Event]], - viewerId: UserId - ): Stitch[Boolean] = - dmEventSource.getConversationId(dmEventOptStitch).flatMap { - case Some(conversationId) => - dmConversationFeatures - .dmConversationIsOneToOneConversation(conversationId, Some(viewerId)).flatMap { - isOneToOne => - if (isOneToOne) { - Stitch - .join( - dmConversationFeatures - .dmConversationHasSuspendedParticipant(conversationId, Some(viewerId)), - dmConversationFeatures - .dmConversationHasDeactivatedParticipant(conversationId, Some(viewerId)), - dmConversationFeatures - .dmConversationHasErasedParticipant(conversationId, Some(viewerId)) - ).flatMap { - case ( - convoParticipantIsSuspended, - convoParticipantIsDeactivated, - convoParticipantIsErased) => - Stitch.value( - convoParticipantIsSuspended || convoParticipantIsDeactivated || convoParticipantIsErased) - } - } else { - Stitch.False - } - } - case _ => Stitch.False - } - - private def dmEventInOneToOneConversation( - dmEventOptStitch: Stitch[Option[Event]], - viewerId: UserId - ): Stitch[Boolean] = - dmEventSource.getConversationId(dmEventOptStitch).flatMap { - case Some(conversationId) => - dmConversationFeatures - .dmConversationIsOneToOneConversation(conversationId, Some(viewerId)) - case _ => Stitch.False - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/BUILD deleted file mode 100644 index d74c9c5d0..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/BUILD +++ /dev/null @@ -1,31 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "mediaservices/commons/src/main/thrift:thrift-scala", - "mediaservices/media-util/src/main/scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/thrift/com/twitter/context:twitter-context-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "tweetypie/src/scala/com/twitter/tweetypie/additionalfields", - "twitter-context/src/main/scala", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/tweets", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/tweets", - "visibility/lib/src/main/scala/com/twitter/visibility/util", - "visibility/lib/src/main/thrift/com/twitter/visibility/safety_label_store:safety-label-store-scala", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/BUILD.docx new file mode 100644 index 000000000..b497d7847 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaFeatures.docx new file mode 100644 index 000000000..f16741d09 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaFeatures.scala deleted file mode 100644 index e20429846..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaFeatures.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.visibility.builder.media - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.mediaservices.media_util.GenericMediaKey -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.MediaSafetyLabelMapSource -import com.twitter.visibility.features.MediaSafetyLabels -import com.twitter.visibility.models.MediaSafetyLabel -import com.twitter.visibility.models.MediaSafetyLabelType -import com.twitter.visibility.models.SafetyLabel - -class MediaFeatures( - mediaSafetyLabelMap: StratoMediaLabelMaps, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("media_features") - - private[this] val requests = - scopedStatsReceiver - .counter("requests") - - private[this] val mediaSafetyLabelsStats = - scopedStatsReceiver - .scope(MediaSafetyLabels.name) - .counter("requests") - - private[this] val nonEmptyMediaStats = scopedStatsReceiver.scope("non_empty_media") - private[this] val nonEmptyMediaRequests = nonEmptyMediaStats.counter("requests") - private[this] val nonEmptyMediaKeysCount = nonEmptyMediaStats.counter("keys") - private[this] val nonEmptyMediaKeysLength = nonEmptyMediaStats.stat("keys_length") - - def forMediaKeys( - mediaKeys: Seq[GenericMediaKey], - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - nonEmptyMediaKeysCount.incr(mediaKeys.size) - mediaSafetyLabelsStats.incr() - - if (mediaKeys.nonEmpty) { - nonEmptyMediaRequests.incr() - nonEmptyMediaKeysLength.add(mediaKeys.size) - } - - _.withFeature(MediaSafetyLabels, mediaSafetyLabelMap.forGenericMediaKeys(mediaKeys)) - } - - def forGenericMediaKey( - genericMediaKey: GenericMediaKey - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - nonEmptyMediaKeysCount.incr() - mediaSafetyLabelsStats.incr() - nonEmptyMediaRequests.incr() - nonEmptyMediaKeysLength.add(1L) - - _.withFeature(MediaSafetyLabels, mediaSafetyLabelMap.forGenericMediaKey(genericMediaKey)) - } -} - -class StratoMediaLabelMaps(source: MediaSafetyLabelMapSource) { - - def forGenericMediaKeys( - mediaKeys: Seq[GenericMediaKey], - ): Stitch[Seq[MediaSafetyLabel]] = { - Stitch - .collect( - mediaKeys - .map(getFilteredSafetyLabels) - ).map(_.flatten) - } - - def forGenericMediaKey( - genericMediaKey: GenericMediaKey - ): Stitch[Seq[MediaSafetyLabel]] = { - getFilteredSafetyLabels(genericMediaKey) - } - - private def getFilteredSafetyLabels( - genericMediaKey: GenericMediaKey, - ): Stitch[Seq[MediaSafetyLabel]] = - source - .fetch(genericMediaKey).map(_.flatMap(_.labels.map { stratoSafetyLabelMap => - stratoSafetyLabelMap - .map(label => - MediaSafetyLabel( - MediaSafetyLabelType.fromThrift(label._1), - SafetyLabel.fromThrift(label._2))) - }).toSeq.flatten) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaMetadataFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaMetadataFeatures.docx new file mode 100644 index 000000000..ef95fcabc Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaMetadataFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaMetadataFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaMetadataFeatures.scala deleted file mode 100644 index 97f35b27c..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/media/MediaMetadataFeatures.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.visibility.builder.media - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.mediaservices.media_util.GenericMediaKey -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.MediaMetadataSource -import com.twitter.visibility.features.HasDmcaMediaFeature -import com.twitter.visibility.features.MediaGeoRestrictionsAllowList -import com.twitter.visibility.features.MediaGeoRestrictionsDenyList -import com.twitter.visibility.features.AuthorId - -class MediaMetadataFeatures( - mediaMetadataSource: MediaMetadataSource, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("media_metadata_features") - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val hasDmcaMedia = - scopedStatsReceiver.scope(HasDmcaMediaFeature.name).counter("requests") - private[this] val mediaGeoAllowList = - scopedStatsReceiver.scope(MediaGeoRestrictionsAllowList.name).counter("requests") - private[this] val mediaGeoDenyList = - scopedStatsReceiver.scope(MediaGeoRestrictionsDenyList.name).counter("requests") - private[this] val uploaderId = - scopedStatsReceiver.scope(AuthorId.name).counter("requests") - - def forGenericMediaKey( - genericMediaKey: GenericMediaKey - ): FeatureMapBuilder => FeatureMapBuilder = { featureMapBuilder => - requests.incr() - - featureMapBuilder.withFeature( - HasDmcaMediaFeature, - mediaIsDmca(genericMediaKey) - ) - - featureMapBuilder.withFeature( - MediaGeoRestrictionsAllowList, - geoRestrictionsAllowList(genericMediaKey) - ) - - featureMapBuilder.withFeature( - MediaGeoRestrictionsDenyList, - geoRestrictionsDenyList(genericMediaKey) - ) - - featureMapBuilder.withFeature( - AuthorId, - mediaUploaderId(genericMediaKey) - ) - } - - private def mediaIsDmca(genericMediaKey: GenericMediaKey) = { - hasDmcaMedia.incr() - mediaMetadataSource.getMediaIsDmca(genericMediaKey) - } - - private def geoRestrictionsAllowList(genericMediaKey: GenericMediaKey) = { - mediaGeoAllowList.incr() - mediaMetadataSource.getGeoRestrictionsAllowList(genericMediaKey).map { allowListOpt => - allowListOpt.getOrElse(Nil) - } - } - - private def geoRestrictionsDenyList(genericMediaKey: GenericMediaKey) = { - mediaGeoDenyList.incr() - mediaMetadataSource.getGeoRestrictionsDenyList(genericMediaKey).map { denyListOpt => - denyListOpt.getOrElse(Nil) - } - } - - private def mediaUploaderId(genericMediaKey: GenericMediaKey) = { - uploaderId.incr() - mediaMetadataSource.getMediaUploaderId(genericMediaKey).map { uploaderIdOpt => - uploaderIdOpt.map(Set(_)).getOrElse(Set.empty) - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/BUILD deleted file mode 100644 index ec82612ef..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/thrift/com/twitter/context:twitter-context-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "stitch/stitch-core", - "twitter-context/src/main/scala", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/common", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/util", - "visibility/lib/src/main/thrift/com/twitter/visibility/safety_label_store:safety-label-store-scala", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/BUILD.docx new file mode 100644 index 000000000..4b2040557 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/SpaceFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/SpaceFeatures.docx new file mode 100644 index 000000000..cbdb99d32 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/SpaceFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/SpaceFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/SpaceFeatures.scala deleted file mode 100644 index 6522f8e65..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/spaces/SpaceFeatures.scala +++ /dev/null @@ -1,131 +0,0 @@ -package com.twitter.visibility.builder.spaces - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.Label -import com.twitter.gizmoduck.thriftscala.MuteSurface -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.builder.common.MutedKeywordFeatures -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.builder.users.RelationshipFeatures -import com.twitter.visibility.common.AudioSpaceSource -import com.twitter.visibility.common.SpaceId -import com.twitter.visibility.common.SpaceSafetyLabelMapSource -import com.twitter.visibility.common.UserId -import com.twitter.visibility.features._ -import com.twitter.visibility.models.{MutedKeyword => VfMutedKeyword} -import com.twitter.visibility.models.SafetyLabel -import com.twitter.visibility.models.SpaceSafetyLabel -import com.twitter.visibility.models.SpaceSafetyLabelType - -class SpaceFeatures( - spaceSafetyLabelMap: StratoSpaceLabelMaps, - authorFeatures: AuthorFeatures, - relationshipFeatures: RelationshipFeatures, - mutedKeywordFeatures: MutedKeywordFeatures, - audioSpaceSource: AudioSpaceSource) { - - def forSpaceAndAuthorIds( - spaceId: SpaceId, - viewerId: Option[UserId], - authorIds: Option[Seq[UserId]] - ): FeatureMapBuilder => FeatureMapBuilder = { - - _.withFeature(SpaceSafetyLabels, spaceSafetyLabelMap.forSpaceId(spaceId)) - .withFeature(AuthorId, getSpaceAuthors(spaceId, authorIds).map(_.toSet)) - .withFeature(AuthorUserLabels, allSpaceAuthorLabels(spaceId, authorIds)) - .withFeature(ViewerFollowsAuthor, viewerFollowsAnySpaceAuthor(spaceId, authorIds, viewerId)) - .withFeature(ViewerMutesAuthor, viewerMutesAnySpaceAuthor(spaceId, authorIds, viewerId)) - .withFeature(ViewerBlocksAuthor, viewerBlocksAnySpaceAuthor(spaceId, authorIds, viewerId)) - .withFeature(AuthorBlocksViewer, anySpaceAuthorBlocksViewer(spaceId, authorIds, viewerId)) - .withFeature( - ViewerMutesKeywordInSpaceTitleForNotifications, - titleContainsMutedKeyword( - audioSpaceSource.getSpaceTitle(spaceId), - audioSpaceSource.getSpaceLanguage(spaceId), - viewerId) - ) - } - - def titleContainsMutedKeyword( - titleOptStitch: Stitch[Option[String]], - languageOptStitch: Stitch[Option[String]], - viewerId: Option[UserId], - ): Stitch[VfMutedKeyword] = { - titleOptStitch.flatMap { - case None => Stitch.value(VfMutedKeyword(None)) - case Some(spaceTitle) => - languageOptStitch.flatMap { languageOpt => - mutedKeywordFeatures.spaceTitleContainsMutedKeyword( - spaceTitle, - languageOpt, - mutedKeywordFeatures.allMutedKeywords(viewerId), - MuteSurface.Notifications) - } - } - } - - def getSpaceAuthors( - spaceId: SpaceId, - authorIdsFromRequest: Option[Seq[UserId]] - ): Stitch[Seq[UserId]] = { - authorIdsFromRequest match { - case Some(authorIds) => Stitch.apply(authorIds) - case _ => audioSpaceSource.getAdminIds(spaceId) - } - } - - def allSpaceAuthorLabels( - spaceId: SpaceId, - authorIdsFromRequest: Option[Seq[UserId]] - ): Stitch[Seq[Label]] = { - getSpaceAuthors(spaceId, authorIdsFromRequest) - .flatMap(authorIds => - Stitch.collect(authorIds.map(authorId => authorFeatures.authorUserLabels(authorId)))).map( - _.flatten) - } - - def viewerMutesAnySpaceAuthor( - spaceId: SpaceId, - authorIdsFromRequest: Option[Seq[UserId]], - viewerId: Option[UserId] - ): Stitch[Boolean] = { - getSpaceAuthors(spaceId, authorIdsFromRequest) - .flatMap(authorIds => - Stitch.collect(authorIds.map(authorId => - relationshipFeatures.viewerMutesAuthor(authorId, viewerId)))).map(_.contains(true)) - } - - def anySpaceAuthorBlocksViewer( - spaceId: SpaceId, - authorIdsFromRequest: Option[Seq[UserId]], - viewerId: Option[UserId] - ): Stitch[Boolean] = { - getSpaceAuthors(spaceId, authorIdsFromRequest) - .flatMap(authorIds => - Stitch.collect(authorIds.map(authorId => - relationshipFeatures.authorBlocksViewer(authorId, viewerId)))).map(_.contains(true)) - } -} - -class StratoSpaceLabelMaps( - spaceSafetyLabelSource: SpaceSafetyLabelMapSource, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("space_features") - private[this] val spaceSafetyLabelsStats = - scopedStatsReceiver.scope(SpaceSafetyLabels.name).counter("requests") - - def forSpaceId( - spaceId: SpaceId, - ): Stitch[Seq[SpaceSafetyLabel]] = { - spaceSafetyLabelSource - .fetch(spaceId).map(_.flatMap(_.labels.map { stratoSafetyLabelMap => - stratoSafetyLabelMap - .map(label => - SpaceSafetyLabel( - SpaceSafetyLabelType.fromThrift(label._1), - SafetyLabel.fromThrift(label._2))) - }).toSeq.flatten).ensure(spaceSafetyLabelsStats.incr) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BUILD deleted file mode 100644 index bde1871b4..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BUILD +++ /dev/null @@ -1,38 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "communities/thrift/src/main/thrift/com/twitter/communities:thrift-scala", - "communities/thrift/src/main/thrift/com/twitter/communities/moderation:thrift-scala", - "communities/thrift/src/main/thrift/com/twitter/communities/visibility:thrift-scala", - "escherbird/src/thrift/com/twitter/escherbird/softintervention:softintervention_thrift-scala", - "mediaservices/media-util/src/main/scala", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/model:alias", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/model/notification", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/thrift/com/twitter/context:twitter-context-scala", - "src/thrift/com/twitter/escherbird/common:common-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - # "tweetypie/src/scala/com/twitter/tweetypie/additionalfields", - "twitter-context/src/main/scala", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/common/src/main/scala/com/twitter/visibility/common/stitch", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/blender", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/search", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/tweets", - "visibility/lib/src/main/thrift/com/twitter/visibility/strato:vf-strato-scala", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BUILD.docx new file mode 100644 index 000000000..920a52612 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BlenderContextFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BlenderContextFeatures.docx new file mode 100644 index 000000000..66d9a6713 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BlenderContextFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BlenderContextFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BlenderContextFeatures.scala deleted file mode 100644 index 931574567..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/BlenderContextFeatures.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.search.common.constants.thriftscala.ThriftQuerySource -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.features.SearchCandidateCount -import com.twitter.visibility.features.SearchQueryHasUser -import com.twitter.visibility.features.SearchQuerySource -import com.twitter.visibility.features.SearchResultsPageNumber -import com.twitter.visibility.interfaces.common.blender.BlenderVFRequestContext - -@Deprecated -class BlenderContextFeatures( - statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("blender_context_features") - private[this] val requests = scopedStatsReceiver.counter("requests") - private[this] val searchResultsPageNumber = - scopedStatsReceiver.scope(SearchResultsPageNumber.name).counter("requests") - private[this] val searchCandidateCount = - scopedStatsReceiver.scope(SearchCandidateCount.name).counter("requests") - private[this] val searchQuerySource = - scopedStatsReceiver.scope(SearchQuerySource.name).counter("requests") - private[this] val searchQueryHasUser = - scopedStatsReceiver.scope(SearchQueryHasUser.name).counter("requests") - - def forBlenderContext( - blenderContext: BlenderVFRequestContext - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - searchResultsPageNumber.incr() - searchCandidateCount.incr() - searchQuerySource.incr() - searchQueryHasUser.incr() - - _.withConstantFeature(SearchResultsPageNumber, blenderContext.resultsPageNumber) - .withConstantFeature(SearchCandidateCount, blenderContext.candidateCount) - .withConstantFeature( - SearchQuerySource, - blenderContext.querySourceOption match { - case Some(querySource) => querySource - case _ => ThriftQuerySource.Unknown - }) - .withConstantFeature(SearchQueryHasUser, blenderContext.queryHasUser) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityNotificationFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityNotificationFeatures.docx new file mode 100644 index 000000000..c287e87eb Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityNotificationFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityNotificationFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityNotificationFeatures.scala deleted file mode 100644 index a4588620b..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityNotificationFeatures.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.notificationservice.model.notification.ActivityNotification -import com.twitter.notificationservice.model.notification.MentionNotification -import com.twitter.notificationservice.model.notification.MentionQuoteNotification -import com.twitter.notificationservice.model.notification.Notification -import com.twitter.notificationservice.model.notification.QuoteTweetNotification -import com.twitter.servo.util.Gate -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.TweetSource -import com.twitter.visibility.features.NotificationIsOnCommunityTweet -import com.twitter.visibility.models.CommunityTweet - -object CommunityNotificationFeatures { - def ForNonCommunityTweetNotification: FeatureMapBuilder => FeatureMapBuilder = { - _.withConstantFeature(NotificationIsOnCommunityTweet, false) - } -} - -class CommunityNotificationFeatures( - tweetSource: TweetSource, - enableCommunityTweetHydration: Gate[Long], - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("community_notification_features") - private[this] val requestsCounter = scopedStatsReceiver.counter("requests") - private[this] val hydrationsCounter = scopedStatsReceiver.counter("hydrations") - private[this] val notificationIsOnCommunityTweetCounter = - scopedStatsReceiver.scope(NotificationIsOnCommunityTweet.name).counter("true") - private[this] val notificationIsNotOnCommunityTweetCounter = - scopedStatsReceiver.scope(NotificationIsOnCommunityTweet.name).counter("false") - - def forNotification(notification: Notification): FeatureMapBuilder => FeatureMapBuilder = { - requestsCounter.incr() - val isCommunityTweetResult = getTweetIdOption(notification) match { - case Some(tweetId) if enableCommunityTweetHydration(notification.target) => - hydrationsCounter.incr() - tweetSource - .getTweet(tweetId) - .map { - case Some(tweet) if CommunityTweet(tweet).nonEmpty => - notificationIsOnCommunityTweetCounter.incr() - true - case _ => - notificationIsNotOnCommunityTweetCounter.incr() - false - } - case _ => Stitch.False - } - _.withFeature(NotificationIsOnCommunityTweet, isCommunityTweetResult) - } - - private[this] def getTweetIdOption(notification: Notification): Option[Long] = { - notification match { - case n: MentionNotification => Some(n.mentioningTweetId) - case n: MentionQuoteNotification => Some(n.mentioningTweetId) - case n: QuoteTweetNotification => Some(n.quotedTweetId) - case n: ActivityNotification[_] if n.visibilityTweets.contains(n.objectId) => Some(n.objectId) - case _ => None - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeatures.docx new file mode 100644 index 000000000..4d2bf82c2 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeatures.scala deleted file mode 100644 index 8a2c752f6..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeatures.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.features.CommunityTweetAuthorIsRemoved -import com.twitter.visibility.features.CommunityTweetCommunityNotFound -import com.twitter.visibility.features.CommunityTweetCommunityDeleted -import com.twitter.visibility.features.CommunityTweetCommunitySuspended -import com.twitter.visibility.features.CommunityTweetCommunityVisible -import com.twitter.visibility.features.CommunityTweetIsHidden -import com.twitter.visibility.features.TweetIsCommunityTweet -import com.twitter.visibility.features.ViewerIsCommunityAdmin -import com.twitter.visibility.features.ViewerIsCommunityMember -import com.twitter.visibility.features.ViewerIsCommunityModerator -import com.twitter.visibility.features.ViewerIsInternalCommunitiesAdmin -import com.twitter.visibility.models.CommunityTweet -import com.twitter.visibility.models.ViewerContext - -trait CommunityTweetFeatures { - - def forTweet( - tweet: Tweet, - viewerContext: ViewerContext - ): FeatureMapBuilder => FeatureMapBuilder - - def forTweetOnly(tweet: Tweet): FeatureMapBuilder => FeatureMapBuilder = { - _.withConstantFeature( - TweetIsCommunityTweet, - CommunityTweet(tweet).isDefined - ) - } - - protected def forNonCommunityTweet(): FeatureMapBuilder => FeatureMapBuilder = { builder => - builder - .withConstantFeature( - TweetIsCommunityTweet, - false - ).withConstantFeature( - CommunityTweetCommunityNotFound, - false - ).withConstantFeature( - CommunityTweetCommunitySuspended, - false - ).withConstantFeature( - CommunityTweetCommunityDeleted, - false - ).withConstantFeature( - CommunityTweetCommunityVisible, - false - ).withConstantFeature( - ViewerIsInternalCommunitiesAdmin, - false - ).withConstantFeature( - ViewerIsCommunityAdmin, - false - ).withConstantFeature( - ViewerIsCommunityModerator, - false - ).withConstantFeature( - ViewerIsCommunityMember, - false - ).withConstantFeature( - CommunityTweetIsHidden, - false - ).withConstantFeature( - CommunityTweetAuthorIsRemoved, - false - ) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesPartitioned.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesPartitioned.docx new file mode 100644 index 000000000..95176a74b Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesPartitioned.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesPartitioned.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesPartitioned.scala deleted file mode 100644 index 695f724eb..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesPartitioned.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.servo.util.Gate -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.models.ViewerContext - -class CommunityTweetFeaturesPartitioned( - a: CommunityTweetFeatures, - b: CommunityTweetFeatures, - bEnabled: Gate[Unit], -) extends CommunityTweetFeatures { - override def forTweet( - tweet: Tweet, - viewerContext: ViewerContext - ): FeatureMapBuilder => FeatureMapBuilder = - bEnabled.pick( - b.forTweet(tweet, viewerContext), - a.forTweet(tweet, viewerContext), - ) - - override def forTweetOnly(tweet: Tweet): FeatureMapBuilder => FeatureMapBuilder = bEnabled.pick( - b.forTweetOnly(tweet), - a.forTweetOnly(tweet), - ) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesV2.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesV2.docx new file mode 100644 index 000000000..2cfc21a18 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesV2.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesV2.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesV2.scala deleted file mode 100644 index 407b5308c..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/CommunityTweetFeaturesV2.scala +++ /dev/null @@ -1,129 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.communities.moderation.thriftscala.CommunityTweetModerationState -import com.twitter.communities.moderation.thriftscala.CommunityUserModerationState -import com.twitter.communities.visibility.thriftscala.CommunityVisibilityFeatures -import com.twitter.communities.visibility.thriftscala.CommunityVisibilityFeaturesV1 -import com.twitter.communities.visibility.thriftscala.CommunityVisibilityResult -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.CommunitiesSource -import com.twitter.visibility.features.CommunityTweetAuthorIsRemoved -import com.twitter.visibility.features.CommunityTweetCommunityNotFound -import com.twitter.visibility.features.CommunityTweetCommunityDeleted -import com.twitter.visibility.features.CommunityTweetCommunitySuspended -import com.twitter.visibility.features.CommunityTweetCommunityVisible -import com.twitter.visibility.features.CommunityTweetIsHidden -import com.twitter.visibility.features.TweetIsCommunityTweet -import com.twitter.visibility.features.ViewerIsCommunityAdmin -import com.twitter.visibility.features.ViewerIsCommunityMember -import com.twitter.visibility.features.ViewerIsCommunityModerator -import com.twitter.visibility.features.ViewerIsInternalCommunitiesAdmin -import com.twitter.visibility.models.CommunityTweet -import com.twitter.visibility.models.ViewerContext - -class CommunityTweetFeaturesV2(communitiesSource: CommunitiesSource) - extends CommunityTweetFeatures { - private[this] def forCommunityTweet( - communityTweet: CommunityTweet - ): FeatureMapBuilder => FeatureMapBuilder = { builder: FeatureMapBuilder => - { - val communityVisibilityFeaturesStitch = - communitiesSource.getCommunityVisibilityFeatures(communityTweet.communityId) - val communityTweetModerationStateStitch = - communitiesSource.getTweetModerationState(communityTweet.tweet.id) - val communityTweetAuthorModerationStateStitch = - communitiesSource.getUserModerationState( - communityTweet.authorId, - communityTweet.communityId - ) - - def getFlagFromFeatures(f: CommunityVisibilityFeaturesV1 => Boolean): Stitch[Boolean] = - communityVisibilityFeaturesStitch.map { - case Some(CommunityVisibilityFeatures.V1(v1)) => f(v1) - case _ => false - } - - def getFlagFromCommunityVisibilityResult( - f: CommunityVisibilityResult => Boolean - ): Stitch[Boolean] = getFlagFromFeatures { v => - f(v.communityVisibilityResult) - } - - builder - .withConstantFeature( - TweetIsCommunityTweet, - true - ) - .withFeature( - CommunityTweetCommunityNotFound, - getFlagFromCommunityVisibilityResult { - case CommunityVisibilityResult.NotFound => true - case _ => false - } - ) - .withFeature( - CommunityTweetCommunitySuspended, - getFlagFromCommunityVisibilityResult { - case CommunityVisibilityResult.Suspended => true - case _ => false - } - ) - .withFeature( - CommunityTweetCommunityDeleted, - getFlagFromCommunityVisibilityResult { - case CommunityVisibilityResult.Deleted => true - case _ => false - } - ) - .withFeature( - CommunityTweetCommunityVisible, - getFlagFromCommunityVisibilityResult { - case CommunityVisibilityResult.Visible => true - case _ => false - } - ) - .withFeature( - ViewerIsInternalCommunitiesAdmin, - getFlagFromFeatures { _.viewerIsInternalAdmin } - ) - .withFeature( - ViewerIsCommunityAdmin, - getFlagFromFeatures { _.viewerIsCommunityAdmin } - ) - .withFeature( - ViewerIsCommunityModerator, - getFlagFromFeatures { _.viewerIsCommunityModerator } - ) - .withFeature( - ViewerIsCommunityMember, - getFlagFromFeatures { _.viewerIsCommunityMember } - ) - .withFeature( - CommunityTweetIsHidden, - communityTweetModerationStateStitch.map { - case Some(CommunityTweetModerationState.Hidden(_)) => true - case _ => false - } - ) - .withFeature( - CommunityTweetAuthorIsRemoved, - communityTweetAuthorModerationStateStitch.map { - case Some(CommunityUserModerationState.Removed(_)) => true - case _ => false - } - ) - } - } - - def forTweet( - tweet: Tweet, - viewerContext: ViewerContext - ): FeatureMapBuilder => FeatureMapBuilder = { - CommunityTweet(tweet) match { - case None => forNonCommunityTweet() - case Some(communityTweet) => forCommunityTweet(communityTweet) - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ConversationControlFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ConversationControlFeatures.docx new file mode 100644 index 000000000..8399a0dd3 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ConversationControlFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ConversationControlFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ConversationControlFeatures.scala deleted file mode 100644 index 3f5ad7281..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ConversationControlFeatures.scala +++ /dev/null @@ -1,178 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.tweetypie.thriftscala.ConversationControl -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.builder.users.RelationshipFeatures -import com.twitter.visibility.common.InvitedToConversationRepo -import com.twitter.visibility.features.ConversationRootAuthorFollowsViewer -import com.twitter.visibility.features.TweetConversationViewerIsInvited -import com.twitter.visibility.features.TweetConversationViewerIsInvitedViaReplyMention -import com.twitter.visibility.features.TweetConversationViewerIsRootAuthor -import com.twitter.visibility.features.TweetHasByInvitationConversationControl -import com.twitter.visibility.features.TweetHasCommunityConversationControl -import com.twitter.visibility.features.TweetHasFollowersConversationControl -import com.twitter.visibility.features.ViewerFollowsConversationRootAuthor - -class ConversationControlFeatures( - relationshipFeatures: RelationshipFeatures, - isInvitedToConversationRepository: InvitedToConversationRepo, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("conversation_control_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val tweetCommunityConversationRequest = - scopedStatsReceiver.scope(TweetHasCommunityConversationControl.name).counter("requests") - private[this] val tweetByInvitationConversationRequest = - scopedStatsReceiver.scope(TweetHasByInvitationConversationControl.name).counter("requests") - private[this] val tweetFollowersConversationRequest = - scopedStatsReceiver.scope(TweetHasFollowersConversationControl.name).counter("requests") - private[this] val rootAuthorFollowsViewer = - scopedStatsReceiver.scope(ConversationRootAuthorFollowsViewer.name).counter("requests") - private[this] val viewerFollowsRootAuthor = - scopedStatsReceiver.scope(ViewerFollowsConversationRootAuthor.name).counter("requests") - - def isCommunityConversation(conversationControl: Option[ConversationControl]): Boolean = - conversationControl - .collect { - case _: ConversationControl.Community => - tweetCommunityConversationRequest.incr() - true - }.getOrElse(false) - - def isByInvitationConversation(conversationControl: Option[ConversationControl]): Boolean = - conversationControl - .collect { - case _: ConversationControl.ByInvitation => - tweetByInvitationConversationRequest.incr() - true - }.getOrElse(false) - - def isFollowersConversation(conversationControl: Option[ConversationControl]): Boolean = - conversationControl - .collect { - case _: ConversationControl.Followers => - tweetFollowersConversationRequest.incr() - true - }.getOrElse(false) - - def conversationRootAuthorId( - conversationControl: Option[ConversationControl] - ): Option[Long] = - conversationControl match { - case Some(ConversationControl.Community(community)) => - Some(community.conversationTweetAuthorId) - case Some(ConversationControl.ByInvitation(byInvitation)) => - Some(byInvitation.conversationTweetAuthorId) - case Some(ConversationControl.Followers(followers)) => - Some(followers.conversationTweetAuthorId) - case _ => None - } - - def viewerIsRootAuthor( - conversationControl: Option[ConversationControl], - viewerIdOpt: Option[Long] - ): Boolean = - (conversationRootAuthorId(conversationControl), viewerIdOpt) match { - case (Some(rootAuthorId), Some(viewerId)) if rootAuthorId == viewerId => true - case _ => false - } - - def viewerIsInvited( - conversationControl: Option[ConversationControl], - viewerId: Option[Long] - ): Boolean = { - val invitedUserIds = conversationControl match { - case Some(ConversationControl.Community(community)) => - community.invitedUserIds - case Some(ConversationControl.ByInvitation(byInvitation)) => - byInvitation.invitedUserIds - case Some(ConversationControl.Followers(followers)) => - followers.invitedUserIds - case _ => Seq() - } - - viewerId.exists(invitedUserIds.contains(_)) - } - - def conversationAuthorFollows( - conversationControl: Option[ConversationControl], - viewerId: Option[Long] - ): Stitch[Boolean] = { - val conversationAuthorId = conversationControl.collect { - case ConversationControl.Community(community) => - community.conversationTweetAuthorId - } - - conversationAuthorId match { - case Some(authorId) => - rootAuthorFollowsViewer.incr() - relationshipFeatures.authorFollowsViewer(authorId, viewerId) - case None => - Stitch.False - } - } - - def followsConversationAuthor( - conversationControl: Option[ConversationControl], - viewerId: Option[Long] - ): Stitch[Boolean] = { - val conversationAuthorId = conversationControl.collect { - case ConversationControl.Followers(followers) => - followers.conversationTweetAuthorId - } - - conversationAuthorId match { - case Some(authorId) => - viewerFollowsRootAuthor.incr() - relationshipFeatures.viewerFollowsAuthor(authorId, viewerId) - case None => - Stitch.False - } - } - - def viewerIsInvitedViaReplyMention( - tweet: Tweet, - viewerIdOpt: Option[Long] - ): Stitch[Boolean] = { - val conversationIdOpt: Option[Long] = tweet.conversationControl match { - case Some(ConversationControl.Community(community)) - if community.inviteViaMention.contains(true) => - tweet.coreData.flatMap(_.conversationId) - case Some(ConversationControl.ByInvitation(invitation)) - if invitation.inviteViaMention.contains(true) => - tweet.coreData.flatMap(_.conversationId) - case Some(ConversationControl.Followers(followers)) - if followers.inviteViaMention.contains(true) => - tweet.coreData.flatMap(_.conversationId) - case _ => None - } - - (conversationIdOpt, viewerIdOpt) match { - case (Some(conversationId), Some(viewerId)) => - isInvitedToConversationRepository(conversationId, viewerId) - case _ => Stitch.False - } - } - - def forTweet(tweet: Tweet, viewerId: Option[Long]): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - val cc = tweet.conversationControl - - _.withConstantFeature(TweetHasCommunityConversationControl, isCommunityConversation(cc)) - .withConstantFeature(TweetHasByInvitationConversationControl, isByInvitationConversation(cc)) - .withConstantFeature(TweetHasFollowersConversationControl, isFollowersConversation(cc)) - .withConstantFeature(TweetConversationViewerIsRootAuthor, viewerIsRootAuthor(cc, viewerId)) - .withConstantFeature(TweetConversationViewerIsInvited, viewerIsInvited(cc, viewerId)) - .withFeature(ConversationRootAuthorFollowsViewer, conversationAuthorFollows(cc, viewerId)) - .withFeature(ViewerFollowsConversationRootAuthor, followsConversationAuthor(cc, viewerId)) - .withFeature( - TweetConversationViewerIsInvitedViaReplyMention, - viewerIsInvitedViaReplyMention(tweet, viewerId)) - - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/EditTweetFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/EditTweetFeatures.docx new file mode 100644 index 000000000..4400f677d Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/EditTweetFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/EditTweetFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/EditTweetFeatures.scala deleted file mode 100644 index 224f4a9a4..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/EditTweetFeatures.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.tweetypie.thriftscala.EditControl -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.features.TweetIsEditTweet -import com.twitter.visibility.features.TweetIsInitialTweet -import com.twitter.visibility.features.TweetIsLatestTweet -import com.twitter.visibility.features.TweetIsStaleTweet - -class EditTweetFeatures( - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("edit_tweet_features") - private[this] val tweetIsEditTweet = - scopedStatsReceiver.scope(TweetIsEditTweet.name).counter("requests") - private[this] val tweetIsStaleTweet = - scopedStatsReceiver.scope(TweetIsStaleTweet.name).counter("requests") - private[this] val tweetIsLatestTweet = - scopedStatsReceiver.scope(TweetIsLatestTweet.name).counter("requests") - private[this] val tweetIsInitialTweet = - scopedStatsReceiver.scope(TweetIsInitialTweet.name).counter("requests") - - def forTweet( - tweet: Tweet - ): FeatureMapBuilder => FeatureMapBuilder = { - _.withConstantFeature(TweetIsEditTweet, tweetIsEditTweet(tweet)) - .withConstantFeature(TweetIsStaleTweet, tweetIsStaleTweet(tweet)) - .withConstantFeature(TweetIsLatestTweet, tweetIsLatestTweet(tweet)) - .withConstantFeature(TweetIsInitialTweet, tweetIsInitialTweet(tweet)) - } - - def tweetIsStaleTweet(tweet: Tweet, incrementMetric: Boolean = true): Boolean = { - if (incrementMetric) tweetIsStaleTweet.incr() - - tweet.editControl match { - case None => false - case Some(ec) => - ec match { - case eci: EditControl.Initial => eci.initial.editTweetIds.last != tweet.id - case ece: EditControl.Edit => - ece.edit.editControlInitial.exists(_.editTweetIds.last != tweet.id) - case _ => false - } - } - } - - def tweetIsEditTweet(tweet: Tweet, incrementMetric: Boolean = true): Boolean = { - if (incrementMetric) tweetIsEditTweet.incr() - - tweet.editControl match { - case None => false - case Some(ec) => - ec match { - case _: EditControl.Initial => false - case _ => true - } - } - } - - def tweetIsLatestTweet(tweet: Tweet): Boolean = { - tweetIsLatestTweet.incr() - !tweetIsStaleTweet(tweet = tweet, incrementMetric = false) - } - - def tweetIsInitialTweet(tweet: Tweet): Boolean = { - tweetIsInitialTweet.incr() - !tweetIsEditTweet(tweet = tweet, incrementMetric = false) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ExclusiveTweetFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ExclusiveTweetFeatures.docx new file mode 100644 index 000000000..19beee478 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ExclusiveTweetFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ExclusiveTweetFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ExclusiveTweetFeatures.scala deleted file mode 100644 index 94aee6430..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ExclusiveTweetFeatures.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.builder.users.ViewerVerbsAuthor -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.features.TweetIsExclusiveTweet -import com.twitter.visibility.features.ViewerIsExclusiveTweetRootAuthor -import com.twitter.visibility.features.ViewerSuperFollowsExclusiveTweetRootAuthor -import com.twitter.visibility.models.ViewerContext - -class ExclusiveTweetFeatures( - userRelationshipSource: UserRelationshipSource, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("exclusive_tweet_features") - private[this] val viewerSuperFollowsAuthor = - scopedStatsReceiver.scope(ViewerSuperFollowsExclusiveTweetRootAuthor.name).counter("requests") - - def rootAuthorId(tweet: Tweet): Option[Long] = - tweet.exclusiveTweetControl.map(_.conversationAuthorId) - - def viewerIsRootAuthor( - tweet: Tweet, - viewerIdOpt: Option[Long] - ): Boolean = - (rootAuthorId(tweet), viewerIdOpt) match { - case (Some(rootAuthorId), Some(viewerId)) if rootAuthorId == viewerId => true - case _ => false - } - - def viewerSuperFollowsRootAuthor( - tweet: Tweet, - viewerId: Option[Long] - ): Stitch[Boolean] = - rootAuthorId(tweet) match { - case Some(authorId) => - ViewerVerbsAuthor( - authorId, - viewerId, - userRelationshipSource.superFollows, - viewerSuperFollowsAuthor) - case None => - Stitch.False - } - - def forTweet( - tweet: Tweet, - viewerContext: ViewerContext - ): FeatureMapBuilder => FeatureMapBuilder = { - val viewerId = viewerContext.userId - - _.withConstantFeature(TweetIsExclusiveTweet, tweet.exclusiveTweetControl.isDefined) - .withConstantFeature(ViewerIsExclusiveTweetRootAuthor, viewerIsRootAuthor(tweet, viewerId)) - .withFeature( - ViewerSuperFollowsExclusiveTweetRootAuthor, - viewerSuperFollowsRootAuthor(tweet, viewerId)) - } - - def forTweetOnly(tweet: Tweet): FeatureMapBuilder => FeatureMapBuilder = { - _.withConstantFeature(TweetIsExclusiveTweet, tweet.exclusiveTweetControl.isDefined) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrPefetchedLabelsRelationshipFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrPefetchedLabelsRelationshipFeatures.docx new file mode 100644 index 000000000..2cf0e504a Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrPefetchedLabelsRelationshipFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrPefetchedLabelsRelationshipFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrPefetchedLabelsRelationshipFeatures.scala deleted file mode 100644 index f9a0445f9..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrPefetchedLabelsRelationshipFeatures.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.builder.users.ViewerVerbsAuthor -import com.twitter.visibility.common.UserId -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.features._ -import com.twitter.visibility.models.TweetSafetyLabel -import com.twitter.visibility.models.ViolationLevel - -class FosnrPefetchedLabelsRelationshipFeatures( - userRelationshipSource: UserRelationshipSource, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = - statsReceiver.scope("fonsr_prefetched_relationship_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val viewerFollowsAuthorOfViolatingTweet = - scopedStatsReceiver.scope(ViewerFollowsAuthorOfViolatingTweet.name).counter("requests") - - private[this] val viewerDoesNotFollowAuthorOfViolatingTweet = - scopedStatsReceiver.scope(ViewerDoesNotFollowAuthorOfViolatingTweet.name).counter("requests") - - def forNonFosnr(): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - _.withConstantFeature(ViewerFollowsAuthorOfViolatingTweet, false) - .withConstantFeature(ViewerDoesNotFollowAuthorOfViolatingTweet, false) - } - def forTweetWithSafetyLabelsAndAuthorId( - safetyLabels: Seq[TweetSafetyLabel], - authorId: Long, - viewerId: Option[Long] - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - _.withFeature( - ViewerFollowsAuthorOfViolatingTweet, - viewerFollowsAuthorOfViolatingTweet(safetyLabels, authorId, viewerId)) - .withFeature( - ViewerDoesNotFollowAuthorOfViolatingTweet, - viewerDoesNotFollowAuthorOfViolatingTweet(safetyLabels, authorId, viewerId)) - } - def viewerFollowsAuthorOfViolatingTweet( - safetyLabels: Seq[TweetSafetyLabel], - authorId: UserId, - viewerId: Option[UserId] - ): Stitch[Boolean] = { - if (safetyLabels - .map(ViolationLevel.fromTweetSafetyLabelOpt).collect { - case Some(level) => level - }.isEmpty) { - return Stitch.False - } - ViewerVerbsAuthor( - authorId, - viewerId, - userRelationshipSource.follows, - viewerFollowsAuthorOfViolatingTweet) - } - def viewerDoesNotFollowAuthorOfViolatingTweet( - safetyLabels: Seq[TweetSafetyLabel], - authorId: UserId, - viewerId: Option[UserId] - ): Stitch[Boolean] = { - if (safetyLabels - .map(ViolationLevel.fromTweetSafetyLabelOpt).collect { - case Some(level) => level - }.isEmpty) { - return Stitch.False - } - ViewerVerbsAuthor( - authorId, - viewerId, - userRelationshipSource.follows, - viewerDoesNotFollowAuthorOfViolatingTweet).map(following => !following) - } - -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrRelationshipFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrRelationshipFeatures.docx new file mode 100644 index 000000000..7de8433a7 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrRelationshipFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrRelationshipFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrRelationshipFeatures.scala deleted file mode 100644 index a6758eefa..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/FosnrRelationshipFeatures.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.builder.users.ViewerVerbsAuthor -import com.twitter.visibility.common.UserId -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.features._ -import com.twitter.visibility.models.ViolationLevel - -class FosnrRelationshipFeatures( - tweetLabels: TweetLabels, - userRelationshipSource: UserRelationshipSource, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("fonsr_relationship_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val viewerFollowsAuthorOfViolatingTweet = - scopedStatsReceiver.scope(ViewerFollowsAuthorOfViolatingTweet.name).counter("requests") - - private[this] val viewerDoesNotFollowAuthorOfViolatingTweet = - scopedStatsReceiver.scope(ViewerDoesNotFollowAuthorOfViolatingTweet.name).counter("requests") - - def forTweetAndAuthorId( - tweet: Tweet, - authorId: Long, - viewerId: Option[Long] - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - _.withFeature( - ViewerFollowsAuthorOfViolatingTweet, - viewerFollowsAuthorOfViolatingTweet(tweet, authorId, viewerId)) - .withFeature( - ViewerDoesNotFollowAuthorOfViolatingTweet, - viewerDoesNotFollowAuthorOfViolatingTweet(tweet, authorId, viewerId)) - } - - def viewerFollowsAuthorOfViolatingTweet( - tweet: Tweet, - authorId: UserId, - viewerId: Option[UserId] - ): Stitch[Boolean] = - tweetLabels.forTweet(tweet).flatMap { safetyLabels => - if (safetyLabels - .map(ViolationLevel.fromTweetSafetyLabelOpt).collect { - case Some(level) => level - }.isEmpty) { - Stitch.False - } else { - ViewerVerbsAuthor( - authorId, - viewerId, - userRelationshipSource.follows, - viewerFollowsAuthorOfViolatingTweet) - } - } - - def viewerDoesNotFollowAuthorOfViolatingTweet( - tweet: Tweet, - authorId: UserId, - viewerId: Option[UserId] - ): Stitch[Boolean] = - tweetLabels.forTweet(tweet).flatMap { safetyLabels => - if (safetyLabels - .map(ViolationLevel.fromTweetSafetyLabelOpt).collect { - case Some(level) => level - }.isEmpty) { - Stitch.False - } else { - ViewerVerbsAuthor( - authorId, - viewerId, - userRelationshipSource.follows, - viewerDoesNotFollowAuthorOfViolatingTweet).map(following => !following) - } - } - -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/MisinformationPolicyFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/MisinformationPolicyFeatures.docx new file mode 100644 index 000000000..28e924d08 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/MisinformationPolicyFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/MisinformationPolicyFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/MisinformationPolicyFeatures.scala deleted file mode 100644 index 49d01dc14..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/MisinformationPolicyFeatures.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.EscherbirdEntityAnnotations -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.MisinformationPolicySource -import com.twitter.visibility.features._ -import com.twitter.visibility.models.MisinformationPolicy -import com.twitter.visibility.models.SemanticCoreMisinformation -import com.twitter.visibility.models.ViewerContext - -class MisinformationPolicyFeatures( - misinformationPolicySource: MisinformationPolicySource, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = - statsReceiver.scope("misinformation_policy_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - private[this] val tweetMisinformationPolicies = - scopedStatsReceiver.scope(TweetMisinformationPolicies.name).counter("requests") - - def forTweet( - tweet: Tweet, - viewerContext: ViewerContext - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - tweetMisinformationPolicies.incr() - - _.withFeature( - TweetMisinformationPolicies, - misinformationPolicy(tweet.escherbirdEntityAnnotations, viewerContext)) - .withFeature( - TweetEnglishMisinformationPolicies, - misinformationPolicyEnglishOnly(tweet.escherbirdEntityAnnotations)) - } - - def misinformationPolicyEnglishOnly( - escherbirdEntityAnnotations: Option[EscherbirdEntityAnnotations], - ): Stitch[Seq[MisinformationPolicy]] = { - val locale = Some( - MisinformationPolicySource.LanguageAndCountry( - language = Some("en"), - country = Some("us") - )) - fetchMisinformationPolicy(escherbirdEntityAnnotations, locale) - } - - def misinformationPolicy( - escherbirdEntityAnnotations: Option[EscherbirdEntityAnnotations], - viewerContext: ViewerContext - ): Stitch[Seq[MisinformationPolicy]] = { - val locale = viewerContext.requestLanguageCode.map { language => - MisinformationPolicySource.LanguageAndCountry( - language = Some(language), - country = viewerContext.requestCountryCode - ) - } - fetchMisinformationPolicy(escherbirdEntityAnnotations, locale) - } - - def fetchMisinformationPolicy( - escherbirdEntityAnnotations: Option[EscherbirdEntityAnnotations], - locale: Option[MisinformationPolicySource.LanguageAndCountry] - ): Stitch[Seq[MisinformationPolicy]] = { - Stitch.collect( - escherbirdEntityAnnotations - .map(_.entityAnnotations) - .getOrElse(Seq.empty) - .filter(_.domainId == SemanticCoreMisinformation.domainId) - .map(annotation => - misinformationPolicySource - .fetch( - annotation, - locale - ) - .map(misinformation => - MisinformationPolicy( - annotation = annotation, - misinformation = misinformation - ))) - ) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ModerationFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ModerationFeatures.docx new file mode 100644 index 000000000..5a01c0c74 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ModerationFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ModerationFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ModerationFeatures.scala deleted file mode 100644 index 08fc4ad3b..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ModerationFeatures.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.features.TweetIsModerated - -class ModerationFeatures(moderationSource: Long => Boolean, statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver: StatsReceiver = - statsReceiver.scope("moderation_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val tweetIsModerated = - scopedStatsReceiver.scope(TweetIsModerated.name).counter("requests") - - def forTweetId(tweetId: Long): FeatureMapBuilder => FeatureMapBuilder = { featureMapBuilder => - requests.incr() - tweetIsModerated.incr() - - featureMapBuilder.withConstantFeature(TweetIsModerated, moderationSource(tweetId)) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/SearchContextFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/SearchContextFeatures.docx new file mode 100644 index 000000000..91fcc9ca5 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/SearchContextFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/SearchContextFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/SearchContextFeatures.scala deleted file mode 100644 index 8be075ceb..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/SearchContextFeatures.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.search.common.constants.thriftscala.ThriftQuerySource -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.features.SearchCandidateCount -import com.twitter.visibility.features.SearchQueryHasUser -import com.twitter.visibility.features.SearchQuerySource -import com.twitter.visibility.features.SearchResultsPageNumber -import com.twitter.visibility.interfaces.common.search.SearchVFRequestContext - -class SearchContextFeatures( - statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("search_context_features") - private[this] val requests = scopedStatsReceiver.counter("requests") - private[this] val searchResultsPageNumber = - scopedStatsReceiver.scope(SearchResultsPageNumber.name).counter("requests") - private[this] val searchCandidateCount = - scopedStatsReceiver.scope(SearchCandidateCount.name).counter("requests") - private[this] val searchQuerySource = - scopedStatsReceiver.scope(SearchQuerySource.name).counter("requests") - private[this] val searchQueryHasUser = - scopedStatsReceiver.scope(SearchQueryHasUser.name).counter("requests") - - def forSearchContext( - searchContext: SearchVFRequestContext - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - searchResultsPageNumber.incr() - searchCandidateCount.incr() - searchQuerySource.incr() - searchQueryHasUser.incr() - - _.withConstantFeature(SearchResultsPageNumber, searchContext.resultsPageNumber) - .withConstantFeature(SearchCandidateCount, searchContext.candidateCount) - .withConstantFeature( - SearchQuerySource, - searchContext.querySourceOption match { - case Some(querySource) => querySource - case _ => ThriftQuerySource.Unknown - }) - .withConstantFeature(SearchQueryHasUser, searchContext.queryHasUser) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ToxicReplyFilterFeature.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ToxicReplyFilterFeature.docx new file mode 100644 index 000000000..ce959bda5 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ToxicReplyFilterFeature.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ToxicReplyFilterFeature.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ToxicReplyFilterFeature.scala deleted file mode 100644 index 04af003f0..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/ToxicReplyFilterFeature.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.contenthealth.toxicreplyfilter.thriftscala.FilterState -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.features.ToxicReplyFilterConversationAuthorIsViewer -import com.twitter.visibility.features.ToxicReplyFilterState - -class ToxicReplyFilterFeature( - statsReceiver: StatsReceiver) { - - def forTweet(tweet: Tweet, viewerId: Option[Long]): FeatureMapBuilder => FeatureMapBuilder = { - builder => - requests.incr() - - builder - .withConstantFeature(ToxicReplyFilterState, isTweetFilteredFromAuthor(tweet)) - .withConstantFeature( - ToxicReplyFilterConversationAuthorIsViewer, - isRootAuthorViewer(tweet, viewerId)) - } - - private[this] def isRootAuthorViewer(tweet: Tweet, maybeViewerId: Option[Long]): Boolean = { - val maybeAuthorId = tweet.filteredReplyDetails.map(_.conversationAuthorId) - - (maybeViewerId, maybeAuthorId) match { - case (Some(viewerId), Some(authorId)) if viewerId == authorId => { - rootAuthorViewerStats.incr() - true - } - case _ => false - } - } - - private[this] def isTweetFilteredFromAuthor( - tweet: Tweet, - ): FilterState = { - val result = tweet.filteredReplyDetails.map(_.filterState).getOrElse(FilterState.Unfiltered) - - if (result == FilterState.FilteredFromAuthor) { - filteredFromAuthorStats.incr() - } - result - } - - private[this] val scopedStatsReceiver = - statsReceiver.scope("toxicreplyfilter") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val rootAuthorViewerStats = - scopedStatsReceiver.scope(ToxicReplyFilterConversationAuthorIsViewer.name).counter("requests") - - private[this] val filteredFromAuthorStats = - scopedStatsReceiver.scope(ToxicReplyFilterState.name).counter("requests") -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TrustedFriendsFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TrustedFriendsFeatures.docx new file mode 100644 index 000000000..9d6ba5b22 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TrustedFriendsFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TrustedFriendsFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TrustedFriendsFeatures.scala deleted file mode 100644 index c381c47b3..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TrustedFriendsFeatures.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.TrustedFriendsListId -import com.twitter.visibility.common.TrustedFriendsSource -import com.twitter.visibility.features.TweetIsTrustedFriendTweet -import com.twitter.visibility.features.ViewerIsTrustedFriendOfTweetAuthor -import com.twitter.visibility.features.ViewerIsTrustedFriendTweetAuthor - -class TrustedFriendsFeatures(trustedFriendsSource: TrustedFriendsSource) { - - private[builder] def viewerIsTrustedFriend( - tweet: Tweet, - viewerId: Option[Long] - ): Stitch[Boolean] = - (trustedFriendsListId(tweet), viewerId) match { - case (Some(tfListId), Some(userId)) => - trustedFriendsSource.isTrustedFriend(tfListId, userId) - case _ => Stitch.False - } - - private[builder] def viewerIsTrustedFriendListOwner( - tweet: Tweet, - viewerId: Option[Long] - ): Stitch[Boolean] = - (trustedFriendsListId(tweet), viewerId) match { - case (Some(tfListId), Some(userId)) => - trustedFriendsSource.isTrustedFriendListOwner(tfListId, userId) - case _ => Stitch.False - } - - private[builder] def trustedFriendsListId(tweet: Tweet): Option[TrustedFriendsListId] = - tweet.trustedFriendsControl.map(_.trustedFriendsListId) - - def forTweet( - tweet: Tweet, - viewerId: Option[Long] - ): FeatureMapBuilder => FeatureMapBuilder = { - _.withConstantFeature( - TweetIsTrustedFriendTweet, - tweet.trustedFriendsControl.isDefined - ).withFeature( - ViewerIsTrustedFriendTweetAuthor, - viewerIsTrustedFriendListOwner(tweet, viewerId) - ).withFeature( - ViewerIsTrustedFriendOfTweetAuthor, - viewerIsTrustedFriend(tweet, viewerId) - ) - } - - def forTweetOnly(tweet: Tweet): FeatureMapBuilder => FeatureMapBuilder = { - _.withConstantFeature(TweetIsTrustedFriendTweet, tweet.trustedFriendsControl.isDefined) - } - -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetFeatures.docx new file mode 100644 index 000000000..2a264ab09 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetFeatures.scala deleted file mode 100644 index e28f0bda8..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetFeatures.scala +++ /dev/null @@ -1,210 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.CollabControl -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.util.Duration -import com.twitter.util.Time -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.SafetyLabelMapSource -import com.twitter.visibility.common.TweetId -import com.twitter.visibility.common.UserId -import com.twitter.visibility.features._ -import com.twitter.visibility.models.SemanticCoreAnnotation -import com.twitter.visibility.models.TweetSafetyLabel - -object TweetFeatures { - - def FALLBACK_TIMESTAMP: Time = Time.epoch - - def tweetIsSelfReply(tweet: Tweet): Boolean = { - tweet.coreData match { - case Some(coreData) => - coreData.reply match { - case Some(reply) => - reply.inReplyToUserId == coreData.userId - - case None => - false - } - - case None => - false - } - } - - def tweetReplyToParentTweetDuration(tweet: Tweet): Option[Duration] = for { - coreData <- tweet.coreData - reply <- coreData.reply - inReplyToStatusId <- reply.inReplyToStatusId - replyTime <- SnowflakeId.timeFromIdOpt(tweet.id) - repliedToTime <- SnowflakeId.timeFromIdOpt(inReplyToStatusId) - } yield { - replyTime.diff(repliedToTime) - } - - def tweetReplyToRootTweetDuration(tweet: Tweet): Option[Duration] = for { - coreData <- tweet.coreData - if coreData.reply.isDefined - conversationId <- coreData.conversationId - replyTime <- SnowflakeId.timeFromIdOpt(tweet.id) - rootTime <- SnowflakeId.timeFromIdOpt(conversationId) - } yield { - replyTime.diff(rootTime) - } - - def tweetTimestamp(tweetId: Long): Time = - SnowflakeId.timeFromIdOpt(tweetId).getOrElse(FALLBACK_TIMESTAMP) - - def tweetSemanticCoreAnnotations(tweet: Tweet): Seq[SemanticCoreAnnotation] = { - tweet.escherbirdEntityAnnotations - .map(a => - a.entityAnnotations.map { annotation => - SemanticCoreAnnotation( - annotation.groupId, - annotation.domainId, - annotation.entityId - ) - }).toSeq.flatten - } - - def tweetIsNullcast(tweet: Tweet): Boolean = { - tweet.coreData match { - case Some(coreData) => - coreData.nullcast - case None => - false - } - } - - def tweetAuthorUserId(tweet: Tweet): Option[UserId] = { - tweet.coreData.map(_.userId) - } -} - -sealed trait TweetLabels { - def forTweet(tweet: Tweet): Stitch[Seq[TweetSafetyLabel]] - def forTweetId(tweetId: TweetId): Stitch[Seq[TweetSafetyLabel]] -} - -class StratoTweetLabelMaps(safetyLabelSource: SafetyLabelMapSource) extends TweetLabels { - - override def forTweet(tweet: Tweet): Stitch[Seq[TweetSafetyLabel]] = { - forTweetId(tweet.id) - } - - def forTweetId(tweetId: TweetId): Stitch[Seq[TweetSafetyLabel]] = { - safetyLabelSource - .fetch(tweetId).map( - _.map( - _.labels - .map( - _.map(sl => TweetSafetyLabel.fromTuple(sl._1, sl._2)).toSeq - ).getOrElse(Seq()) - ).getOrElse(Seq())) - } -} - -object NilTweetLabelMaps extends TweetLabels { - override def forTweet(tweet: Tweet): Stitch[Seq[TweetSafetyLabel]] = Stitch.Nil - override def forTweetId(tweetId: TweetId): Stitch[Seq[TweetSafetyLabel]] = Stitch.Nil -} - -class TweetFeatures(tweetLabels: TweetLabels, statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("tweet_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - private[this] val tweetSafetyLabels = - scopedStatsReceiver.scope(TweetSafetyLabels.name).counter("requests") - private[this] val tweetTakedownReasons = - scopedStatsReceiver.scope(TweetTakedownReasons.name).counter("requests") - private[this] val tweetIsSelfReply = - scopedStatsReceiver.scope(TweetIsSelfReply.name).counter("requests") - private[this] val tweetTimestamp = - scopedStatsReceiver.scope(TweetTimestamp.name).counter("requests") - private[this] val tweetReplyToParentTweetDuration = - scopedStatsReceiver.scope(TweetReplyToParentTweetDuration.name).counter("requests") - private[this] val tweetReplyToRootTweetDuration = - scopedStatsReceiver.scope(TweetReplyToRootTweetDuration.name).counter("requests") - private[this] val tweetSemanticCoreAnnotations = - scopedStatsReceiver.scope(TweetSemanticCoreAnnotations.name).counter("requests") - private[this] val tweetId = - scopedStatsReceiver.scope(TweetId.name).counter("requests") - private[this] val tweetHasNsfwUser = - scopedStatsReceiver.scope(TweetHasNsfwUser.name).counter("requests") - private[this] val tweetHasNsfwAdmin = - scopedStatsReceiver.scope(TweetHasNsfwAdmin.name).counter("requests") - private[this] val tweetIsNullcast = - scopedStatsReceiver.scope(TweetIsNullcast.name).counter("requests") - private[this] val tweetHasMedia = - scopedStatsReceiver.scope(TweetHasMedia.name).counter("requests") - private[this] val tweetIsCommunity = - scopedStatsReceiver.scope(TweetIsCommunityTweet.name).counter("requests") - private[this] val tweetIsCollabInvitation = - scopedStatsReceiver.scope(TweetIsCollabInvitationTweet.name).counter("requests") - - def forTweet(tweet: Tweet): FeatureMapBuilder => FeatureMapBuilder = { - forTweetWithoutSafetyLabels(tweet) - .andThen(_.withFeature(TweetSafetyLabels, tweetLabels.forTweet(tweet))) - } - - def forTweetWithoutSafetyLabels(tweet: Tweet): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - tweetTakedownReasons.incr() - tweetIsSelfReply.incr() - tweetTimestamp.incr() - tweetReplyToParentTweetDuration.incr() - tweetReplyToRootTweetDuration.incr() - tweetSemanticCoreAnnotations.incr() - tweetId.incr() - tweetHasNsfwUser.incr() - tweetHasNsfwAdmin.incr() - tweetIsNullcast.incr() - tweetHasMedia.incr() - tweetIsCommunity.incr() - tweetIsCollabInvitation.incr() - - _.withConstantFeature(TweetTakedownReasons, tweet.takedownReasons.getOrElse(Seq.empty)) - .withConstantFeature(TweetIsSelfReply, TweetFeatures.tweetIsSelfReply(tweet)) - .withConstantFeature(TweetTimestamp, TweetFeatures.tweetTimestamp(tweet.id)) - .withConstantFeature( - TweetReplyToParentTweetDuration, - TweetFeatures.tweetReplyToParentTweetDuration(tweet)) - .withConstantFeature( - TweetReplyToRootTweetDuration, - TweetFeatures.tweetReplyToRootTweetDuration(tweet)) - .withConstantFeature( - TweetSemanticCoreAnnotations, - TweetFeatures.tweetSemanticCoreAnnotations(tweet)) - .withConstantFeature(TweetId, tweet.id) - .withConstantFeature(TweetHasNsfwUser, tweetHasNsfwUser(tweet)) - .withConstantFeature(TweetHasNsfwAdmin, tweetHasNsfwAdmin(tweet)) - .withConstantFeature(TweetIsNullcast, TweetFeatures.tweetIsNullcast(tweet)) - .withConstantFeature(TweetHasMedia, tweetHasMedia(tweet)) - .withConstantFeature(TweetIsCommunityTweet, tweetHasCommunity(tweet)) - .withConstantFeature(TweetIsCollabInvitationTweet, tweetIsCollabInvitation(tweet)) - } - - def tweetHasNsfwUser(tweet: Tweet): Boolean = - tweet.coreData.exists(_.nsfwUser) - - def tweetHasNsfwAdmin(tweet: Tweet): Boolean = - tweet.coreData.exists(_.nsfwAdmin) - - def tweetHasMedia(tweet: Tweet): Boolean = - tweet.coreData.exists(_.hasMedia.getOrElse(false)) - - def tweetHasCommunity(tweet: Tweet): Boolean = { - tweet.communities.exists(_.communityIds.nonEmpty) - } - - def tweetIsCollabInvitation(tweet: Tweet): Boolean = { - tweet.collabControl.exists(_ match { - case CollabControl.CollabInvitation(_) => true - case _ => false - }) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetIdFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetIdFeatures.docx new file mode 100644 index 000000000..142eeb0cd Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetIdFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetIdFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetIdFeatures.scala deleted file mode 100644 index b284b4ab6..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetIdFeatures.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.Gate -import com.twitter.spam.rtf.thriftscala.SafetyLabel -import com.twitter.spam.rtf.thriftscala.SafetyLabelType -import com.twitter.spam.rtf.thriftscala.SafetyLabelValue -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.stitch.StitchHelpers -import com.twitter.visibility.features.TweetId -import com.twitter.visibility.features.TweetSafetyLabels -import com.twitter.visibility.features.TweetTimestamp -import com.twitter.visibility.models.TweetSafetyLabel - -class TweetIdFeatures( - statsReceiver: StatsReceiver, - enableStitchProfiling: Gate[Unit]) { - private[this] val scopedStatsReceiver: StatsReceiver = statsReceiver.scope("tweet_id_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - private[this] val tweetSafetyLabels = - scopedStatsReceiver.scope(TweetSafetyLabels.name).counter("requests") - private[this] val tweetTimestamp = - scopedStatsReceiver.scope(TweetTimestamp.name).counter("requests") - - private[this] val labelFetchScope: StatsReceiver = - scopedStatsReceiver.scope("labelFetch") - - private[this] def getTweetLabels( - tweetId: Long, - labelFetcher: Long => Stitch[Map[SafetyLabelType, SafetyLabel]] - ): Stitch[Seq[TweetSafetyLabel]] = { - val stitch = - labelFetcher(tweetId).map { labelMap => - labelMap - .map { case (labelType, label) => SafetyLabelValue(labelType, label) }.toSeq - .map(TweetSafetyLabel.fromThrift) - } - - if (enableStitchProfiling()) { - StitchHelpers.profileStitch( - stitch, - Seq(labelFetchScope) - ) - } else { - stitch - } - } - - def forTweetId( - tweetId: Long, - labelFetcher: Long => Stitch[Map[SafetyLabelType, SafetyLabel]] - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - tweetSafetyLabels.incr() - tweetTimestamp.incr() - - _.withFeature(TweetSafetyLabels, getTweetLabels(tweetId, labelFetcher)) - .withConstantFeature(TweetTimestamp, TweetFeatures.tweetTimestamp(tweetId)) - .withConstantFeature(TweetId, tweetId) - } - - def forTweetId( - tweetId: Long, - constantTweetSafetyLabels: Seq[TweetSafetyLabel] - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - tweetSafetyLabels.incr() - tweetTimestamp.incr() - - _.withConstantFeature(TweetSafetyLabels, constantTweetSafetyLabels) - .withConstantFeature(TweetTimestamp, TweetFeatures.tweetTimestamp(tweetId)) - .withConstantFeature(TweetId, tweetId) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetMediaMetadataFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetMediaMetadataFeatures.docx new file mode 100644 index 000000000..4c6b92c1a Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetMediaMetadataFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetMediaMetadataFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetMediaMetadataFeatures.scala deleted file mode 100644 index e421bd7b6..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetMediaMetadataFeatures.scala +++ /dev/null @@ -1,130 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.mediaservices.commons.mediainformation.thriftscala.AdditionalMetadata -import com.twitter.mediaservices.media_util.GenericMediaKey -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.TweetMediaMetadataSource -import com.twitter.visibility.features.HasDmcaMediaFeature -import com.twitter.visibility.features.MediaGeoRestrictionsAllowList -import com.twitter.visibility.features.MediaGeoRestrictionsDenyList - -class TweetMediaMetadataFeatures( - mediaMetadataSource: TweetMediaMetadataSource, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("tweet_media_metadata_features") - private[this] val reportedStats = scopedStatsReceiver.scope("dmcaStats") - - def forTweet( - tweet: Tweet, - mediaKeys: Seq[GenericMediaKey], - enableFetchMediaMetadata: Boolean - ): FeatureMapBuilder => FeatureMapBuilder = { featureMapBuilder => - featureMapBuilder.withFeature( - HasDmcaMediaFeature, - mediaIsDmca(tweet, mediaKeys, enableFetchMediaMetadata)) - featureMapBuilder.withFeature( - MediaGeoRestrictionsAllowList, - allowlist(tweet, mediaKeys, enableFetchMediaMetadata)) - featureMapBuilder.withFeature( - MediaGeoRestrictionsDenyList, - denylist(tweet, mediaKeys, enableFetchMediaMetadata)) - } - - private def mediaIsDmca( - tweet: Tweet, - mediaKeys: Seq[GenericMediaKey], - enableFetchMediaMetadata: Boolean - ) = getMediaAdditionalMetadata(tweet, mediaKeys, enableFetchMediaMetadata) - .map(_.exists(_.restrictions.exists(_.isDmca))) - - private def allowlist( - tweet: Tweet, - mediaKeys: Seq[GenericMediaKey], - enableFetchMediaMetadata: Boolean - ) = getMediaGeoRestrictions(tweet, mediaKeys, enableFetchMediaMetadata) - .map(_.flatMap(_.whitelistedCountryCodes)) - - private def denylist( - tweet: Tweet, - mediaKeys: Seq[GenericMediaKey], - enableFetchMediaMetadata: Boolean - ) = getMediaGeoRestrictions(tweet, mediaKeys, enableFetchMediaMetadata) - .map(_.flatMap(_.blacklistedCountryCodes)) - - private def getMediaGeoRestrictions( - tweet: Tweet, - mediaKeys: Seq[GenericMediaKey], - enableFetchMediaMetadata: Boolean - ) = { - getMediaAdditionalMetadata(tweet, mediaKeys, enableFetchMediaMetadata) - .map(additionalMetadatasSeq => { - for { - additionalMetadata <- additionalMetadatasSeq - restrictions <- additionalMetadata.restrictions - geoRestrictions <- restrictions.geoRestrictions - } yield { - geoRestrictions - } - }) - } - - private def getMediaAdditionalMetadata( - tweet: Tweet, - mediaKeys: Seq[GenericMediaKey], - enableFetchMediaMetadata: Boolean - ): Stitch[Seq[AdditionalMetadata]] = { - if (mediaKeys.isEmpty) { - reportedStats.counter("empty").incr() - Stitch.value(Seq.empty) - } else { - tweet.media.flatMap { mediaEntities => - val alreadyHydratedMetadata = mediaEntities - .filter(_.mediaKey.isDefined) - .flatMap(_.additionalMetadata) - - if (alreadyHydratedMetadata.nonEmpty) { - Some(alreadyHydratedMetadata) - } else { - None - } - } match { - case Some(additionalMetadata) => - reportedStats.counter("already_hydrated").incr() - Stitch.value(additionalMetadata) - case None => - Stitch - .collect( - mediaKeys.map(fetchAdditionalMetadata(tweet.id, _, enableFetchMediaMetadata)) - ).map(maybeMetadatas => { - maybeMetadatas - .filter(_.isDefined) - .map(_.get) - }) - } - } - } - - private def fetchAdditionalMetadata( - tweetId: Long, - genericMediaKey: GenericMediaKey, - enableFetchMediaMetadata: Boolean - ): Stitch[Option[AdditionalMetadata]] = - if (enableFetchMediaMetadata) { - genericMediaKey.toThriftMediaKey() match { - case Some(mediaKey) => - reportedStats.counter("request").incr() - mediaMetadataSource.fetch(tweetId, mediaKey) - case None => - reportedStats.counter("empty_key").incr() - Stitch.None - } - } else { - reportedStats.counter("light_request").incr() - Stitch.None - } - -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetPerspectiveFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetPerspectiveFeatures.docx new file mode 100644 index 000000000..38d44b1d4 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetPerspectiveFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetPerspectiveFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetPerspectiveFeatures.scala deleted file mode 100644 index e2e25740b..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetPerspectiveFeatures.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.TweetPerspectiveSource -import com.twitter.visibility.features.ViewerReportedTweet - -class TweetPerspectiveFeatures( - tweetPerspectiveSource: TweetPerspectiveSource, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("tweet_perspective_features") - private[this] val reportedStats = scopedStatsReceiver.scope("reported") - - def forTweet( - tweet: Tweet, - viewerId: Option[Long], - enableFetchReportedPerspective: Boolean - ): FeatureMapBuilder => FeatureMapBuilder = - _.withFeature( - ViewerReportedTweet, - tweetIsReported(tweet, viewerId, enableFetchReportedPerspective)) - - private[builder] def tweetIsReported( - tweet: Tweet, - viewerId: Option[Long], - enableFetchReportedPerspective: Boolean = true - ): Stitch[Boolean] = { - ((tweet.perspective, viewerId) match { - case (Some(perspective), _) => - Stitch.value(perspective.reported).onSuccess { _ => - reportedStats.counter("already_hydrated").incr() - } - case (None, Some(viewerId)) => - if (enableFetchReportedPerspective) { - tweetPerspectiveSource.reported(tweet.id, viewerId).onSuccess { _ => - reportedStats.counter("request").incr() - } - } else { - Stitch.False.onSuccess { _ => - reportedStats.counter("light_request").incr() - } - } - case _ => - Stitch.False.onSuccess { _ => - reportedStats.counter("empty").incr() - } - }).onSuccess { _ => - reportedStats.counter("").incr() - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetVisibilityNudgeSourceWrapper.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetVisibilityNudgeSourceWrapper.docx new file mode 100644 index 000000000..644367e26 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetVisibilityNudgeSourceWrapper.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetVisibilityNudgeSourceWrapper.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetVisibilityNudgeSourceWrapper.scala deleted file mode 100644 index 99ad6a46a..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/TweetVisibilityNudgeSourceWrapper.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.spam.rtf.thriftscala.SafetyLabelType -import com.twitter.spam.rtf.thriftscala.SafetyLabelType.ExperimentalNudge -import com.twitter.spam.rtf.thriftscala.SafetyLabelType.SemanticCoreMisinformation -import com.twitter.spam.rtf.thriftscala.SafetyLabelType.UnsafeUrl -import com.twitter.visibility.common.LocalizedNudgeSource -import com.twitter.visibility.common.actions.TweetVisibilityNudgeReason -import com.twitter.visibility.common.actions.TweetVisibilityNudgeReason.ExperimentalNudgeSafetyLabelReason -import com.twitter.visibility.common.actions.TweetVisibilityNudgeReason.SemanticCoreMisinformationLabelReason -import com.twitter.visibility.common.actions.TweetVisibilityNudgeReason.UnsafeURLLabelReason -import com.twitter.visibility.rules.LocalizedNudge - -class TweetVisibilityNudgeSourceWrapper(localizedNudgeSource: LocalizedNudgeSource) { - - def getLocalizedNudge( - reason: TweetVisibilityNudgeReason, - languageCode: String, - countryCode: Option[String] - ): Option[LocalizedNudge] = - reason match { - case ExperimentalNudgeSafetyLabelReason => - fetchNudge(ExperimentalNudge, languageCode, countryCode) - case SemanticCoreMisinformationLabelReason => - fetchNudge(SemanticCoreMisinformation, languageCode, countryCode) - case UnsafeURLLabelReason => - fetchNudge(UnsafeUrl, languageCode, countryCode) - } - - private def fetchNudge( - safetyLabel: SafetyLabelType, - languageCode: String, - countryCode: Option[String] - ): Option[LocalizedNudge] = { - localizedNudgeSource - .fetch(safetyLabel, languageCode, countryCode) - .map(LocalizedNudge.fromStratoThrift) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/UnmentionNotificationFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/UnmentionNotificationFeatures.docx new file mode 100644 index 000000000..e4b99b81a Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/UnmentionNotificationFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/UnmentionNotificationFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/UnmentionNotificationFeatures.scala deleted file mode 100644 index 513627009..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/tweets/UnmentionNotificationFeatures.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.visibility.builder.tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.notificationservice.model.notification._ -import com.twitter.servo.util.Gate -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.SettingsUnmentions -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.TweetSource -import com.twitter.visibility.features.NotificationIsOnUnmentionedViewer - -object UnmentionNotificationFeatures { - def ForNonUnmentionNotificationFeatures: FeatureMapBuilder => FeatureMapBuilder = { - _.withConstantFeature(NotificationIsOnUnmentionedViewer, false) - } -} - -class UnmentionNotificationFeatures( - tweetSource: TweetSource, - enableUnmentionHydration: Gate[Long], - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = - statsReceiver.scope("unmention_notification_features") - private[this] val requestsCounter = scopedStatsReceiver.counter("requests") - private[this] val hydrationsCounter = scopedStatsReceiver.counter("hydrations") - private[this] val notificationsUnmentionUserCounter = - scopedStatsReceiver - .scope(NotificationIsOnUnmentionedViewer.name).counter("unmentioned_users") - - def forNotification(notification: Notification): FeatureMapBuilder => FeatureMapBuilder = { - requestsCounter.incr() - - val isUnmentionNotification = tweetId(notification) match { - case Some(tweetId) if enableUnmentionHydration(notification.target) => - hydrationsCounter.incr() - tweetSource - .getTweet(tweetId) - .map { - case Some(tweet) => - tweet.settingsUnmentions match { - case Some(SettingsUnmentions(Some(unmentionedUserIds))) => - if (unmentionedUserIds.contains(notification.target)) { - notificationsUnmentionUserCounter.incr() - true - } else { - false - } - case _ => false - } - case _ => false - } - case _ => Stitch.False - } - _.withFeature(NotificationIsOnUnmentionedViewer, isUnmentionNotification) - } - - private[this] def tweetId(notification: Notification): Option[Long] = { - notification match { - case n: MentionNotification => Some(n.mentioningTweetId) - case n: FavoritedMentioningTweetNotification => Some(n.mentioningTweetId) - case n: FavoritedReplyToYourTweetNotification => Some(n.replyTweetId) - case n: MentionQuoteNotification => Some(n.mentioningTweetId) - case n: ReactionMentioningTweetNotification => Some(n.mentioningTweetId) - case n: ReplyNotification => Some(n.replyingTweetId) - case n: RetweetedMentionNotification => Some(n.mentioningTweetId) - case n: RetweetedReplyToYourTweetNotification => Some(n.replyTweetId) - case n: ReplyToConversationNotification => Some(n.replyingTweetId) - case n: ReactionReplyToYourTweetNotification => Some(n.replyTweetId) - case _ => None - } - - } - -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorDeviceFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorDeviceFeatures.docx new file mode 100644 index 000000000..1a34d6220 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorDeviceFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorDeviceFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorDeviceFeatures.scala deleted file mode 100644 index 5ffa3c107..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorDeviceFeatures.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.UserDeviceSource -import com.twitter.visibility.features.AuthorHasConfirmedEmail -import com.twitter.visibility.features.AuthorHasVerifiedPhone - -class AuthorDeviceFeatures(userDeviceSource: UserDeviceSource, statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("author_device_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val authorHasConfirmedEmailRequests = - scopedStatsReceiver.scope(AuthorHasConfirmedEmail.name).counter("requests") - private[this] val authorHasVerifiedPhoneRequests = - scopedStatsReceiver.scope(AuthorHasVerifiedPhone.name).counter("requests") - - def forAuthor(author: User): FeatureMapBuilder => FeatureMapBuilder = forAuthorId(author.id) - - def forAuthorId(authorId: Long): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - _.withFeature(AuthorHasConfirmedEmail, authorHasConfirmedEmail(authorId)) - .withFeature(AuthorHasVerifiedPhone, authorHasVerifiedPhone(authorId)) - } - - def authorHasConfirmedEmail(authorId: Long): Stitch[Boolean] = { - authorHasConfirmedEmailRequests.incr() - userDeviceSource.hasConfirmedEmail(authorId) - } - - def authorHasVerifiedPhone(authorId: Long): Stitch[Boolean] = { - authorHasVerifiedPhoneRequests.incr() - userDeviceSource.hasConfirmedPhone(authorId) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorFeatures.docx new file mode 100644 index 000000000..9b83ccb45 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorFeatures.scala deleted file mode 100644 index bf6529691..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/AuthorFeatures.scala +++ /dev/null @@ -1,221 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.Label -import com.twitter.gizmoduck.thriftscala.Labels -import com.twitter.gizmoduck.thriftscala.Profile -import com.twitter.gizmoduck.thriftscala.Safety -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tseng.withholding.thriftscala.TakedownReason -import com.twitter.util.Duration -import com.twitter.util.Time -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.features._ - -class AuthorFeatures(userSource: UserSource, statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("author_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val authorUserLabels = - scopedStatsReceiver.scope(AuthorUserLabels.name).counter("requests") - private[this] val authorIsSuspended = - scopedStatsReceiver.scope(AuthorIsSuspended.name).counter("requests") - private[this] val authorIsProtected = - scopedStatsReceiver.scope(AuthorIsProtected.name).counter("requests") - private[this] val authorIsDeactivated = - scopedStatsReceiver.scope(AuthorIsDeactivated.name).counter("requests") - private[this] val authorIsErased = - scopedStatsReceiver.scope(AuthorIsErased.name).counter("requests") - private[this] val authorIsOffboarded = - scopedStatsReceiver.scope(AuthorIsOffboarded.name).counter("requests") - private[this] val authorIsNsfwUser = - scopedStatsReceiver.scope(AuthorIsNsfwUser.name).counter("requests") - private[this] val authorIsNsfwAdmin = - scopedStatsReceiver.scope(AuthorIsNsfwAdmin.name).counter("requests") - private[this] val authorTakedownReasons = - scopedStatsReceiver.scope(AuthorTakedownReasons.name).counter("requests") - private[this] val authorHasDefaultProfileImage = - scopedStatsReceiver.scope(AuthorHasDefaultProfileImage.name).counter("requests") - private[this] val authorAccountAge = - scopedStatsReceiver.scope(AuthorAccountAge.name).counter("requests") - private[this] val authorIsVerified = - scopedStatsReceiver.scope(AuthorIsVerified.name).counter("requests") - private[this] val authorScreenName = - scopedStatsReceiver.scope(AuthorScreenName.name).counter("requests") - private[this] val authorIsBlueVerified = - scopedStatsReceiver.scope(AuthorIsBlueVerified.name).counter("requests") - - def forAuthor(author: User): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - _.withConstantFeature(AuthorId, Set(author.id)) - .withConstantFeature(AuthorUserLabels, authorUserLabels(author)) - .withConstantFeature(AuthorIsProtected, authorIsProtected(author)) - .withConstantFeature(AuthorIsSuspended, authorIsSuspended(author)) - .withConstantFeature(AuthorIsDeactivated, authorIsDeactivated(author)) - .withConstantFeature(AuthorIsErased, authorIsErased(author)) - .withConstantFeature(AuthorIsOffboarded, authorIsOffboarded(author)) - .withConstantFeature(AuthorTakedownReasons, authorTakedownReasons(author)) - .withConstantFeature(AuthorHasDefaultProfileImage, authorHasDefaultProfileImage(author)) - .withConstantFeature(AuthorAccountAge, authorAccountAge(author)) - .withConstantFeature(AuthorIsNsfwUser, authorIsNsfwUser(author)) - .withConstantFeature(AuthorIsNsfwAdmin, authorIsNsfwAdmin(author)) - .withConstantFeature(AuthorIsVerified, authorIsVerified(author)) - .withConstantFeature(AuthorScreenName, authorScreenName(author)) - .withConstantFeature(AuthorIsBlueVerified, authorIsBlueVerified(author)) - } - - def forAuthorNoDefaults(author: User): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - _.withConstantFeature(AuthorId, Set(author.id)) - .withConstantFeature(AuthorUserLabels, authorUserLabelsOpt(author)) - .withConstantFeature(AuthorIsProtected, authorIsProtectedOpt(author)) - .withConstantFeature(AuthorIsSuspended, authorIsSuspendedOpt(author)) - .withConstantFeature(AuthorIsDeactivated, authorIsDeactivatedOpt(author)) - .withConstantFeature(AuthorIsErased, authorIsErasedOpt(author)) - .withConstantFeature(AuthorIsOffboarded, authorIsOffboarded(author)) - .withConstantFeature(AuthorTakedownReasons, authorTakedownReasons(author)) - .withConstantFeature(AuthorHasDefaultProfileImage, authorHasDefaultProfileImage(author)) - .withConstantFeature(AuthorAccountAge, authorAccountAge(author)) - .withConstantFeature(AuthorIsNsfwUser, authorIsNsfwUserOpt(author)) - .withConstantFeature(AuthorIsNsfwAdmin, authorIsNsfwAdminOpt(author)) - .withConstantFeature(AuthorIsVerified, authorIsVerifiedOpt(author)) - .withConstantFeature(AuthorScreenName, authorScreenName(author)) - .withConstantFeature(AuthorIsBlueVerified, authorIsBlueVerified(author)) - } - - def forAuthorId(authorId: Long): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - _.withConstantFeature(AuthorId, Set(authorId)) - .withFeature(AuthorUserLabels, authorUserLabels(authorId)) - .withFeature(AuthorIsProtected, authorIsProtected(authorId)) - .withFeature(AuthorIsSuspended, authorIsSuspended(authorId)) - .withFeature(AuthorIsDeactivated, authorIsDeactivated(authorId)) - .withFeature(AuthorIsErased, authorIsErased(authorId)) - .withFeature(AuthorIsOffboarded, authorIsOffboarded(authorId)) - .withFeature(AuthorTakedownReasons, authorTakedownReasons(authorId)) - .withFeature(AuthorHasDefaultProfileImage, authorHasDefaultProfileImage(authorId)) - .withFeature(AuthorAccountAge, authorAccountAge(authorId)) - .withFeature(AuthorIsNsfwUser, authorIsNsfwUser(authorId)) - .withFeature(AuthorIsNsfwAdmin, authorIsNsfwAdmin(authorId)) - .withFeature(AuthorIsVerified, authorIsVerified(authorId)) - .withFeature(AuthorScreenName, authorScreenName(authorId)) - .withFeature(AuthorIsBlueVerified, authorIsBlueVerified(authorId)) - } - - def forNoAuthor(): FeatureMapBuilder => FeatureMapBuilder = { - _.withConstantFeature(AuthorId, Set.empty[Long]) - .withConstantFeature(AuthorUserLabels, Seq.empty) - .withConstantFeature(AuthorIsProtected, false) - .withConstantFeature(AuthorIsSuspended, false) - .withConstantFeature(AuthorIsDeactivated, false) - .withConstantFeature(AuthorIsErased, false) - .withConstantFeature(AuthorIsOffboarded, false) - .withConstantFeature(AuthorTakedownReasons, Seq.empty) - .withConstantFeature(AuthorHasDefaultProfileImage, false) - .withConstantFeature(AuthorAccountAge, Duration.Zero) - .withConstantFeature(AuthorIsNsfwUser, false) - .withConstantFeature(AuthorIsNsfwAdmin, false) - .withConstantFeature(AuthorIsVerified, false) - .withConstantFeature(AuthorIsBlueVerified, false) - } - - def authorUserLabels(author: User): Seq[Label] = - authorUserLabels(author.labels) - - def authorIsSuspended(authorId: Long): Stitch[Boolean] = - userSource.getSafety(authorId).map(safety => authorIsSuspended(Some(safety))) - - def authorIsSuspendedOpt(author: User): Option[Boolean] = { - authorIsSuspended.incr() - author.safety.map(_.suspended) - } - - private def authorIsSuspended(safety: Option[Safety]): Boolean = { - authorIsSuspended.incr() - safety.exists(_.suspended) - } - - def authorIsProtected(author: User): Boolean = - authorIsProtected(author.safety) - - def authorIsDeactivated(authorId: Long): Stitch[Boolean] = - userSource.getSafety(authorId).map(safety => authorIsDeactivated(Some(safety))) - - def authorIsDeactivatedOpt(author: User): Option[Boolean] = { - authorIsDeactivated.incr() - author.safety.map(_.deactivated) - } - - private def authorIsDeactivated(safety: Option[Safety]): Boolean = { - authorIsDeactivated.incr() - safety.exists(_.deactivated) - } - - def authorIsErased(author: User): Boolean = { - authorIsErased.incr() - author.safety.exists(_.erased) - } - - def authorIsOffboarded(authorId: Long): Stitch[Boolean] = { - userSource.getSafety(authorId).map(safety => authorIsOffboarded(Some(safety))) - } - - def authorIsNsfwUser(author: User): Boolean = { - authorIsNsfwUser(author.safety) - } - - def authorIsNsfwUser(authorId: Long): Stitch[Boolean] = { - userSource.getSafety(authorId).map(safety => authorIsNsfwUser(Some(safety))) - } - - def authorIsNsfwUser(safety: Option[Safety]): Boolean = { - authorIsNsfwUser.incr() - safety.exists(_.nsfwUser) - } - - def authorIsNsfwAdminOpt(author: User): Option[Boolean] = { - authorIsNsfwAdmin.incr() - author.safety.map(_.nsfwAdmin) - } - - def authorTakedownReasons(authorId: Long): Stitch[Seq[TakedownReason]] = { - authorTakedownReasons.incr() - userSource.getTakedownReasons(authorId) - } - - def authorHasDefaultProfileImage(authorId: Long): Stitch[Boolean] = - userSource.getProfile(authorId).map(profile => authorHasDefaultProfileImage(Some(profile))) - - def authorAccountAge(authorId: Long): Stitch[Duration] = - userSource.getCreatedAtMsec(authorId).map(authorAccountAgeFromTimestamp) - - def authorIsVerified(authorId: Long): Stitch[Boolean] = - userSource.getSafety(authorId).map(safety => authorIsVerified(Some(safety))) - - def authorIsVerifiedOpt(author: User): Option[Boolean] = { - authorIsVerified.incr() - author.safety.map(_.verified) - } - - private def authorIsVerified(safety: Option[Safety]): Boolean = { - authorIsVerified.incr() - safety.exists(_.verified) - } - - def authorScreenName(author: User): Option[String] = { - authorScreenName.incr() - author.profile.map(_.screenName) - } - - def authorScreenName(authorId: Long): Stitch[String] = { - authorScreenName.incr() - userSource.getProfile(authorId).map(profile => profile.screenName) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/BUILD deleted file mode 100644 index 9da789b38..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/scala/com/twitter/search/blender/services/strato", - "src/thrift/com/twitter/content-health/sensitivemediasettings:sensitivemediasettings-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "stitch/stitch-core", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/common/src/main/scala/com/twitter/visibility/common/user_result", - "visibility/common/src/main/thrift/com/twitter/visibility:action-scala", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/blender", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/search", - "visibility/lib/src/main/thrift/com/twitter/visibility/context:vf-context-scala", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/BUILD.docx new file mode 100644 index 000000000..7e276b73e Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/QuotedTweetFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/QuotedTweetFeatures.docx new file mode 100644 index 000000000..03c06af5f Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/QuotedTweetFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/QuotedTweetFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/QuotedTweetFeatures.scala deleted file mode 100644 index aac96d26f..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/QuotedTweetFeatures.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.features.AuthorBlocksOuterAuthor -import com.twitter.visibility.features.OuterAuthorFollowsAuthor -import com.twitter.visibility.features.OuterAuthorId -import com.twitter.visibility.features.OuterAuthorIsInnerAuthor - -class QuotedTweetFeatures( - relationshipFeatures: RelationshipFeatures, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("quoted_tweet_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val outerAuthorIdStat = - scopedStatsReceiver.scope(OuterAuthorId.name).counter("requests") - private[this] val authorBlocksOuterAuthor = - scopedStatsReceiver.scope(AuthorBlocksOuterAuthor.name).counter("requests") - private[this] val outerAuthorFollowsAuthor = - scopedStatsReceiver.scope(OuterAuthorFollowsAuthor.name).counter("requests") - private[this] val outerAuthorIsInnerAuthor = - scopedStatsReceiver.scope(OuterAuthorIsInnerAuthor.name).counter("requests") - - def forOuterAuthor( - outerAuthorId: Long, - innerAuthorId: Long - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - outerAuthorIdStat.incr() - authorBlocksOuterAuthor.incr() - outerAuthorFollowsAuthor.incr() - outerAuthorIsInnerAuthor.incr() - - val viewer = Some(outerAuthorId) - - _.withConstantFeature(OuterAuthorId, outerAuthorId) - .withFeature( - AuthorBlocksOuterAuthor, - relationshipFeatures.authorBlocksViewer(innerAuthorId, viewer)) - .withFeature( - OuterAuthorFollowsAuthor, - relationshipFeatures.viewerFollowsAuthor(innerAuthorId, viewer)) - .withConstantFeature( - OuterAuthorIsInnerAuthor, - innerAuthorId == outerAuthorId - ) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipFeatures.docx new file mode 100644 index 000000000..670a1aaa9 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipFeatures.scala deleted file mode 100644 index 9795e9408..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipFeatures.scala +++ /dev/null @@ -1,176 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.UserId -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.features._ - -class RelationshipFeatures( - userRelationshipSource: UserRelationshipSource, - statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("relationship_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val authorFollowsViewer = - scopedStatsReceiver.scope(AuthorFollowsViewer.name).counter("requests") - private[this] val viewerFollowsAuthor = - scopedStatsReceiver.scope(ViewerFollowsAuthor.name).counter("requests") - private[this] val authorBlocksViewer = - scopedStatsReceiver.scope(AuthorBlocksViewer.name).counter("requests") - private[this] val viewerBlocksAuthor = - scopedStatsReceiver.scope(ViewerBlocksAuthor.name).counter("requests") - private[this] val authorMutesViewer = - scopedStatsReceiver.scope(AuthorMutesViewer.name).counter("requests") - private[this] val viewerMutesAuthor = - scopedStatsReceiver.scope(ViewerMutesAuthor.name).counter("requests") - private[this] val authorHasReportedViewer = - scopedStatsReceiver.scope(AuthorReportsViewerAsSpam.name).counter("requests") - private[this] val viewerHasReportedAuthor = - scopedStatsReceiver.scope(ViewerReportsAuthorAsSpam.name).counter("requests") - private[this] val viewerMutesRetweetsFromAuthor = - scopedStatsReceiver.scope(ViewerMutesRetweetsFromAuthor.name).counter("requests") - - def forAuthorId( - authorId: Long, - viewerId: Option[Long] - ): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - _.withFeature(AuthorFollowsViewer, authorFollowsViewer(authorId, viewerId)) - .withFeature(ViewerFollowsAuthor, viewerFollowsAuthor(authorId, viewerId)) - .withFeature(AuthorBlocksViewer, authorBlocksViewer(authorId, viewerId)) - .withFeature(ViewerBlocksAuthor, viewerBlocksAuthor(authorId, viewerId)) - .withFeature(AuthorMutesViewer, authorMutesViewer(authorId, viewerId)) - .withFeature(ViewerMutesAuthor, viewerMutesAuthor(authorId, viewerId)) - .withFeature(AuthorReportsViewerAsSpam, authorHasReportedViewer(authorId, viewerId)) - .withFeature(ViewerReportsAuthorAsSpam, viewerHasReportedAuthor(authorId, viewerId)) - .withFeature(ViewerMutesRetweetsFromAuthor, viewerMutesRetweetsFromAuthor(authorId, viewerId)) - } - - def forNoAuthor(): FeatureMapBuilder => FeatureMapBuilder = { - _.withConstantFeature(AuthorFollowsViewer, false) - .withConstantFeature(ViewerFollowsAuthor, false) - .withConstantFeature(AuthorBlocksViewer, false) - .withConstantFeature(ViewerBlocksAuthor, false) - .withConstantFeature(AuthorMutesViewer, false) - .withConstantFeature(ViewerMutesAuthor, false) - .withConstantFeature(AuthorReportsViewerAsSpam, false) - .withConstantFeature(ViewerReportsAuthorAsSpam, false) - .withConstantFeature(ViewerMutesRetweetsFromAuthor, false) - } - - def forAuthor(author: User, viewerId: Option[Long]): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - - _.withFeature(AuthorFollowsViewer, authorFollowsViewer(author, viewerId)) - .withFeature(ViewerFollowsAuthor, viewerFollowsAuthor(author, viewerId)) - .withFeature(AuthorBlocksViewer, authorBlocksViewer(author, viewerId)) - .withFeature(ViewerBlocksAuthor, viewerBlocksAuthor(author, viewerId)) - .withFeature(AuthorMutesViewer, authorMutesViewer(author, viewerId)) - .withFeature(ViewerMutesAuthor, viewerMutesAuthor(author, viewerId)) - .withFeature(AuthorReportsViewerAsSpam, authorHasReportedViewer(author.id, viewerId)) - .withFeature(ViewerReportsAuthorAsSpam, viewerHasReportedAuthor(author.id, viewerId)) - .withFeature(ViewerMutesRetweetsFromAuthor, viewerMutesRetweetsFromAuthor(author, viewerId)) - } - - def viewerFollowsAuthor(authorId: UserId, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor(authorId, viewerId, userRelationshipSource.follows, viewerFollowsAuthor) - - def viewerFollowsAuthor(author: User, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor( - author, - viewerId, - p => p.following, - userRelationshipSource.follows, - viewerFollowsAuthor) - - def authorFollowsViewer(authorId: UserId, viewerId: Option[UserId]): Stitch[Boolean] = - AuthorVerbsViewer(authorId, viewerId, userRelationshipSource.follows, authorFollowsViewer) - - def authorFollowsViewer(author: User, viewerId: Option[UserId]): Stitch[Boolean] = - AuthorVerbsViewer( - author, - viewerId, - p => p.followedBy, - userRelationshipSource.follows, - authorFollowsViewer) - - def viewerBlocksAuthor(authorId: UserId, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor(authorId, viewerId, userRelationshipSource.blocks, viewerBlocksAuthor) - - def viewerBlocksAuthor(author: User, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor( - author, - viewerId, - p => p.blocking, - userRelationshipSource.blocks, - viewerBlocksAuthor) - - def authorBlocksViewer(authorId: UserId, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor(authorId, viewerId, userRelationshipSource.blockedBy, authorBlocksViewer) - - def authorBlocksViewer(author: User, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor( - author, - viewerId, - p => p.blockedBy, - userRelationshipSource.blockedBy, - authorBlocksViewer) - - def viewerMutesAuthor(authorId: UserId, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor(authorId, viewerId, userRelationshipSource.mutes, viewerMutesAuthor) - - def viewerMutesAuthor(author: User, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor( - author, - viewerId, - p => p.muting, - userRelationshipSource.mutes, - viewerMutesAuthor) - - def authorMutesViewer(authorId: UserId, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor(authorId, viewerId, userRelationshipSource.mutedBy, authorMutesViewer) - - def authorMutesViewer(author: User, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor( - author, - viewerId, - p => p.mutedBy, - userRelationshipSource.mutedBy, - authorMutesViewer) - - def viewerHasReportedAuthor(authorId: UserId, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor( - authorId, - viewerId, - userRelationshipSource.reportsAsSpam, - viewerHasReportedAuthor) - - def authorHasReportedViewer(authorId: UserId, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor( - authorId, - viewerId, - userRelationshipSource.reportedAsSpamBy, - authorHasReportedViewer) - - def viewerMutesRetweetsFromAuthor(authorId: UserId, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor( - authorId, - viewerId, - userRelationshipSource.noRetweetsFrom, - viewerMutesRetweetsFromAuthor) - - def viewerMutesRetweetsFromAuthor(author: User, viewerId: Option[UserId]): Stitch[Boolean] = - ViewerVerbsAuthor( - author, - viewerId, - p => p.noRetweetsFrom, - userRelationshipSource.noRetweetsFrom, - viewerMutesRetweetsFromAuthor) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipVerbHelpers.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipVerbHelpers.docx new file mode 100644 index 000000000..4f14901a7 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipVerbHelpers.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipVerbHelpers.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipVerbHelpers.scala deleted file mode 100644 index 0e654a69f..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/RelationshipVerbHelpers.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.Counter -import com.twitter.gizmoduck.thriftscala.Perspective -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.Stitch -import com.twitter.visibility.common.UserId - -case object ViewerVerbsAuthor { - def apply( - authorId: UserId, - viewerIdOpt: Option[UserId], - relationship: (UserId, UserId) => Stitch[Boolean], - relationshipCounter: Counter - ): Stitch[Boolean] = { - relationshipCounter.incr() - - viewerIdOpt match { - case Some(viewerId) => relationship(viewerId, authorId) - case _ => Stitch.False - } - } - - def apply( - author: User, - viewerId: Option[UserId], - checkPerspective: Perspective => Option[Boolean], - relationship: (UserId, UserId) => Stitch[Boolean], - relationshipCounter: Counter - ): Stitch[Boolean] = { - author.perspective match { - case Some(perspective) => - checkPerspective(perspective) match { - case Some(status) => - relationshipCounter.incr() - Stitch.value(status) - case None => - ViewerVerbsAuthor(author.id, viewerId, relationship, relationshipCounter) - } - case None => ViewerVerbsAuthor(author.id, viewerId, relationship, relationshipCounter) - } - } -} - -case object AuthorVerbsViewer { - - def apply( - authorId: UserId, - viewerIdOpt: Option[UserId], - relationship: (UserId, UserId) => Stitch[Boolean], - relationshipCounter: Counter - ): Stitch[Boolean] = { - relationshipCounter.incr() - - viewerIdOpt match { - case Some(viewerId) => relationship(authorId, viewerId) - case _ => Stitch.False - } - } - def apply( - author: User, - viewerId: Option[UserId], - checkPerspective: Perspective => Option[Boolean], - relationship: (UserId, UserId) => Stitch[Boolean], - relationshipCounter: Counter - ): Stitch[Boolean] = { - author.perspective match { - case Some(perspective) => - checkPerspective(perspective) match { - case Some(status) => - relationshipCounter.incr() - Stitch.value(status) - case None => - AuthorVerbsViewer(author.id, viewerId, relationship, relationshipCounter) - } - case None => AuthorVerbsViewer(author.id, viewerId, relationship, relationshipCounter) - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/SearchFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/SearchFeatures.docx new file mode 100644 index 000000000..920ae00ba Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/SearchFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/SearchFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/SearchFeatures.scala deleted file mode 100644 index 7602f2788..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/SearchFeatures.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.features._ -import com.twitter.visibility.context.thriftscala.SearchContext - -class SearchFeatures(statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("search_features") - private[this] val requests = scopedStatsReceiver.counter("requests") - private[this] val rawQueryCounter = - scopedStatsReceiver.scope(RawQuery.name).counter("requests") - - def forSearchContext( - searchContext: Option[SearchContext] - ): FeatureMapBuilder => FeatureMapBuilder = { builder => - requests.incr() - searchContext match { - case Some(context: SearchContext) => - rawQueryCounter.incr() - builder - .withConstantFeature(RawQuery, context.rawQuery) - case _ => builder - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/UserUnavailableFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/UserUnavailableFeatures.docx new file mode 100644 index 000000000..e3221ba4e Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/UserUnavailableFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/UserUnavailableFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/UserUnavailableFeatures.scala deleted file mode 100644 index 4a196fe5f..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/UserUnavailableFeatures.scala +++ /dev/null @@ -1,145 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.user_result.UserVisibilityResultHelper -import com.twitter.visibility.features.AuthorBlocksViewer -import com.twitter.visibility.features.AuthorIsDeactivated -import com.twitter.visibility.features.AuthorIsErased -import com.twitter.visibility.features.AuthorIsOffboarded -import com.twitter.visibility.features.AuthorIsProtected -import com.twitter.visibility.features.AuthorIsSuspended -import com.twitter.visibility.features.AuthorIsUnavailable -import com.twitter.visibility.features.ViewerBlocksAuthor -import com.twitter.visibility.features.ViewerMutesAuthor -import com.twitter.visibility.models.UserUnavailableStateEnum - -case class UserUnavailableFeatures(statsReceiver: StatsReceiver) { - - private[this] val scopedStatsReceiver = statsReceiver.scope("user_unavailable_features") - private[this] val suspendedAuthorStats = scopedStatsReceiver.scope("suspended_author") - private[this] val deactivatedAuthorStats = scopedStatsReceiver.scope("deactivated_author") - private[this] val offboardedAuthorStats = scopedStatsReceiver.scope("offboarded_author") - private[this] val erasedAuthorStats = scopedStatsReceiver.scope("erased_author") - private[this] val protectedAuthorStats = scopedStatsReceiver.scope("protected_author") - private[this] val authorBlocksViewerStats = scopedStatsReceiver.scope("author_blocks_viewer") - private[this] val viewerBlocksAuthorStats = scopedStatsReceiver.scope("viewer_blocks_author") - private[this] val viewerMutesAuthorStats = scopedStatsReceiver.scope("viewer_mutes_author") - private[this] val unavailableStats = scopedStatsReceiver.scope("unavailable") - - def forState(state: UserUnavailableStateEnum): FeatureMapBuilder => FeatureMapBuilder = { - builder => - builder - .withConstantFeature(AuthorIsSuspended, isSuspended(state)) - .withConstantFeature(AuthorIsDeactivated, isDeactivated(state)) - .withConstantFeature(AuthorIsOffboarded, isOffboarded(state)) - .withConstantFeature(AuthorIsErased, isErased(state)) - .withConstantFeature(AuthorIsProtected, isProtected(state)) - .withConstantFeature(AuthorBlocksViewer, authorBlocksViewer(state)) - .withConstantFeature(ViewerBlocksAuthor, viewerBlocksAuthor(state)) - .withConstantFeature(ViewerMutesAuthor, viewerMutesAuthor(state)) - .withConstantFeature(AuthorIsUnavailable, isUnavailable(state)) - } - - private[this] def isSuspended(state: UserUnavailableStateEnum): Boolean = - state match { - case UserUnavailableStateEnum.Suspended => - suspendedAuthorStats.counter().incr() - true - case UserUnavailableStateEnum.Filtered(result) - if UserVisibilityResultHelper.isDropSuspendedAuthor(result) => - suspendedAuthorStats.counter().incr() - suspendedAuthorStats.counter("filtered").incr() - true - case _ => false - } - - private[this] def isDeactivated(state: UserUnavailableStateEnum): Boolean = - state match { - case UserUnavailableStateEnum.Deactivated => - deactivatedAuthorStats.counter().incr() - true - case _ => false - } - - private[this] def isOffboarded(state: UserUnavailableStateEnum): Boolean = - state match { - case UserUnavailableStateEnum.Offboarded => - offboardedAuthorStats.counter().incr() - true - case _ => false - } - - private[this] def isErased(state: UserUnavailableStateEnum): Boolean = - state match { - case UserUnavailableStateEnum.Erased => - erasedAuthorStats.counter().incr() - true - case _ => false - } - - private[this] def isProtected(state: UserUnavailableStateEnum): Boolean = - state match { - case UserUnavailableStateEnum.Protected => - protectedAuthorStats.counter().incr() - true - case UserUnavailableStateEnum.Filtered(result) - if UserVisibilityResultHelper.isDropProtectedAuthor(result) => - protectedAuthorStats.counter().incr() - protectedAuthorStats.counter("filtered").incr() - true - case _ => false - } - - private[this] def authorBlocksViewer(state: UserUnavailableStateEnum): Boolean = - state match { - case UserUnavailableStateEnum.AuthorBlocksViewer => - authorBlocksViewerStats.counter().incr() - true - case UserUnavailableStateEnum.Filtered(result) - if UserVisibilityResultHelper.isDropAuthorBlocksViewer(result) => - authorBlocksViewerStats.counter().incr() - authorBlocksViewerStats.counter("filtered").incr() - true - case _ => false - } - - private[this] def viewerBlocksAuthor(state: UserUnavailableStateEnum): Boolean = - state match { - case UserUnavailableStateEnum.ViewerBlocksAuthor => - viewerBlocksAuthorStats.counter().incr() - true - case UserUnavailableStateEnum.Filtered(result) - if UserVisibilityResultHelper.isDropViewerBlocksAuthor(result) => - viewerBlocksAuthorStats.counter().incr() - viewerBlocksAuthorStats.counter("filtered").incr() - true - case _ => false - } - - private[this] def viewerMutesAuthor(state: UserUnavailableStateEnum): Boolean = - state match { - case UserUnavailableStateEnum.ViewerMutesAuthor => - viewerMutesAuthorStats.counter().incr() - true - case UserUnavailableStateEnum.Filtered(result) - if UserVisibilityResultHelper.isDropViewerMutesAuthor(result) => - viewerMutesAuthorStats.counter().incr() - viewerMutesAuthorStats.counter("filtered").incr() - true - case _ => false - } - - private[this] def isUnavailable(state: UserUnavailableStateEnum): Boolean = - state match { - case UserUnavailableStateEnum.Unavailable => - unavailableStats.counter().incr() - true - case UserUnavailableStateEnum.Filtered(result) - if UserVisibilityResultHelper.isDropUnspecifiedAuthor(result) => - unavailableStats.counter().incr() - unavailableStats.counter("filtered").incr() - true - case _ => false - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerAdvancedFilteringFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerAdvancedFilteringFeatures.docx new file mode 100644 index 000000000..0db06e81f Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerAdvancedFilteringFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerAdvancedFilteringFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerAdvancedFilteringFeatures.scala deleted file mode 100644 index 38b3106f0..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerAdvancedFilteringFeatures.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.AdvancedFilters -import com.twitter.gizmoduck.thriftscala.MentionFilter -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.features.ViewerFiltersDefaultProfileImage -import com.twitter.visibility.features.ViewerFiltersNewUsers -import com.twitter.visibility.features.ViewerFiltersNoConfirmedEmail -import com.twitter.visibility.features.ViewerFiltersNoConfirmedPhone -import com.twitter.visibility.features.ViewerFiltersNotFollowedBy -import com.twitter.visibility.features.ViewerMentionFilter - -class ViewerAdvancedFilteringFeatures(userSource: UserSource, statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("viewer_advanced_filtering_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val viewerFiltersNoConfirmedEmail = - scopedStatsReceiver.scope(ViewerFiltersNoConfirmedEmail.name).counter("requests") - private[this] val viewerFiltersNoConfirmedPhone = - scopedStatsReceiver.scope(ViewerFiltersNoConfirmedPhone.name).counter("requests") - private[this] val viewerFiltersDefaultProfileImage = - scopedStatsReceiver.scope(ViewerFiltersDefaultProfileImage.name).counter("requests") - private[this] val viewerFiltersNewUsers = - scopedStatsReceiver.scope(ViewerFiltersNewUsers.name).counter("requests") - private[this] val viewerFiltersNotFollowedBy = - scopedStatsReceiver.scope(ViewerFiltersNotFollowedBy.name).counter("requests") - private[this] val viewerMentionFilter = - scopedStatsReceiver.scope(ViewerMentionFilter.name).counter("requests") - - def forViewerId(viewerId: Option[Long]): FeatureMapBuilder => FeatureMapBuilder = { - requests.incr() - - _.withFeature(ViewerFiltersNoConfirmedEmail, viewerFiltersNoConfirmedEmail(viewerId)) - .withFeature(ViewerFiltersNoConfirmedPhone, viewerFiltersNoConfirmedPhone(viewerId)) - .withFeature(ViewerFiltersDefaultProfileImage, viewerFiltersDefaultProfileImage(viewerId)) - .withFeature(ViewerFiltersNewUsers, viewerFiltersNewUsers(viewerId)) - .withFeature(ViewerFiltersNotFollowedBy, viewerFiltersNotFollowedBy(viewerId)) - .withFeature(ViewerMentionFilter, viewerMentionFilter(viewerId)) - } - - def viewerFiltersNoConfirmedEmail(viewerId: Option[Long]): Stitch[Boolean] = - viewerAdvancedFilters(viewerId, af => af.filterNoConfirmedEmail, viewerFiltersNoConfirmedEmail) - - def viewerFiltersNoConfirmedPhone(viewerId: Option[Long]): Stitch[Boolean] = - viewerAdvancedFilters(viewerId, af => af.filterNoConfirmedPhone, viewerFiltersNoConfirmedPhone) - - def viewerFiltersDefaultProfileImage(viewerId: Option[Long]): Stitch[Boolean] = - viewerAdvancedFilters( - viewerId, - af => af.filterDefaultProfileImage, - viewerFiltersDefaultProfileImage - ) - - def viewerFiltersNewUsers(viewerId: Option[Long]): Stitch[Boolean] = - viewerAdvancedFilters(viewerId, af => af.filterNewUsers, viewerFiltersNewUsers) - - def viewerFiltersNotFollowedBy(viewerId: Option[Long]): Stitch[Boolean] = - viewerAdvancedFilters(viewerId, af => af.filterNotFollowedBy, viewerFiltersNotFollowedBy) - - def viewerMentionFilter(viewerId: Option[Long]): Stitch[MentionFilter] = { - viewerMentionFilter.incr() - viewerId match { - case Some(id) => - userSource.getMentionFilter(id).handle { - case NotFound => - MentionFilter.Unfiltered - } - case _ => Stitch.value(MentionFilter.Unfiltered) - } - } - - private[this] def viewerAdvancedFilters( - viewerId: Option[Long], - advancedFilterCheck: AdvancedFilters => Boolean, - featureCounter: Counter - ): Stitch[Boolean] = { - featureCounter.incr() - - val advancedFilters = viewerId match { - case Some(id) => userSource.getAdvancedFilters(id) - case _ => Stitch.value(AdvancedFilters()) - } - - advancedFilters.map(advancedFilterCheck) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerFeatures.docx new file mode 100644 index 000000000..c954a3eb7 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerFeatures.scala deleted file mode 100644 index 4e97ce5d4..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerFeatures.scala +++ /dev/null @@ -1,245 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.Label -import com.twitter.gizmoduck.thriftscala.Safety -import com.twitter.gizmoduck.thriftscala.UniversalQualityFiltering -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.UserId -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.features._ -import com.twitter.visibility.interfaces.common.blender.BlenderVFRequestContext -import com.twitter.visibility.interfaces.common.search.SearchVFRequestContext -import com.twitter.visibility.models.UserAge -import com.twitter.visibility.models.ViewerContext - -class ViewerFeatures(userSource: UserSource, statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("viewer_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val viewerIdCount = - scopedStatsReceiver.scope(ViewerId.name).counter("requests") - private[this] val requestCountryCode = - scopedStatsReceiver.scope(RequestCountryCode.name).counter("requests") - private[this] val requestIsVerifiedCrawler = - scopedStatsReceiver.scope(RequestIsVerifiedCrawler.name).counter("requests") - private[this] val viewerUserLabels = - scopedStatsReceiver.scope(ViewerUserLabels.name).counter("requests") - private[this] val viewerIsDeactivated = - scopedStatsReceiver.scope(ViewerIsDeactivated.name).counter("requests") - private[this] val viewerIsProtected = - scopedStatsReceiver.scope(ViewerIsProtected.name).counter("requests") - private[this] val viewerIsSuspended = - scopedStatsReceiver.scope(ViewerIsSuspended.name).counter("requests") - private[this] val viewerRoles = - scopedStatsReceiver.scope(ViewerRoles.name).counter("requests") - private[this] val viewerCountryCode = - scopedStatsReceiver.scope(ViewerCountryCode.name).counter("requests") - private[this] val viewerAge = - scopedStatsReceiver.scope(ViewerAge.name).counter("requests") - private[this] val viewerHasUniversalQualityFilterEnabled = - scopedStatsReceiver.scope(ViewerHasUniversalQualityFilterEnabled.name).counter("requests") - private[this] val viewerIsSoftUserCtr = - scopedStatsReceiver.scope(ViewerIsSoftUser.name).counter("requests") - - def forViewerBlenderContext( - blenderContext: BlenderVFRequestContext, - viewerContext: ViewerContext - ): FeatureMapBuilder => FeatureMapBuilder = - forViewerContext(viewerContext) - .andThen( - _.withConstantFeature( - ViewerOptInBlocking, - blenderContext.userSearchSafetySettings.optInBlocking) - .withConstantFeature( - ViewerOptInFiltering, - blenderContext.userSearchSafetySettings.optInFiltering) - ) - - def forViewerSearchContext( - searchContext: SearchVFRequestContext, - viewerContext: ViewerContext - ): FeatureMapBuilder => FeatureMapBuilder = - forViewerContext(viewerContext) - .andThen( - _.withConstantFeature( - ViewerOptInBlocking, - searchContext.userSearchSafetySettings.optInBlocking) - .withConstantFeature( - ViewerOptInFiltering, - searchContext.userSearchSafetySettings.optInFiltering) - ) - - def forViewerContext(viewerContext: ViewerContext): FeatureMapBuilder => FeatureMapBuilder = - forViewerId(viewerContext.userId) - .andThen( - _.withConstantFeature(RequestCountryCode, requestCountryCode(viewerContext)) - ).andThen( - _.withConstantFeature(RequestIsVerifiedCrawler, requestIsVerifiedCrawler(viewerContext)) - ) - - def forViewerId(viewerId: Option[UserId]): FeatureMapBuilder => FeatureMapBuilder = { builder => - requests.incr() - - val builderWithFeatures = builder - .withConstantFeature(ViewerId, viewerId) - .withFeature(ViewerIsProtected, viewerIsProtected(viewerId)) - .withFeature( - ViewerHasUniversalQualityFilterEnabled, - viewerHasUniversalQualityFilterEnabled(viewerId) - ) - .withFeature(ViewerIsSuspended, viewerIsSuspended(viewerId)) - .withFeature(ViewerIsDeactivated, viewerIsDeactivated(viewerId)) - .withFeature(ViewerUserLabels, viewerUserLabels(viewerId)) - .withFeature(ViewerRoles, viewerRoles(viewerId)) - .withFeature(ViewerAge, viewerAgeInYears(viewerId)) - .withFeature(ViewerIsSoftUser, viewerIsSoftUser(viewerId)) - - viewerId match { - case Some(_) => - viewerIdCount.incr() - builderWithFeatures - .withFeature(ViewerCountryCode, viewerCountryCode(viewerId)) - - case _ => - builderWithFeatures - } - } - - def forViewerNoDefaults(viewerOpt: Option[User]): FeatureMapBuilder => FeatureMapBuilder = { - builder => - requests.incr() - - viewerOpt match { - case Some(viewer) => - builder - .withConstantFeature(ViewerId, viewer.id) - .withConstantFeature(ViewerIsProtected, viewerIsProtectedOpt(viewer)) - .withConstantFeature(ViewerIsSuspended, viewerIsSuspendedOpt(viewer)) - .withConstantFeature(ViewerIsDeactivated, viewerIsDeactivatedOpt(viewer)) - .withConstantFeature(ViewerCountryCode, viewerCountryCode(viewer)) - case None => - builder - .withConstantFeature(ViewerIsProtected, false) - .withConstantFeature(ViewerIsSuspended, false) - .withConstantFeature(ViewerIsDeactivated, false) - } - } - - private def checkSafetyValue( - viewerId: Option[UserId], - safetyCheck: Safety => Boolean, - featureCounter: Counter - ): Stitch[Boolean] = - viewerId match { - case Some(id) => - userSource.getSafety(id).map(safetyCheck).ensure { - featureCounter.incr() - } - case None => Stitch.False - } - - private def checkSafetyValue( - viewer: User, - safetyCheck: Safety => Boolean - ): Boolean = { - viewer.safety.exists(safetyCheck) - } - - def viewerIsProtected(viewerId: Option[UserId]): Stitch[Boolean] = - checkSafetyValue(viewerId, s => s.isProtected, viewerIsProtected) - - def viewerIsProtected(viewer: User): Boolean = - checkSafetyValue(viewer, s => s.isProtected) - - def viewerIsProtectedOpt(viewer: User): Option[Boolean] = - viewer.safety.map(_.isProtected) - - def viewerIsDeactivated(viewerId: Option[UserId]): Stitch[Boolean] = - checkSafetyValue(viewerId, s => s.deactivated, viewerIsDeactivated) - - def viewerIsDeactivated(viewer: User): Boolean = - checkSafetyValue(viewer, s => s.deactivated) - - def viewerIsDeactivatedOpt(viewer: User): Option[Boolean] = - viewer.safety.map(_.deactivated) - - def viewerHasUniversalQualityFilterEnabled(viewerId: Option[UserId]): Stitch[Boolean] = - checkSafetyValue( - viewerId, - s => s.universalQualityFiltering == UniversalQualityFiltering.Enabled, - viewerHasUniversalQualityFilterEnabled - ) - - def viewerUserLabels(viewerIdOpt: Option[UserId]): Stitch[Seq[Label]] = - viewerIdOpt match { - case Some(viewerId) => - userSource - .getLabels(viewerId).map(_.labels) - .handle { - case NotFound => Seq.empty - }.ensure { - viewerUserLabels.incr() - } - case _ => Stitch.value(Seq.empty) - } - - def viewerAgeInYears(viewerId: Option[UserId]): Stitch[UserAge] = - (viewerId match { - case Some(id) => - userSource - .getExtendedProfile(id).map(_.ageInYears) - .handle { - case NotFound => None - }.ensure { - viewerAge.incr() - } - case _ => Stitch.value(None) - }).map(UserAge) - - def viewerIsSoftUser(viewerId: Option[UserId]): Stitch[Boolean] = { - viewerId match { - case Some(id) => - userSource - .getUserType(id).map { userType => - userType == UserType.Soft - }.ensure { - viewerIsSoftUserCtr.incr() - } - case _ => Stitch.False - } - } - - def requestCountryCode(viewerContext: ViewerContext): Option[String] = { - requestCountryCode.incr() - viewerContext.requestCountryCode - } - - def requestIsVerifiedCrawler(viewerContext: ViewerContext): Boolean = { - requestIsVerifiedCrawler.incr() - viewerContext.isVerifiedCrawler - } - - def viewerCountryCode(viewerId: Option[UserId]): Stitch[String] = - viewerId match { - case Some(id) => - userSource - .getAccount(id).map(_.countryCode).flatMap { - case Some(countryCode) => Stitch.value(countryCode.toLowerCase) - case _ => Stitch.NotFound - }.ensure { - viewerCountryCode.incr() - } - - case _ => Stitch.NotFound - } - - def viewerCountryCode(viewer: User): Option[String] = - viewer.account.flatMap(_.countryCode) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSearchSafetyFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSearchSafetyFeatures.docx new file mode 100644 index 000000000..76270d14f Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSearchSafetyFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSearchSafetyFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSearchSafetyFeatures.scala deleted file mode 100644 index 6cddcd74c..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSearchSafetyFeatures.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.UserId -import com.twitter.visibility.common.UserSearchSafetySource -import com.twitter.visibility.features.ViewerId -import com.twitter.visibility.features.ViewerOptInBlocking -import com.twitter.visibility.features.ViewerOptInFiltering - -class ViewerSearchSafetyFeatures( - userSearchSafetySource: UserSearchSafetySource, - statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = statsReceiver.scope("viewer_search_safety_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - private[this] val viewerOptInBlockingRequests = - scopedStatsReceiver.scope(ViewerOptInBlocking.name).counter("requests") - - private[this] val viewerOptInFilteringRequests = - scopedStatsReceiver.scope(ViewerOptInFiltering.name).counter("requests") - - def forViewerId(viewerId: Option[UserId]): FeatureMapBuilder => FeatureMapBuilder = { builder => - requests.incr() - - builder - .withConstantFeature(ViewerId, viewerId) - .withFeature(ViewerOptInBlocking, viewerOptInBlocking(viewerId)) - .withFeature(ViewerOptInFiltering, viewerOptInFiltering(viewerId)) - } - - def viewerOptInBlocking(viewerId: Option[UserId]): Stitch[Boolean] = { - viewerOptInBlockingRequests.incr() - viewerId match { - case Some(userId) => userSearchSafetySource.optInBlocking(userId) - case _ => Stitch.False - } - } - - def viewerOptInFiltering(viewerId: Option[UserId]): Stitch[Boolean] = { - viewerOptInFilteringRequests.incr() - viewerId match { - case Some(userId) => userSearchSafetySource.optInFiltering(userId) - case _ => Stitch.False - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSensitiveMediaSettingsFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSensitiveMediaSettingsFeatures.docx new file mode 100644 index 000000000..19c2a78da Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSensitiveMediaSettingsFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSensitiveMediaSettingsFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSensitiveMediaSettingsFeatures.scala deleted file mode 100644 index 31b982886..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/builder/users/ViewerSensitiveMediaSettingsFeatures.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.visibility.builder.users - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.stitch.NotFound -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.common.UserId -import com.twitter.visibility.common.UserSensitiveMediaSettingsSource -import com.twitter.visibility.features.ViewerId -import com.twitter.visibility.features.ViewerSensitiveMediaSettings -import com.twitter.visibility.models.UserSensitiveMediaSettings - - -class ViewerSensitiveMediaSettingsFeatures( - userSensitiveMediaSettingsSource: UserSensitiveMediaSettingsSource, - statsReceiver: StatsReceiver) { - private[this] val scopedStatsReceiver = - statsReceiver.scope("viewer_sensitive_media_settings_features") - - private[this] val requests = scopedStatsReceiver.counter("requests") - - def forViewerId(viewerId: Option[UserId]): FeatureMapBuilder => FeatureMapBuilder = { builder => - requests.incr() - - builder - .withConstantFeature(ViewerId, viewerId) - .withFeature(ViewerSensitiveMediaSettings, viewerSensitiveMediaSettings(viewerId)) - } - - def viewerSensitiveMediaSettings(viewerId: Option[UserId]): Stitch[UserSensitiveMediaSettings] = { - (viewerId match { - case Some(userId) => - userSensitiveMediaSettingsSource - .userSensitiveMediaSettings(userId) - .handle { - case NotFound => None - } - case _ => Stitch.value(None) - }).map(UserSensitiveMediaSettings) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/BUILD deleted file mode 100644 index b0562b356..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "abdecider/src/main/scala", - "configapi/configapi-abdecider", - "configapi/configapi-core", - "configapi/configapi-featureswitches:v2", - "decider", - "featureswitches/featureswitches-core/src/main/scala", - "finagle/finagle-stats", - "servo/decider/src/main/scala", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/configs", - "visibility/lib/src/main/scala/com/twitter/visibility/models", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/BUILD.docx new file mode 100644 index 000000000..5b4f3ec11 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/ConfigBuilder.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/ConfigBuilder.docx new file mode 100644 index 000000000..c9229d40c Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/ConfigBuilder.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/ConfigBuilder.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/ConfigBuilder.scala deleted file mode 100644 index df634d400..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/ConfigBuilder.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.visibility.configapi - -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi.CompositeConfig -import com.twitter.timelines.configapi.Config -import com.twitter.util.Memoize -import com.twitter.visibility.configapi.configs.VisibilityDeciders -import com.twitter.visibility.configapi.configs.VisibilityExperimentsConfig -import com.twitter.visibility.configapi.configs.VisibilityFeatureSwitches -import com.twitter.visibility.models.SafetyLevel - -object ConfigBuilder { - - def apply(statsReceiver: StatsReceiver, decider: Decider, logger: Logger): ConfigBuilder = { - val deciderGateBuilder: DeciderGateBuilder = - new DeciderGateBuilder(decider) - - new ConfigBuilder( - deciderGateBuilder, - statsReceiver, - logger - ) - } -} - -class ConfigBuilder( - deciderGateBuilder: DeciderGateBuilder, - statsReceiver: StatsReceiver, - logger: Logger) { - - def buildMemoized: SafetyLevel => Config = Memoize(build) - - def build(safetyLevel: SafetyLevel): Config = { - new CompositeConfig( - VisibilityExperimentsConfig.config(safetyLevel) :+ - VisibilityDeciders.config(deciderGateBuilder, logger, statsReceiver, safetyLevel) :+ - VisibilityFeatureSwitches.config(statsReceiver, logger) - ) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityParams.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityParams.docx new file mode 100644 index 000000000..5211cf178 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityParams.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityParams.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityParams.scala deleted file mode 100644 index e45485b52..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityParams.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.visibility.configapi - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.timelines.configapi.Params -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.UnitOfDiversion -import com.twitter.visibility.models.ViewerContext - -object VisibilityParams { - def apply( - log: Logger, - statsReceiver: StatsReceiver, - decider: Decider, - abDecider: LoggingABDecider, - featureSwitches: FeatureSwitches - ): VisibilityParams = - new VisibilityParams(log, statsReceiver, decider, abDecider, featureSwitches) -} - -class VisibilityParams( - log: Logger, - statsReceiver: StatsReceiver, - decider: Decider, - abDecider: LoggingABDecider, - featureSwitches: FeatureSwitches) { - - private[this] val contextFactory = new VisibilityRequestContextFactory( - abDecider, - featureSwitches - ) - - private[this] val configBuilder = ConfigBuilder(statsReceiver.scope("config"), decider, log) - - private[this] val paramStats: MemoizingStatsReceiver = new MemoizingStatsReceiver( - statsReceiver.scope("configapi_params")) - - def apply( - viewerContext: ViewerContext, - safetyLevel: SafetyLevel, - unitsOfDiversion: Seq[UnitOfDiversion] = Seq.empty - ): Params = { - val config = configBuilder.build(safetyLevel) - val requestContext = contextFactory(viewerContext, safetyLevel, unitsOfDiversion) - config.apply(requestContext, paramStats) - } - - def memoized( - viewerContext: ViewerContext, - safetyLevel: SafetyLevel, - unitsOfDiversion: Seq[UnitOfDiversion] = Seq.empty - ): Params = { - val config = configBuilder.buildMemoized(safetyLevel) - val requestContext = contextFactory(viewerContext, safetyLevel, unitsOfDiversion) - config.apply(requestContext, paramStats) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContext.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContext.docx new file mode 100644 index 000000000..b390e7995 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContext.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContext.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContext.scala deleted file mode 100644 index 9e9564392..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContext.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.visibility.configapi - -import com.twitter.timelines.configapi._ - -case class VisibilityRequestContext( - userId: Option[Long], - guestId: Option[Long], - experimentContext: ExperimentContext = NullExperimentContext, - featureContext: FeatureContext = NullFeatureContext) - extends BaseRequestContext - with WithUserId - with WithGuestId - with WithExperimentContext - with WithFeatureContext diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContextFactory.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContextFactory.docx new file mode 100644 index 000000000..858cfe246 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContextFactory.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContextFactory.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContextFactory.scala deleted file mode 100644 index 1d389d68e..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/VisibilityRequestContextFactory.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.visibility.configapi - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.featureswitches.FSRecipient -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.timelines.configapi.abdecider.UserRecipientExperimentContextFactory -import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext -import com.twitter.timelines.configapi.FeatureContext -import com.twitter.timelines.configapi.NullExperimentContext -import com.twitter.timelines.configapi.UseFeatureContextExperimentContext -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.UnitOfDiversion -import com.twitter.visibility.models.ViewerContext - -class VisibilityRequestContextFactory( - loggingABDecider: LoggingABDecider, - featureSwitches: FeatureSwitches) { - private val userExperimentContextFactory = new UserRecipientExperimentContextFactory( - loggingABDecider - ) - private[this] def getFeatureContext( - context: ViewerContext, - safetyLevel: SafetyLevel, - unitsOfDiversion: Seq[UnitOfDiversion] - ): FeatureContext = { - val uodCustomFields = unitsOfDiversion.map(_.apply) - val recipient = FSRecipient( - userId = context.userId, - guestId = context.guestId, - userAgent = context.fsUserAgent, - clientApplicationId = context.clientApplicationId, - countryCode = context.requestCountryCode, - deviceId = context.deviceId, - languageCode = context.requestLanguageCode, - isTwoffice = Some(context.isTwOffice), - userRoles = context.userRoles, - ).withCustomFields(("safety_level", safetyLevel.name), uodCustomFields: _*) - - val results = featureSwitches.matchRecipient(recipient) - new FeatureSwitchResultsFeatureContext(results) - } - - def apply( - context: ViewerContext, - safetyLevel: SafetyLevel, - unitsOfDiversion: Seq[UnitOfDiversion] = Seq.empty - ): VisibilityRequestContext = { - val experimentContextBase = - context.userId - .map(userId => userExperimentContextFactory.apply(userId)).getOrElse(NullExperimentContext) - - val featureContext = getFeatureContext(context, safetyLevel, unitsOfDiversion) - - val experimentContext = - UseFeatureContextExperimentContext(experimentContextBase, featureContext) - - VisibilityRequestContext( - context.userId, - context.guestId, - experimentContext, - featureContext - ) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/BUILD deleted file mode 100644 index 89e1b7c83..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "configapi/configapi-decider", - "decider", - "finagle/finagle-stats", - "servo/decider", - "servo/decider/src/main/scala", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/params", - "visibility/lib/src/main/scala/com/twitter/visibility/models", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/BUILD.docx new file mode 100644 index 000000000..aead7573f Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/DeciderKey.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/DeciderKey.docx new file mode 100644 index 000000000..eaf73db4f Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/DeciderKey.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/DeciderKey.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/DeciderKey.scala deleted file mode 100644 index 58331779c..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/DeciderKey.scala +++ /dev/null @@ -1,1067 +0,0 @@ -package com.twitter.visibility.configapi.configs - -import com.twitter.servo.decider.DeciderKeyEnum - -private[visibility] object DeciderKey extends DeciderKeyEnum { - - val EnableAllSubscribedListsSafetyLevel: Value = Value( - "visibility_library_enable_all_subscribed_lists_safety_level" - ) - val EnableAdsBusinessSettingsSafetyLevel: Value = Value( - "visibility_library_enable_ads_business_settings_safety_level" - ) - val EnableAdsCampaignSafetyLevel: Value = Value( - "visibility_library_enable_ads_campaign_safety_level" - ) - val EnableAdsManagerSafetyLevel: Value = Value( - "visibility_library_enable_ads_manager_safety_level" - ) - val EnableAdsReportingDashboardSafetyLevel: Value = Value( - "visibility_library_enable_ads_reporting_dashboard_safety_level" - ) - val EnableAppealsSafetyLevel: Value = Value( - "visibility_library_enable_appeals_safety_level" - ) - val EnableArticleTweetTimelineSafetyLevel: Value = Value( - "visibility_library_enable_article_tweet_timeline_safety_level" - ) - val EnableBaseQig: Value = Value( - "visibility_library_enable_base_qig_safety_level" - ) - val EnableBirdwatchNoteAuthorSafetyLevel: Value = Value( - "visibility_library_enable_birdwatch_note_author_safety_level" - ) - val EnableBirdwatchNoteTweetsTimelineSafetyLevel: Value = Value( - "visibility_library_enable_birdwatch_note_tweets_timeline_safety_level" - ) - val EnableBirdwatchNeedsYourHelpNotificationsSafetyLevel: Value = Value( - "visibility_library_enable_birdwatch_needs_your_help_notifications_safety_level" - ) - val EnableBlockMuteUsersTimelineSafetyLevel: Value = Value( - "visibility_library_enable_block_mute_users_timeline_safety_level" - ) - val EnableBrandSafetySafetyLevel: Value = Value( - "visibility_library_enable_brand_safety_safety_level" - ) - val EnableCardPollVotingSafetyLevel: Value = Value( - "visibility_library_enable_card_poll_voting_safety_level" - ) - val EnableCardsServiceSafetyLevel: Value = Value( - "visibility_library_enable_cards_service_safety_level" - ) - val EnableCommunitiesSafetyLevel: Value = Value( - "visibility_library_enable_communities_safety_level" - ) - val EnableContentControlToolInstallSafetyLevel: Value = Value( - "visibility_library_enable_content_control_tool_install_safety_level" - ) - val EnableConversationFocalPrehydrationSafetyLevel: Value = Value( - "visibility_library_enable_conversation_focal_prehydration_safety_level" - ) - val EnableConversationFocalTweetSafetyLevel: Value = Value( - "visibility_library_enable_conversation_focal_tweet_safety_level" - ) - val EnableConversationInjectedTweetSafetyLevel: Value = Value( - "visibility_library_enable_conversation_injected_tweet_safety_level" - ) - val EnableConversationReplySafetyLevel: Value = Value( - "visibility_library_enable_conversation_reply_safety_level" - ) - val EnableCuratedTrendsRepresentativeTweet: Value = Value( - "visibility_library_curated_trends_representative_tweet" - ) - val EnableCurationPolicyViolations: Value = Value( - "visibility_library_curation_policy_violations" - ) - val EnableDeprecatedSafetyLevelSafetyLevel: Value = Value( - "visibility_library_enable_deprecated_safety_level_safety_level" - ) - val EnableDevPlatformGetListTweetsSafetyLevel: Value = Value( - "visibility_library_enable_dev_platform_get_list_tweets_safety_level" - ) - val EnableDesFollowingAndFollowersUserListSafetyLevel: Value = Value( - "visibility_library_enable_des_following_and_followers_user_list_safety_level" - ) - val EnableDesHomeTimelineSafetyLevel: Value = Value( - "visibility_library_enable_des_home_timeline_safety_level" - ) - val EnableDesQuoteTweetTimelineSafetyLevel: Value = Value( - "visibility_library_enable_des_quote_tweet_timeline_safety_level" - ) - val EnableDesRealtimeSafetyLevel: Value = Value( - "visibility_library_enable_des_realtime_safety_level" - ) - val EnableDesRealtimeSpamEnrichmentSafetyLevel: Value = Value( - "visibility_library_enable_des_realtime_spam_enrichment_safety_level" - ) - val EnableDesRealtimeTweetFilterSafetyLevel: Value = Value( - "visibility_library_enable_des_realtime_tweet_filter_safety_level" - ) - val EnableDesRetweetingUsersSafetyLevel: Value = Value( - "visibility_library_enable_des_retweeting_users_safety_level" - ) - val EnableDesTweetDetailSafetyLevel: Value = Value( - "visibility_library_enable_des_tweet_detail_safety_level" - ) - val EnableDesTweetLikingUsersSafetyLevel: Value = Value( - "visibility_library_enable_des_tweet_liking_users_safety_level" - ) - val EnableDesUserBookmarksSafetyLevel: Value = Value( - "visibility_library_enable_des_user_bookmarks_safety_level" - ) - val EnableDesUserLikedTweetsSafetyLevel: Value = Value( - "visibility_library_enable_des_user_liked_tweets_safety_level" - ) - val EnableDesUserMentionsSafetyLevel: Value = Value( - "visibility_library_enable_des_user_mentions_safety_level" - ) - val EnableDesUserTweetsSafetyLevel: Value = Value( - "visibility_library_enable_des_user_tweets_safety_level" - ) - val EnableDevPlatformComplianceStreamSafetyLevel: Value = Value( - "visibility_library_enable_dev_platform_compliance_stream_safety_level" - ) - val EnableDirectMessagesSafetyLevel: Value = Value( - "visibility_library_enable_direct_messages_safety_level" - ) - val EnableDirectMessagesConversationListSafetyLevel: Value = Value( - "visibility_library_enable_direct_messages_conversation_list_safety_level" - ) - val EnableDirectMessagesConversationTimelineSafetyLevel: Value = Value( - "visibility_library_enable_direct_messages_conversation_timeline_safety_level" - ) - val EnableDirectMessagesInboxSafetyLevel: Value = Value( - "visibility_library_enable_direct_messages_inbox_safety_level" - ) - val EnableDirectMessagesMutedUsersSafetyLevel: Value = Value( - "visibility_library_enable_direct_messages_muted_users_safety_level" - ) - val EnableDirectMessagesPinnedSafetyLevel: Value = Value( - "visibility_library_enable_direct_messages_pinned_safety_level" - ) - val EnableDirectMessagesSearchSafetyLevel: Value = Value( - "visibility_library_enable_direct_messages_search_safety_level" - ) - val EnableElevatedQuoteTweetTimelineSafetyLevel: Value = Value( - "visibility_library_enable_elevated_quote_tweet_timeline_safety_level" - ) - val EnableEmbeddedTweetSafetyLevel: Value = Value( - "visibility_library_enable_embedded_tweet_safety_level" - ) - val EnableEmbedsPublicInterestNoticeSafetyLevel: Value = Value( - "visibility_library_enable_embeds_public_interest_notice_safety_level" - ) - val EnableEmbedTweetMarkupSafetyLevel: Value = Value( - "visibility_library_enable_embed_tweet_markup_safety_level" - ) - val EnableWritePathLimitedActionsEnforcementSafetyLevel: Value = Value( - "visibility_library_enable_write_path_limited_actions_enforcement_safety_level" - ) - val EnableFilterDefaultSafetyLevel: Value = Value( - "visibility_library_enable_filter_default_safety_level" - ) - val EnableFilterNoneSafetyLevel: Value = Value( - "visibility_library_enable_filter_none_safety_level" - ) - - val EnableFilterAllSafetyLevel: Value = Value( - "visibility_library_enable_filter_all_safety_level" - ) - val EnableFilterAllPlaceholderSafetyLevel: Value = Value( - "visibility_library_enable_filter_all_placeholder_safety_level" - ) - - val EnableFollowedTopicsTimelineSafetyLevel: Value = Value( - "visibility_library_enable_followed_topics_timeline_safety_level" - ) - - val EnableFollowerConnectionsSafetyLevel: Value = Value( - "visibility_library_enable_follower_connections_safety_level" - ) - val EnableFollowingAndFollowersUserListSafetyLevel: Value = Value( - "visibility_library_enable_following_and_followers_user_list_safety_level" - ) - - val EnableForDevelopmentOnlySafetyLevel: Value = Value( - "visibility_library_enable_for_development_only_safety_level" - ) - - val EnableFriendsFollowingListSafetyLevel: Value = Value( - "visibility_library_enable_friends_following_list_safety_level" - ) - - val EnableGraphqlDefaultSafetyLevel: Value = Value( - "visibility_library_enable_graphql_default_safety_level" - ) - - val EnableGryphonDecksAndColumnsSafetyLevel: Value = Value( - "visibility_library_enable_gryphon_decks_and_columns_safety_level" - ) - - val EnableHumanizationNudgeSafetyLevel: Value = Value( - "visibility_library_enable_humanization_nudge_safety_level" - ) - - val EnableKitchenSinkDevelopmentSafetyLevel: Value = Value( - "visibility_library_enable_kitchen_sink_development_safety_level" - ) - - val EnableListHeaderSafetyLevel: Value = Value( - "visibility_library_enable_list_header_safety_level" - ) - - val EnableListMembershipsSafetyLevel: Value = Value( - "visibility_library_enable_list_memberships_safety_level" - ) - - val EnableListOwnershipsSafetyLevel: Value = Value( - "visibility_library_enable_list_ownerships_safety_level" - ) - - val EnableListRecommendationsSafetyLevel: Value = Value( - "visibility_library_enable_list_recommendations_safety_level" - ) - - val EnableListSearchSafetyLevel: Value = Value( - "visibility_library_enable_list_search_safety_level" - ) - - val EnableListSubscriptionsSafetyLevel: Value = Value( - "visibility_library_enable_list_subscriptions_safety_level" - ) - - val EnableLivePipelineEngagementCountsSafetyLevel: Value = Value( - "visibility_library_enable_live_pipeline_engagement_counts_safety_level" - ) - val EnableLiveVideoTimelineSafetyLevel: Value = Value( - "visibility_library_enable_live_video_timeline_safety_level" - ) - val EnableMagicRecsAggressiveSafetyLevel: Value = Value( - "visibility_library_enable_magic_recs_aggressive_safety_level" - ) - val EnableMagicRecsAggressiveV2SafetyLevel: Value = Value( - "visibility_library_enable_magic_recs_aggressive_v2_safety_level" - ) - val EnableMagicRecsSafetyLevel: Value = Value( - "visibility_library_enable_magic_recs_safety_level" - ) - val EnableMagicRecsV2SafetyLevel: Value = Value( - "visibility_library_enable_magic_recs_v2_safety_level" - ) - val EnableMinimalSafetyLevel: Value = Value( - "visibility_library_enable_minimal_safety_level" - ) - val EnableModeratedTweetsTimelineSafetyLevel: Value = Value( - "visibility_library_enable_moderated_tweets_timeline_safety_level" - ) - val EnableMomentsSafetyLevel: Value = Value( - "visibility_library_enable_moments_safety_level" - ) - val EnableNearbyTimelineSafetyLevel: Value = Value( - "visibility_library_enable_nearby_timeline_safety_level" - ) - val EnableNewUserExperienceSafetyLevel: Value = Value( - "visibility_library_enable_new_user_experience_safety_level" - ) - val EnableNotificationsIbisSafetyLevel: Value = Value( - "visibility_library_enable_notifications_ibis_safety_level" - ) - val EnableNotificationsPlatformSafetyLevel: Value = Value( - "visibility_library_enable_notifications_platform_safety_level" - ) - val EnableNotificationsPlatformPushSafetyLevel: Value = Value( - "visibility_library_enable_notifications_platform_push_safety_level" - ) - val EnableNotificationsReadSafetyLevel: Value = Value( - "visibility_library_enable_notifications_read_safety_level" - ) - val EnableNotificationsTimelineDeviceFollowSafetyLevel: Value = Value( - "visibility_library_enable_notifications_timeline_device_follow_safety_level" - ) - val EnableNotificationsWriteSafetyLevel: Value = Value( - "visibility_library_enable_notifications_write_safety_level" - ) - val EnableNotificationsWriterV2SafetyLevel: Value = Value( - "visibility_library_enable_notification_writer_v2_safety_level" - ) - val EnableNotificationsWriterTweetHydratorSafetyLevel: Value = Value( - "visibility_library_enable_notifications_writer_tweet_hydrator_safety_level" - ) - val EnableQuickPromoteTweetEligibilitySafetyLevel: Value = Value( - "visibility_library_enable_quick_promote_tweet_eligibility_safety_level" - ) - val EnableQuoteTweetTimelineSafetyLevel: Value = Value( - "visibility_library_enable_quote_tweet_timeline_safety_level" - ) - val EnableQuotedTweetRulesSafetyLevel: Value = Value( - "visibility_library_enable_quoted_tweet_rules_safety_level" - ) - val EnableRecommendationsSafetyLevel: Value = Value( - "visibility_library_enable_recommendations_safety_level" - ) - val EnableRecosVideoSafetyLevel: Value = Value( - "visibility_library_enable_recos_video_safety_level" - ) - val EnableRecosWritePathSafetyLevel: Value = Value( - "visibility_library_enable_recos_write_path_safety_level" - ) - val EnableRepliesGroupingSafetyLevel: Value = Value( - "visibility_library_enable_replies_grouping_safety_level" - ) - val EnableReportCenterSafetyLevel: Value = Value( - "visibility_library_enable_report_center_safety_level" - ) - val EnableReturningUserExperienceSafetyLevel: Value = Value( - "visibility_library_enable_returning_user_experience_safety_level" - ) - val EnableReturningUserExperienceFocalTweetSafetyLevel: Value = Value( - "visibility_library_enable_returning_user_experience_focal_tweet_safety_level" - ) - val EnableRevenueSafetyLevel: Value = Value( - "visibility_library_enable_revenue_safety_level" - ) - val EnableRitoActionedTweetTimelineSafetyLevel: Value = Value( - "visibility_library_enable_rito_actioned_tweet_timeline_safety_level" - ) - val EnableSafeSearchMinimalSafetyLevel: Value = Value( - "visibility_library_enable_safe_search_minimal_safety_level" - ) - val EnableSafeSearchStrictSafetyLevel: Value = Value( - "visibility_library_enable_safe_search_strict_safety_level" - ) - val EnableAccessInternalPromotedContentSafetyLevel: Value = Value( - "visibility_library_enable_access_internal_promoted_content_safety_level" - ) - val EnableSearchHydration: Value = Value( - "visibility_library_enable_search_hydration_safety_level" - ) - val EnableSearchLatest: Value = Value( - "visibility_library_enable_search_latest_safety_level" - ) - val EnableSearchMixerSrpMinimalSafetyLevel: Value = Value( - "visibility_library_enable_search_mixer_srp_minimal_safety_level" - ) - val EnableSearchMixerSrpStrictSafetyLevel: Value = Value( - "visibility_library_enable_search_mixer_srp_strict_safety_level" - ) - val EnableUserSearchSrpSafetyLevel: Value = Value( - "visibility_library_enable_user_search_srp_safety_level" - ) - val EnableUserSearchTypeaheadSafetyLevel: Value = Value( - "visibility_library_enable_user_search_typeahead_safety_level" - ) - val EnableSearchPeopleSrp: Value = Value( - "visibility_library_enable_search_people_srp_safety_level" - ) - val EnableSearchPeopleTypeahead: Value = Value( - "visibility_library_enable_search_people_typeahead_safety_level" - ) - val EnableSearchPhoto: Value = Value( - "visibility_library_enable_search_photo_safety_level" - ) - val EnableSearchTop: Value = Value( - "visibility_library_enable_search_top_safety_level" - ) - val EnableSearchTopQig: Value = Value( - "visibility_library_enable_search_top_qig_safety_level" - ) - val EnableSearchTrendTakeoverPromotedTweet: Value = Value( - "visibility_library_enable_search_trend_takeover_promoted_tweet_safety_level" - ) - val EnableSearchVideo: Value = Value( - "visibility_library_enable_search_video_safety_level" - ) - val EnableSearchLatestUserRules: Value = Value( - "visibility_library_enable_search_latest_user_rules_safety_level" - ) - val EnableShoppingManagerSpyModeSafetyLevel: Value = Value( - "visibility_library_enable_shopping_manager_spy_mode_safety_level" - ) - val EnableSignalsReactions: Value = Value( - "visibility_library_enable_signals_reactions_safety_level" - ) - val EnableSignalsTweetReactingUsers: Value = Value( - "visibility_library_enable_signals_tweet_reacting_users_safety_level" - ) - val EnableSocialProof: Value = Value( - "visibility_library_enable_social_proof_safety_level" - ) - val EnableSoftInterventionPivot: Value = Value( - "visibility_library_enable_soft_intervention_pivot_safety_level" - ) - val EnableSpaceFleetlineSafetyLevel: Value = Value( - "visibility_library_enable_space_fleetline_safety_level" - ) - val EnableSpaceHomeTimelineUprankingSafetyLevel: Value = Value( - "visibility_library_enable_space_home_timeline_upranking_safety_level" - ) - val EnableSpaceJoinScreenSafetyLevel: Value = Value( - "visibility_library_enable_space_join_screen_safety_level" - ) - val EnableSpaceNotificationsSafetyLevel: Value = Value( - "visibility_library_enable_space_notifications_safety_level" - ) - val EnableSpacesSafetyLevel: Value = Value( - "visibility_library_enable_spaces_safety_level" - ) - val EnableSpacesParticipantsSafetyLevel: Value = Value( - "visibility_library_enable_spaces_participants_safety_level" - ) - val EnableSpacesSellerApplicationStatus: Value = Value( - "visibility_library_enable_spaces_seller_application_status_safety_level" - ) - val EnableSpacesSharingSafetyLevel: Value = Value( - "visibility_library_enable_spaces_sharing_safety_level" - ) - val EnableSpaceTweetAvatarHomeTimelineSafetyLevel: Value = Value( - "visibility_library_enable_space_tweet_avatar_home_timeline_safety_level" - ) - val EnableStickersTimelineSafetyLevel: Value = Value( - "visibility_library_enable_stickers_timeline_safety_level" - ) - val EnableStratoExtLimitedEngagementsSafetyLevel: Value = Value( - "visibility_library_enable_strato_ext_limited_engagements_safety_level" - ) - val EnableStreamServicesSafetyLevel: Value = Value( - "visibility_library_enable_stream_services_safety_level" - ) - val EnableTestSafetyLevel: Value = Value( - "visibility_library_enable_test_safety_level" - ) - val EnableTimelineBookmarkSafetyLevel: Value = Value( - "visibility_library_enable_timeline_bookmark_safety_level" - ) - val EnableTimelineContentControlsSafetyLevel: Value = Value( - "visibility_library_enable_timeline_content_controls_safety_level" - ) - val EnableTimelineConversationsSafetyLevel: Value = Value( - "visibility_library_enable_timeline_conversations_safety_level" - ) - val EnableTimelineConversationsDownrankingSafetyLevel: Value = Value( - "visibility_library_enable_timeline_conversations_downranking_safety_level" - ) - val EnableTimelineConversationsDownrankingMinimalSafetyLevel: Value = Value( - "visibility_library_enable_timeline_conversations_downranking_minimal_safety_level" - ) - val EnableTimelineFavoritesSafetyLevel: Value = Value( - "visibility_library_enable_timeline_favorites_safety_level" - ) - val EnableSelfViewTimelineFavoritesSafetyLevel: Value = Value( - "visibility_library_enable_self_view_timeline_favorites_safety_level" - ) - val EnableTweetTimelineFocalTweetSafetyLevel: Value = Value( - "visibility_library_enable_timeline_focal_tweet_safety_level" - ) - val EnableTweetScopedTimelineSafetyLevel: Value = Value( - "visibility_library_enable_tweet_scoped_timeline_safety_level" - ) - val EnableTimelineFollowingActivitySafetyLevel: Value = Value( - "visibility_library_enable_timeline_following_activity_safety_level" - ) - val EnableTimelineHomeSafetyLevel: Value = Value( - "visibility_library_enable_timeline_home_safety_level" - ) - val EnableTimelineHomeCommunitiesSafetyLevel: Value = Value( - "visibility_library_enable_timeline_home_communities_safety_level" - ) - val EnableTimelineHomeHydrationSafetyLevel: Value = Value( - "visibility_library_enable_timeline_home_hydration_safety_level" - ) - val EnableTimelineHomeLatestSafetyLevel: Value = Value( - "visibility_library_enable_timeline_home_latest_safety_level" - ) - val EnableTimelineHomeRecommendationsSafetyLevel: Value = Value( - "visibility_library_enable_timeline_home_recommendations_safety_level" - ) - val EnableTimelineHomeTopicFollowRecommendationsSafetyLevel: Value = Value( - "visibility_library_enable_timeline_home_topic_follow_recommendations_safety_level" - ) - val EnableTimelineScorerSafetyLevel: Value = Value( - "visibility_library_enable_timeline_scorer_safety_level" - ) - val EnableTopicsLandingPageTopicRecommendationsSafetyLevel: Value = Value( - "visibility_library_enable_topics_landing_page_topic_recommendations_safety_level" - ) - val EnableExploreRecommendationsSafetyLevel: Value = Value( - "visibility_library_enable_explore_recommendations_safety_level" - ) - val EnableTimelineInjectionSafetyLevel: Value = Value( - "visibility_library_enable_timeline_injection_safety_level" - ) - val EnableTimelineLikedBySafetyLevel: Value = Value( - "visibility_library_enable_timeline_liked_by_safety_level" - ) - val EnableTimelineListsSafetyLevel: Value = Value( - "visibility_library_enable_timeline_lists_safety_level" - ) - val EnableTimelineMediaSafetyLevel: Value = Value( - "visibility_library_enable_timeline_media_safety_level" - ) - val EnableTimelineMentionsSafetyLevel: Value = Value( - "visibility_library_enable_timeline_mentions_safety_level" - ) - val EnableTimelineModeratedTweetsHydrationSafetyLevel: Value = Value( - "visibility_library_enable_timeline_moderated_tweets_hydration_safety_level" - ) - val EnableTimelineProfileSafetyLevel: Value = Value( - "visibility_library_enable_timeline_profile_safety_level" - ) - val EnableTimelineProfileAllSafetyLevel: Value = Value( - "visibility_library_enable_timeline_profile_all_safety_level" - ) - val EnableTimelineProfileSpacesSafetyLevel: Value = Value( - "visibility_library_enable_timeline_profile_spaces_safety_level") - val EnableTimelineProfileSuperFollowsSafetyLevel: Value = Value( - "visibility_library_enable_timeline_profile_super_follows_safety_level" - ) - val EnableTimelineReactiveBlendingSafetyLevel: Value = Value( - "visibility_library_enable_timeline_reactive_blending_safety_level" - ) - val EnableTimelineRetweetedBySafetyLevel: Value = Value( - "visibility_library_enable_timeline_retweeted_by_safety_level" - ) - val EnableTimelineSuperLikedBySafetyLevel: Value = Value( - "visibility_library_enable_timeline_super_liked_by_safety_level" - ) - val EnableTombstoningSafetyLevel: Value = Value( - "visibility_library_enable_tombstoning_safety_level" - ) - val EnableTopicRecommendationsSafetyLevel: Value = Value( - "visibility_library_enable_topic_recommendations_safety_level" - ) - val EnableTrendsRepresentativeTweetSafetyLevel: Value = Value( - "visibility_library_enable_trends_representative_tweet_safety_level" - ) - val EnableTrustedFriendsUserListSafetyLevel: Value = Value( - "visibility_library_enable_trusted_friends_user_list_safety_level" - ) - val EnableTwitterDelegateUserListSafetyLevel: Value = Value( - "visibility_library_enable_twitter_delegate_user_list_safety_level" - ) - val EnableTweetDetailSafetyLevel: Value = Value( - "visibility_library_enable_tweet_detail_safety_level" - ) - val EnableTweetDetailNonTooSafetyLevel: Value = Value( - "visibility_library_enable_tweet_detail_non_too_safety_level" - ) - val EnableTweetDetailWithInjectionsHydrationSafetyLevel: Value = Value( - "visibility_library_enable_tweet_detail_with_injections_hydration_safety_level" - ) - val EnableTweetEngagersSafetyLevel: Value = Value( - "visibility_library_enable_tweet_engagers_safety_level" - ) - val EnableTweetReplyNudgeSafetyLevel: Value = Value( - "visibility_library_enable_tweet_reply_nudge_safety_level" - ) - val EnableTweetWritesApiSafetyLevel: Value = Value( - "visibility_library_enable_tweet_writes_api_safety_level" - ) - val EnableTwitterArticleComposeSafetyLevel: Value = Value( - "visibility_library_enable_twitter_article_compose_safety_level" - ) - val EnableTwitterArticleProfileTabSafetyLevel: Value = Value( - "visibility_library_enable_twitter_article_profile_tab_safety_level" - ) - val EnableTwitterArticleReadSafetyLevel: Value = Value( - "visibility_library_enable_twitter_article_read_safety_level" - ) - val EnableUserProfileHeaderSafetyLevel: Value = Value( - "visibility_library_enable_user_profile_header_safety_level" - ) - val EnableUserMilestoneRecommendationSafetyLevel: Value = Value( - "visibility_library_enable_user_milestone_recommendation_safety_level" - ) - val EnableUserScopedTimelineSafetyLevel: Value = Value( - "visibility_library_enable_user_scoped_timeline_safety_level" - ) - val EnableUserSettingsSafetyLevel: Value = Value( - "visibility_library_enable_user_settings_safety_level" - ) - val EnableVideoAdsSafetyLevel: Value = Value( - "visibility_library_enable_video_ads_safety_level" - ) - val EnableTimelineHomePromotedHydrationSafetyLevel: Value = Value( - "visibility_library_enable_timeline_home_promoted_hydration_safety_level" - ) - - val EnableSuperFollowerConnectionsSafetyLevel: Value = Value( - "visibility_library_enable_super_follower_connnections_safety_level" - ) - - val EnableSuperLikeSafetyLevel: Value = Value( - "visibility_library_enable_super_like_safety_level" - ) - - val EnableZipbirdConsumerArchivesSafetyLevel: Value = Value( - "visibility_library_enable_zipbird_consumer_archives_safety_level" - ) - - val EnableTweetAwardSafetyLevel: Value = Value( - "visibility_library_enable_tweet_award_safety_level" - ) - - val EnableTweetConversationControlRules: Value = Value( - "visibility_library_enable_conversation_control_rules" - ) - val EnableCommunityTweetsControlRules: Value = Value( - "visibility_library_enable_community_tweets_rules" - ) - val EnableDropCommunityTweetWithUndefinedCommunityRule: Value = Value( - "visibility_library_enable_drop_community_tweet_with_undefined_community_rule" - ) - val EnablePSpammyTweetDownrankConvosLowQuality: Value = Value( - "visibility_library_enable_p_spammy_tweet_downrank_convos_low_quality" - ) - val EnableHighPSpammyTweetScoreSearchTweetLabelDropRule: Value = Value( - "visibility_library_enable_high_p_spammy_tweet_score_search_tweet_label_drop_rule" - ) - - val EnableRitoActionedTweetDownrankConvosLowQuality: Value = Value( - "visibility_library_enable_rito_actioned_tweet_downrank_convos_low_quality" - ) - - val EnableNewSensitiveMediaSettingsInterstitialRulesHomeTimeline: Value = Value( - "visibility_library_enable_new_sensitive_media_settings_interstitial_rules_home_timeline") - - val EnableLegacySensitiveMediaRulesHomeTimeline: Value = Value( - "visibility_library_enable_legacy_sensitive_media_rules_home_timeline" - ) - - val EnableNewSensitiveMediaSettingsInterstitialRulesConversation: Value = Value( - "visibility_library_enable_new_sensitive_media_settings_interstitial_rules_conversation" - ) - - val EnableLegacySensitiveMediaRulesConversation: Value = Value( - "visibility_library_enable_legacy_sensitive_media_rules_conversation" - ) - - val EnableNewSensitiveMediaSettingsInterstitialRulesProfileTimeline: Value = Value( - "visibility_library_enable_new_sensitive_media_settings_interstitials_rules_profile_timeline" - ) - - val EnableLegacySensitiveMediaRulesProfileTimeline: Value = Value( - "visibility_library_enable_legacy_sensitive_media_rules_profile_timeline" - ) - - val EnableNewSensitiveMediaSettingsInterstitialRulesTweetDetail: Value = Value( - "visibility_library_enable_new_sensitive_media_settings_interstitials_rules_tweet_detail" - ) - - val EnableLegacySensitiveMediaRulesTweetDetail: Value = Value( - "visibility_library_enable_legacy_sensitive_media_rules_tweet_detail" - ) - - val EnableLegacySensitiveMediaRulesDirectMessages: Value = Value( - "visibility_library_enable_legacy_sensitive_media_rules_direct_messages" - ) - - val VisibilityLibraryEnableToxicReplyFilterConversation: Value = Value( - "visibility_library_enable_toxic_reply_filter_conversation" - ) - - val VisibilityLibraryEnableToxicReplyFilterNotifications: Value = Value( - "visibility_library_enable_toxic_reply_filter_notifications" - ) - - val EnableSmyteSpamTweetRule: Value = Value( - "visibility_library_enable_smyte_spam_tweet_rule" - ) - - val EnableHighSpammyTweetContentScoreSearchLatestTweetLabelDropRule: Value = Value( - "visibility_library_enable_high_spammy_tweet_content_score_search_latest_tweet_label_drop_rule" - ) - val EnableHighSpammyTweetContentScoreSearchTopTweetLabelDropRule: Value = Value( - "visibility_library_enable_high_spammy_tweet_content_score_search_top_tweet_label_drop_rule" - ) - val EnableHighSpammyTweetContentScoreConvoDownrankAbusiveQualityRule: Value = Value( - "visibility_library_enable_high_spammy_tweet_content_score_convo_downrank_abusive_quality_rule" - ) - - val EnableHighCryptospamScoreConvoDownrankAbusiveQualityRule: Value = Value( - "visibility_library_enable_high_cryptospam_score_convo_downrank_abusive_quality_rule" - ) - val EnableHighSpammyTweetContentScoreTrendsTopTweetLabelDropRule: Value = Value( - "visibility_library_enable_high_spammy_tweet_content_score_trends_top_tweet_label_drop_rule" - ) - - val EnableHighSpammyTweetContentScoreTrendsLatestTweetLabelDropRule: Value = Value( - "visibility_library_enable_high_spammy_tweet_content_score_trends_latest_tweet_label_drop_rule" - ) - - val EnableGoreAndViolenceTopicHighRecallTweetLabelRule: Value = Value( - "visibility_library_enable_gore_and_violence_topic_high_recall_tweet_label_rule" - ) - - val EnableLimitRepliesFollowersConversationRule: Value = Value( - "visibility_library_enable_limit_replies_followers_conversation_rule" - ) - - val EnableBlinkBadDownrankingRule: Value = Value( - "visibility_library_enable_blink_bad_downranking_rule" - ) - - val EnableBlinkWorstDownrankingRule: Value = Value( - "visibility_library_enable_blink_worst_downranking_rule" - ) - - val EnableCopypastaSpamDownrankConvosAbusiveQualityRule: Value = Value( - "visibility_library_enable_copypasta_spam_downrank_convos_abusive_quality_rule" - ) - - val EnableCopypastaSpamSearchDropRule: Value = Value( - "visibility_library_enable_copypasta_spam_search_drop_rule" - ) - - val EnableSpammyUserModelHighPrecisionDropTweetRule: Value = Value( - "visibility_library_enable_spammy_user_model_high_precision_drop_tweet_rule" - ) - - val EnableAvoidNsfwRules: Value = Value( - "visibility_library_enable_avoid_nsfw_rules" - ) - - val EnableReportedTweetInterstitialRule: Value = Value( - "visibility_library_enable_reported_tweet_interstitial_rule" - ) - - val EnableReportedTweetInterstitialSearchRule: Value = Value( - "visibility_library_enable_reported_tweet_interstitial_search_rule" - ) - - val EnableDropExclusiveTweetContentRule: Value = Value( - "visibility_library_enable_drop_exclusive_tweet_content_rule" - ) - - val EnableDropExclusiveTweetContentRuleFailClosed: Value = Value( - "visibility_library_enable_drop_exclusive_tweet_content_rule_fail_closed" - ) - - val EnableTombstoneExclusiveQtProfileTimelineParam: Value = Value( - "visibility_library_enable_tombstone_exclusive_quoted_tweet_content_rule" - ) - - val EnableDropAllExclusiveTweetsRule: Value = Value( - "visibility_library_enable_drop_all_exclusive_tweets_rule" - ) - - val EnableDropAllExclusiveTweetsRuleFailClosed: Value = Value( - "visibility_library_enable_drop_all_exclusive_tweets_rule_fail_closed" - ) - - val EnableDownrankSpamReplySectioningRule: Value = Value( - "visibility_library_enable_downrank_spam_reply_sectioning_rule" - ) - - val EnableNsfwTextSectioningRule: Value = Value( - "visibility_library_enable_nsfw_text_sectioning_rule" - ) - - val EnableNsfwAgeBasedMediaDropRules: Value = Value( - "visibility_library_enable_nsfw_age_based_media_drop_rules" - ) - - val EnableNsfwUnderageRules: Value = Value( - "visibility_library_enable_nsfw_underage_rules" - ) - - val EnableNsfwUnderageMediaRules: Value = Value( - "visibility_library_enable_nsfw_underage_media_rules" - ) - - val EnableNsfwMissingAgeRules: Value = Value( - "visibility_library_enable_nsfw_missing_age_rules" - ) - - val EnableNsfwMissingAgeMediaRules: Value = Value( - "visibility_library_enable_nsfw_missing_age_media_rules" - ) - - val EnableSearchIpiSafeSearchWithoutUserInQueryDropRule: Value = Value( - "visibility_library_enable_search_ipi_safe_search_without_user_in_query_drop_rule" - ) - - val EnableTimelineHomePromotedTweetHealthEnforcementRules: Value = Value( - "visibility_library_enable_timeline_home_promoted_tweet_health_enforcement_rules" - ) - - val EnableMutedKeywordFilteringSpaceTitleNotificationsRule: Value = Value( - "visibility_library_enable_muted_keyword_filtering_space_title_notifications_rule" - ) - - val EnableDropTweetsWithGeoRestrictedMediaRule: Value = Value( - "visibility_library_enable_drop_tweets_with_georestricted_media_rule" - ) - val EnableDropAllTrustedFriendsTweetsRule: Value = Value( - "visibility_library_enable_drop_all_trusted_friends_tweets_rule" - ) - - val EnableDropTrustedFriendsTweetContentRule: Value = Value( - "visibility_library_enable_drop_all_trusted_friends_tweet_content_rule" - ) - - val EnableDropCollabInvitationTweetsRule: Value = Value( - "visibility_library_enable_drop_all_collab_invitation_tweets_rule" - ) - - val EnableFetchTweetReportedPerspective: Value = Value( - "visibility_library_enable_fetch_tweet_reported_perspective" - ) - - val EnableFetchTweetMediaMetadata: Value = Value( - "visibility_library_enable_fetch_tweet_media_metadata" - ) - - val VisibilityLibraryEnableFollowCheckInMutedKeyword: Value = Value( - "visibility_library_enable_follow_check_in_mutedkeyword" - ) - - val VisibilityLibraryEnableMediaInterstitialComposition: Value = Value( - "visibility_library_enable_media_interstitial_composition" - ) - - val EnableVerdictScribingFromTweetVisibilityLibrary: Value = Value( - "visibility_library_enable_verdict_scribing_from_tweet_visibility_library" - ) - - val EnableVerdictLoggerEventPublisherInstantiationFromTweetVisibilityLibrary: Value = Value( - "visibility_library_enable_verdict_logger_event_publisher_instantiation_from_tweet_visibility_library" - ) - - val EnableVerdictScribingFromTimelineConversationsVisibilityLibrary: Value = Value( - "visibility_library_enable_verdict_scribing_from_timeline_conversations_visibility_library" - ) - - val EnableVerdictLoggerEventPublisherInstantiationFromTimelineConversationsVisibilityLibrary: Value = - Value( - "visibility_library_enable_verdict_logger_event_publisher_instantiation_from_timeline_conversations_visibility_library" - ) - - val EnableVerdictScribingFromBlenderVisibilityLibrary: Value = Value( - "visibility_library_enable_verdict_scribing_from_blender_visibility_library" - ) - - val EnableVerdictScribingFromSearchVisibilityLibrary: Value = Value( - "visibility_library_enable_verdict_scribing_from_search_visibility_library" - ) - - val EnableVerdictLoggerEventPublisherInstantiationFromBlenderVisibilityLibrary: Value = Value( - "visibility_library_enable_verdict_logger_event_publisher_instantiation_from_blender_visibility_library" - ) - - val EnableVerdictLoggerEventPublisherInstantiationFromSearchVisibilityLibrary: Value = Value( - "visibility_library_enable_verdict_logger_event_publisher_instantiation_from_search_visibility_library" - ) - - val EnableLocalizedTombstoneOnVisibilityResults: Value = Value( - "visibility_library_enable_localized_tombstones_on_visibility_results" - ) - - val EnableShortCircuitingFromTweetVisibilityLibrary: Value = Value( - "visibility_library_enable_short_circuiting_from_tweet_visibility_library" - ) - - val EnableShortCircuitingFromTimelineConversationsVisibilityLibrary: Value = Value( - "visibility_library_enable_short_circuiting_from_timeline_conversations_visibility_library" - ) - - val EnableShortCircuitingFromBlenderVisibilityLibrary: Value = Value( - "visibility_library_enable_short_circuiting_from_blender_visibility_library" - ) - - val EnableShortCircuitingFromSearchVisibilityLibrary: Value = Value( - "visibility_library_enable_short_circuiting_from_search_visibility_library" - ) - - val EnableNsfwTextHighPrecisionDropRule: Value = Value( - "visibility_library_enable_nsfw_text_high_precision_drop_rule" - ) - - val EnableSpammyTweetRuleVerdictLogging: Value = Value( - "visibility_library_enable_spammy_tweet_rule_verdict_logging" - ) - - val EnableDownlevelRuleVerdictLogging: Value = Value( - "visibility_library_enable_downlevel_rule_verdict_logging" - ) - - val EnableLikelyIvsUserLabelDropRule: Value = Value( - "visibility_library_enable_likely_likely_ivs_user_label_drop_rule" - ) - - val EnableCardVisibilityLibraryCardUriParsing: Value = Value( - "visibility_library_enable_card_visibility_library_card_uri_parsing" - ) - - val EnableCardUriRootDomainDenylistRule: Value = Value( - "visibility_library_enable_card_uri_root_domain_deny_list_rule" - ) - - val EnableCommunityNonMemberPollCardRule: Value = Value( - "visibility_library_enable_community_non_member_poll_card_rule" - ) - - val EnableCommunityNonMemberPollCardRuleFailClosed: Value = Value( - "visibility_library_enable_community_non_member_poll_card_rule_fail_closed" - ) - - val EnableExperimentalNudgeLabelRule: Value = Value( - "visibility_library_enable_experimental_nudge_label_rule" - ) - - val NsfwHighPrecisionUserLabelAvoidTweetRuleEnabledParam: Value = Value( - "visibility_library_nsfw_high_precision_user_label_avoid_tweet_rule_enabled" - ) - - val EnableUserSelfViewOnlySafetyLevel: Value = Value( - "visibility_library_enable_user_self_view_only_safety_level" - ) - - val EnableNewAdAvoidanceRules: Value = Value( - "visibility_library_enable_new_ad_avoidance_rules" - ) - - val EnableNsfaHighRecallAdAvoidanceParam: Value = Value( - "visibility_library_enable_nsfa_high_recall_ad_avoidance_rules" - ) - - val EnableNsfaKeywordsHighPrecisionAdAvoidanceParam: Value = Value( - "visibility_library_enable_nsfa_keywords_high_precision_ad_avoidance_rules" - ) - - val EnableStaleTweetDropRuleParam: Value = Value( - "visibility_library_enable_stale_tweet_drop_rule" - ) - - val EnableStaleTweetDropRuleFailClosedParam: Value = Value( - "visibility_library_enable_stale_tweet_drop_rule_fail_closed" - ) - - val EnableEditHistoryTimelineSafetyLevel: Value = Value( - "visibility_library_enable_edit_history_timeline_safety_level" - ) - - val EnableDeleteStateTweetRules: Value = Value( - "visibility_library_enable_delete_state_tweet_rules" - ) - - val EnableSpacesSharingNsfwDropRulesParam: Value = Value( - "visibility_library_enable_spaces_sharing_nsfw_drop_rule" - ) - - val EnableDropMediaLegalRulesParam: Value = Value( - "visibility_library_enable_drop_media_legal_rules" - ) - - val EnableTombstoneMediaLegalRulesParam: Value = Value( - "visibility_library_enable_tombstone_media_legal_rules" - ) - - val EnableInterstitialMediaLegalRulesParam: Value = Value( - "visibility_library_enable_interstitial_media_legal_rules" - ) - - val EnableViewerIsSoftUserDropRuleParam: Value = Value( - "visibility_library_enable_viewer_is_soft_user_drop_rule" - ) - - val EnableBackendLimitedActions: Value = Value( - "visibility_library_enable_backend_limited_actions" - ) - - val EnableNotificationsQig: Value = Value( - "visibility_library_enable_notifications_qig_safety_level" - ) - - val EnablePdnaQuotedTweetTombstoneRule: Value = Value( - "visibility_library_enable_pdna_quoted_tweet_tombstone_rule" - ) - - val EnableSpamQuotedTweetTombstoneRule: Value = Value( - "visibility_library_enable_spam_quoted_tweet_tombstone_rule" - ) - - val EnableNsfwHpQuotedTweetDropRule: Value = Value( - "visibility_library_enable_nsfw_hp_quoted_tweet_drop_experiment_rule" - ) - val EnableNsfwHpQuotedTweetTombstoneRule: Value = Value( - "visibility_library_enable_nsfw_hp_quoted_tweet_tombstone_experiment_rule" - ) - - val EnableExperimentalRuleEngine: Value = Value( - "visibility_library_enable_experimental_rule_engine" - ) - - val EnableFosnrRules: Value = Value( - "visibility_library_enable_fosnr_rules" - ) - - val EnableInnerQuotedTweetViewerBlocksAuthorInterstitialRule: Value = Value( - "visibility_library_enable_inner_quoted_tweet_viewer_blocks_author_interstitial_rule" - ) - - val EnableInnerQuotedTweetViewerMutesAuthorInterstitialRule: Value = Value( - "visibility_library_enable_inner_quoted_tweet_viewer_mutes_author_interstitial_rule" - ) - - val EnableLocalizedInterstitialGenerator: Value = Value( - "visibility_library_enable_localized_interstitial_generator" - ) - - val EnableProfileMixeMediaSafetyLevel: Value = Value( - "visibility_library_enable_profile_mixer_media_safety_level") - - val EnableConvosLocalizedInterstitial: Value = Value( - "visibility_library_convos_enable_localized_interstitial" - ) - - val EnableConvosLegacyInterstitial: Value = Value( - "visibility_library_convos_enable_legacy_interstitial" - ) - - val DisableLegacyInterstitialFilteredReason: Value = Value( - "visibility_library_disable_legacy_interstitial_filtered_reason" - ) - - val EnableSearchBasicBlockMuteRules: Value = Value( - "visibility_library_enable_search_basic_block_mute_rules" - ) - - val EnableLocalizedInterstitialInUserStateLib: Value = Value( - "visibility_library_enable_localized_interstitial_user_state_lib" - ) - - val EnableProfileMixerFavoritesSafetyLevel: Value = Value( - "visibility_library_enable_profile_mixer_favorites_safety_level") - - val EnableAbusiveBehaviorDropRule: Value = Value( - "visibility_library_enable_abusive_behavior_drop_rule" - ) - - val EnableAbusiveBehaviorInterstitialRule: Value = Value( - "visibility_library_enable_abusive_behavior_interstitial_rule" - ) - - val EnableAbusiveBehaviorLimitedEngagementsRule: Value = Value( - "visibility_library_enable_abusive_behavior_limited_engagements_rule" - ) - - val EnableNotGraduatedDownrankConvosAbusiveQualityRule: Value = Value( - "visibility_library_enable_not_graduated_downrank_convos_abusive_quality_rule" - ) - - val EnableNotGraduatedSearchDropRule: Value = Value( - "visibility_library_enable_not_graduated_search_drop_rule" - ) - - val EnableNotGraduatedDropRule: Value = Value( - "visibility_library_enable_not_graduated_drop_rule" - ) - - val EnableMemoizeSafetyLevelParams: Value = Value( - "visibility_library_enable_memoize_safety_level_params" - ) - - val EnableAuthorBlocksViewerDropRule: Value = Value( - "visibility_library_enable_author_blocks_viewer_drop_rule" - ) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/ExperimentsHelper.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/ExperimentsHelper.docx new file mode 100644 index 000000000..ace862843 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/ExperimentsHelper.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/ExperimentsHelper.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/ExperimentsHelper.scala deleted file mode 100644 index f2240311b..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/ExperimentsHelper.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.visibility.configapi.configs - -import com.twitter.timelines.configapi.Config -import com.twitter.timelines.configapi.ExperimentConfigBuilder -import com.twitter.timelines.configapi.Param -import com.twitter.visibility.configapi.params.VisibilityExperiment -import com.twitter.visibility.models.SafetyLevel - -object ExperimentsHelper { - - def mkABExperimentConfig(experiment: VisibilityExperiment, param: Param[Boolean]): Config = { - ExperimentConfigBuilder(experiment) - .addBucket( - experiment.ControlBucket, - param := true - ) - .addBucket( - experiment.TreatmentBucket, - param := false - ) - .build - } - - def mkABExperimentConfig(experiment: VisibilityExperiment, safetyLevel: SafetyLevel): Config = - mkABExperimentConfig(experiment, safetyLevel.enabledParam) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciderGates.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciderGates.docx new file mode 100644 index 000000000..74a448ade Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciderGates.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciderGates.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciderGates.scala deleted file mode 100644 index 7df596ce0..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciderGates.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.visibility.configapi.configs - -import com.twitter.decider.Decider -import com.twitter.servo.gate.DeciderGate -import com.twitter.servo.util.Gate - -case class VisibilityDeciderGates(decider: Decider) { - import DeciderKey._ - - private[this] def feature(deciderKey: Value) = decider.feature(deciderKey.toString) - - val enableFetchTweetReportedPerspective: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.EnableFetchTweetReportedPerspective)) - val enableFetchTweetMediaMetadata: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.EnableFetchTweetMediaMetadata)) - val enableFollowCheckInMutedKeyword: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.VisibilityLibraryEnableFollowCheckInMutedKeyword)) - val enableMediaInterstitialComposition: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.VisibilityLibraryEnableMediaInterstitialComposition)) - val enableExperimentalRuleEngine: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.EnableExperimentalRuleEngine)) - - val enableLocalizedInterstitialGenerator: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.EnableLocalizedInterstitialGenerator)) - - val enableShortCircuitingTVL: Gate[Unit] = - DeciderGate.linear(feature(EnableShortCircuitingFromTweetVisibilityLibrary)) - val enableVerdictLoggerTVL = DeciderGate.linear( - feature(DeciderKey.EnableVerdictLoggerEventPublisherInstantiationFromTweetVisibilityLibrary)) - val enableVerdictScribingTVL = - DeciderGate.linear(feature(DeciderKey.EnableVerdictScribingFromTweetVisibilityLibrary)) - val enableBackendLimitedActions = - DeciderGate.linear(feature(DeciderKey.EnableBackendLimitedActions)) - val enableMemoizeSafetyLevelParams: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.EnableMemoizeSafetyLevelParams)) - - val enableShortCircuitingBVL: Gate[Unit] = - DeciderGate.linear(feature(EnableShortCircuitingFromBlenderVisibilityLibrary)) - val enableVerdictLoggerBVL = DeciderGate.linear( - feature(DeciderKey.EnableVerdictLoggerEventPublisherInstantiationFromBlenderVisibilityLibrary)) - val enableVerdictScribingBVL = - DeciderGate.linear(feature(DeciderKey.EnableVerdictScribingFromBlenderVisibilityLibrary)) - - val enableShortCircuitingSVL: Gate[Unit] = - DeciderGate.linear(feature(EnableShortCircuitingFromSearchVisibilityLibrary)) - val enableVerdictLoggerSVL = DeciderGate.linear( - feature(DeciderKey.EnableVerdictLoggerEventPublisherInstantiationFromSearchVisibilityLibrary)) - val enableVerdictScribingSVL = - DeciderGate.linear(feature(DeciderKey.EnableVerdictScribingFromSearchVisibilityLibrary)) - - val enableShortCircuitingTCVL: Gate[Unit] = - DeciderGate.linear(feature(EnableShortCircuitingFromTimelineConversationsVisibilityLibrary)) - val enableVerdictLoggerTCVL = DeciderGate.linear(feature( - DeciderKey.EnableVerdictLoggerEventPublisherInstantiationFromTimelineConversationsVisibilityLibrary)) - val enableVerdictScribingTCVL = - DeciderGate.linear( - feature(DeciderKey.EnableVerdictScribingFromTimelineConversationsVisibilityLibrary)) - - val enableCardVisibilityLibraryCardUriParsing = - DeciderGate.linear(feature(DeciderKey.EnableCardVisibilityLibraryCardUriParsing)) - - val enableConvosLocalizedInterstitial: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.EnableConvosLocalizedInterstitial)) - - val enableConvosLegacyInterstitial: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.EnableConvosLegacyInterstitial)) - - val disableLegacyInterstitialFilteredReason: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.DisableLegacyInterstitialFilteredReason)) - - val enableLocalizedInterstitialInUserStateLibrary: Gate[Unit] = - DeciderGate.linear(feature(DeciderKey.EnableLocalizedInterstitialInUserStateLib)) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciders.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciders.docx new file mode 100644 index 000000000..3fc861b31 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciders.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciders.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciders.scala deleted file mode 100644 index e359d443d..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityDeciders.scala +++ /dev/null @@ -1,390 +0,0 @@ -package com.twitter.visibility.configapi.configs - -import com.twitter.decider.Recipient -import com.twitter.decider.SimpleRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.BaseRequestContext -import com.twitter.timelines.configapi.Config -import com.twitter.timelines.configapi.Param -import com.twitter.timelines.configapi.WithGuestId -import com.twitter.timelines.configapi.WithUserId -import com.twitter.timelines.configapi.decider.DeciderSwitchOverrideValue -import com.twitter.timelines.configapi.decider.GuestRecipient -import com.twitter.timelines.configapi.decider.RecipientBuilder -import com.twitter.visibility.configapi.params.RuleParams -import com.twitter.visibility.configapi.params.TimelineConversationsDownrankingSpecificParams -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.SafetyLevel._ - -private[visibility] object VisibilityDeciders { - val SafetyLevelToDeciderMap: Map[SafetyLevel, DeciderKey.Value] = Map( - AllSubscribedLists -> DeciderKey.EnableAllSubscribedListsSafetyLevel, - AccessInternalPromotedContent -> DeciderKey.EnableAccessInternalPromotedContentSafetyLevel, - AdsBusinessSettings -> DeciderKey.EnableAdsBusinessSettingsSafetyLevel, - AdsCampaign -> DeciderKey.EnableAdsCampaignSafetyLevel, - AdsManager -> DeciderKey.EnableAdsManagerSafetyLevel, - AdsReportingDashboard -> DeciderKey.EnableAdsReportingDashboardSafetyLevel, - Appeals -> DeciderKey.EnableAppealsSafetyLevel, - ArticleTweetTimeline -> DeciderKey.EnableArticleTweetTimelineSafetyLevel, - BaseQig -> DeciderKey.EnableBaseQig, - BirdwatchNoteAuthor -> DeciderKey.EnableBirdwatchNoteAuthorSafetyLevel, - BirdwatchNoteTweetsTimeline -> DeciderKey.EnableBirdwatchNoteTweetsTimelineSafetyLevel, - BirdwatchNeedsYourHelpNotifications -> DeciderKey.EnableBirdwatchNeedsYourHelpNotificationsSafetyLevel, - BlockMuteUsersTimeline -> DeciderKey.EnableBlockMuteUsersTimelineSafetyLevel, - BrandSafety -> DeciderKey.EnableBrandSafetySafetyLevel, - CardPollVoting -> DeciderKey.EnableCardPollVotingSafetyLevel, - CardsService -> DeciderKey.EnableCardsServiceSafetyLevel, - Communities -> DeciderKey.EnableCommunitiesSafetyLevel, - ContentControlToolInstall -> DeciderKey.EnableContentControlToolInstallSafetyLevel, - ConversationFocalPrehydration -> DeciderKey.EnableConversationFocalPrehydrationSafetyLevel, - ConversationFocalTweet -> DeciderKey.EnableConversationFocalTweetSafetyLevel, - ConversationInjectedTweet -> DeciderKey.EnableConversationInjectedTweetSafetyLevel, - ConversationReply -> DeciderKey.EnableConversationReplySafetyLevel, - CuratedTrendsRepresentativeTweet -> DeciderKey.EnableCuratedTrendsRepresentativeTweet, - CurationPolicyViolations -> DeciderKey.EnableCurationPolicyViolations, - DeprecatedSafetyLevel -> DeciderKey.EnableDeprecatedSafetyLevelSafetyLevel, - DevPlatformGetListTweets -> DeciderKey.EnableDevPlatformGetListTweetsSafetyLevel, - DesFollowingAndFollowersUserList -> DeciderKey.EnableDesFollowingAndFollowersUserListSafetyLevel, - DesHomeTimeline -> DeciderKey.EnableDesHomeTimelineSafetyLevel, - DesQuoteTweetTimeline -> DeciderKey.EnableDesQuoteTweetTimelineSafetyLevel, - DesRealtime -> DeciderKey.EnableDesRealtimeSafetyLevel, - DesRealtimeSpamEnrichment -> DeciderKey.EnableDesRealtimeSpamEnrichmentSafetyLevel, - DesRealtimeTweetFilter -> DeciderKey.EnableDesRealtimeTweetFilterSafetyLevel, - DesRetweetingUsers -> DeciderKey.EnableDesRetweetingUsersSafetyLevel, - DesTweetDetail -> DeciderKey.EnableDesTweetDetailSafetyLevel, - DesTweetLikingUsers -> DeciderKey.EnableDesTweetLikingUsersSafetyLevel, - DesUserBookmarks -> DeciderKey.EnableDesUserBookmarksSafetyLevel, - DesUserLikedTweets -> DeciderKey.EnableDesUserLikedTweetsSafetyLevel, - DesUserMentions -> DeciderKey.EnableDesUserMentionsSafetyLevel, - DesUserTweets -> DeciderKey.EnableDesUserTweetsSafetyLevel, - DevPlatformComplianceStream -> DeciderKey.EnableDevPlatformComplianceStreamSafetyLevel, - DirectMessages -> DeciderKey.EnableDirectMessagesSafetyLevel, - DirectMessagesConversationList -> DeciderKey.EnableDirectMessagesConversationListSafetyLevel, - DirectMessagesConversationTimeline -> DeciderKey.EnableDirectMessagesConversationTimelineSafetyLevel, - DirectMessagesInbox -> DeciderKey.EnableDirectMessagesInboxSafetyLevel, - DirectMessagesMutedUsers -> DeciderKey.EnableDirectMessagesMutedUsersSafetyLevel, - DirectMessagesPinned -> DeciderKey.EnableDirectMessagesPinnedSafetyLevel, - DirectMessagesSearch -> DeciderKey.EnableDirectMessagesSearchSafetyLevel, - EditHistoryTimeline -> DeciderKey.EnableEditHistoryTimelineSafetyLevel, - ElevatedQuoteTweetTimeline -> DeciderKey.EnableElevatedQuoteTweetTimelineSafetyLevel, - EmbeddedTweet -> DeciderKey.EnableEmbeddedTweetSafetyLevel, - EmbedsPublicInterestNotice -> DeciderKey.EnableEmbedsPublicInterestNoticeSafetyLevel, - EmbedTweetMarkup -> DeciderKey.EnableEmbedTweetMarkupSafetyLevel, - FilterAll -> DeciderKey.EnableFilterAllSafetyLevel, - FilterAllPlaceholder -> DeciderKey.EnableFilterAllPlaceholderSafetyLevel, - FilterNone -> DeciderKey.EnableFilterNoneSafetyLevel, - FilterDefault -> DeciderKey.EnableFilterDefaultSafetyLevel, - FollowedTopicsTimeline -> DeciderKey.EnableFollowedTopicsTimelineSafetyLevel, - FollowerConnections -> DeciderKey.EnableFollowerConnectionsSafetyLevel, - FollowingAndFollowersUserList -> DeciderKey.EnableFollowingAndFollowersUserListSafetyLevel, - ForDevelopmentOnly -> DeciderKey.EnableForDevelopmentOnlySafetyLevel, - FriendsFollowingList -> DeciderKey.EnableFriendsFollowingListSafetyLevel, - GraphqlDefault -> DeciderKey.EnableGraphqlDefaultSafetyLevel, - GryphonDecksAndColumns -> DeciderKey.EnableGryphonDecksAndColumnsSafetyLevel, - HumanizationNudge -> DeciderKey.EnableHumanizationNudgeSafetyLevel, - KitchenSinkDevelopment -> DeciderKey.EnableKitchenSinkDevelopmentSafetyLevel, - ListHeader -> DeciderKey.EnableListHeaderSafetyLevel, - ListMemberships -> DeciderKey.EnableListMembershipsSafetyLevel, - ListOwnerships -> DeciderKey.EnableListOwnershipsSafetyLevel, - ListRecommendations -> DeciderKey.EnableListRecommendationsSafetyLevel, - ListSearch -> DeciderKey.EnableListSearchSafetyLevel, - ListSubscriptions -> DeciderKey.EnableListSubscriptionsSafetyLevel, - LiveVideoTimeline -> DeciderKey.EnableLiveVideoTimelineSafetyLevel, - LivePipelineEngagementCounts -> DeciderKey.EnableLivePipelineEngagementCountsSafetyLevel, - MagicRecs -> DeciderKey.EnableMagicRecsSafetyLevel, - MagicRecsAggressive -> DeciderKey.EnableMagicRecsAggressiveSafetyLevel, - MagicRecsAggressiveV2 -> DeciderKey.EnableMagicRecsAggressiveV2SafetyLevel, - MagicRecsV2 -> DeciderKey.EnableMagicRecsV2SafetyLevel, - Minimal -> DeciderKey.EnableMinimalSafetyLevel, - ModeratedTweetsTimeline -> DeciderKey.EnableModeratedTweetsTimelineSafetyLevel, - Moments -> DeciderKey.EnableMomentsSafetyLevel, - NearbyTimeline -> DeciderKey.EnableNearbyTimelineSafetyLevel, - NewUserExperience -> DeciderKey.EnableNewUserExperienceSafetyLevel, - NotificationsIbis -> DeciderKey.EnableNotificationsIbisSafetyLevel, - NotificationsPlatform -> DeciderKey.EnableNotificationsPlatformSafetyLevel, - NotificationsPlatformPush -> DeciderKey.EnableNotificationsPlatformPushSafetyLevel, - NotificationsQig -> DeciderKey.EnableNotificationsQig, - NotificationsRead -> DeciderKey.EnableNotificationsReadSafetyLevel, - NotificationsTimelineDeviceFollow -> DeciderKey.EnableNotificationsTimelineDeviceFollowSafetyLevel, - NotificationsWrite -> DeciderKey.EnableNotificationsWriteSafetyLevel, - NotificationsWriterV2 -> DeciderKey.EnableNotificationsWriterV2SafetyLevel, - NotificationsWriterTweetHydrator -> DeciderKey.EnableNotificationsWriterTweetHydratorSafetyLevel, - ProfileMixerMedia -> DeciderKey.EnableProfileMixeMediaSafetyLevel, - ProfileMixerFavorites -> DeciderKey.EnableProfileMixerFavoritesSafetyLevel, - QuickPromoteTweetEligibility -> DeciderKey.EnableQuickPromoteTweetEligibilitySafetyLevel, - QuoteTweetTimeline -> DeciderKey.EnableQuoteTweetTimelineSafetyLevel, - QuotedTweetRules -> DeciderKey.EnableQuotedTweetRulesSafetyLevel, - Recommendations -> DeciderKey.EnableRecommendationsSafetyLevel, - RecosVideo -> DeciderKey.EnableRecosVideoSafetyLevel, - RecosWritePath -> DeciderKey.EnableRecosWritePathSafetyLevel, - RepliesGrouping -> DeciderKey.EnableRepliesGroupingSafetyLevel, - ReportCenter -> DeciderKey.EnableReportCenterSafetyLevel, - ReturningUserExperience -> DeciderKey.EnableReturningUserExperienceSafetyLevel, - ReturningUserExperienceFocalTweet -> DeciderKey.EnableReturningUserExperienceFocalTweetSafetyLevel, - Revenue -> DeciderKey.EnableRevenueSafetyLevel, - RitoActionedTweetTimeline -> DeciderKey.EnableRitoActionedTweetTimelineSafetyLevel, - SafeSearchMinimal -> DeciderKey.EnableSafeSearchMinimalSafetyLevel, - SafeSearchStrict -> DeciderKey.EnableSafeSearchStrictSafetyLevel, - SearchMixerSrpMinimal -> DeciderKey.EnableSearchMixerSrpMinimalSafetyLevel, - SearchMixerSrpStrict -> DeciderKey.EnableSearchMixerSrpStrictSafetyLevel, - SearchHydration -> DeciderKey.EnableSearchHydration, - SearchLatest -> DeciderKey.EnableSearchLatest, - SearchPeopleSrp -> DeciderKey.EnableSearchPeopleSrp, - SearchPeopleTypeahead -> DeciderKey.EnableSearchPeopleTypeahead, - SearchPhoto -> DeciderKey.EnableSearchPhoto, - SearchTrendTakeoverPromotedTweet -> DeciderKey.EnableSearchTrendTakeoverPromotedTweet, - SearchTop -> DeciderKey.EnableSearchTop, - SearchTopQig -> DeciderKey.EnableSearchTopQig, - SearchVideo -> DeciderKey.EnableSearchVideo, - SearchBlenderUserRules -> DeciderKey.EnableSearchLatestUserRules, - SearchLatestUserRules -> DeciderKey.EnableSearchLatestUserRules, - ShoppingManagerSpyMode -> DeciderKey.EnableShoppingManagerSpyModeSafetyLevel, - SignalsReactions -> DeciderKey.EnableSignalsReactions, - SignalsTweetReactingUsers -> DeciderKey.EnableSignalsTweetReactingUsers, - SocialProof -> DeciderKey.EnableSocialProof, - SoftInterventionPivot -> DeciderKey.EnableSoftInterventionPivot, - SpaceFleetline -> DeciderKey.EnableSpaceFleetlineSafetyLevel, - SpaceHomeTimelineUpranking -> DeciderKey.EnableSpaceHomeTimelineUprankingSafetyLevel, - SpaceJoinScreen -> DeciderKey.EnableSpaceJoinScreenSafetyLevel, - SpaceNotifications -> DeciderKey.EnableSpaceNotificationsSafetyLevel, - Spaces -> DeciderKey.EnableSpacesSafetyLevel, - SpacesParticipants -> DeciderKey.EnableSpacesParticipantsSafetyLevel, - SpacesSellerApplicationStatus -> DeciderKey.EnableSpacesSellerApplicationStatus, - SpacesSharing -> DeciderKey.EnableSpacesSharingSafetyLevel, - SpaceTweetAvatarHomeTimeline -> DeciderKey.EnableSpaceTweetAvatarHomeTimelineSafetyLevel, - StickersTimeline -> DeciderKey.EnableStickersTimelineSafetyLevel, - StratoExtLimitedEngagements -> DeciderKey.EnableStratoExtLimitedEngagementsSafetyLevel, - StreamServices -> DeciderKey.EnableStreamServicesSafetyLevel, - SuperFollowerConnections -> DeciderKey.EnableSuperFollowerConnectionsSafetyLevel, - SuperLike -> DeciderKey.EnableSuperLikeSafetyLevel, - Test -> DeciderKey.EnableTestSafetyLevel, - TimelineContentControls -> DeciderKey.EnableTimelineContentControlsSafetyLevel, - TimelineConversations -> DeciderKey.EnableTimelineConversationsSafetyLevel, - TimelineConversationsDownranking -> DeciderKey.EnableTimelineConversationsDownrankingSafetyLevel, - TimelineConversationsDownrankingMinimal -> DeciderKey.EnableTimelineConversationsDownrankingMinimalSafetyLevel, - TimelineFollowingActivity -> DeciderKey.EnableTimelineFollowingActivitySafetyLevel, - TimelineHome -> DeciderKey.EnableTimelineHomeSafetyLevel, - TimelineHomeCommunities -> DeciderKey.EnableTimelineHomeCommunitiesSafetyLevel, - TimelineHomeHydration -> DeciderKey.EnableTimelineHomeHydrationSafetyLevel, - TimelineHomePromotedHydration -> DeciderKey.EnableTimelineHomePromotedHydrationSafetyLevel, - TimelineHomeRecommendations -> DeciderKey.EnableTimelineHomeRecommendationsSafetyLevel, - TimelineHomeTopicFollowRecommendations -> DeciderKey.EnableTimelineHomeTopicFollowRecommendationsSafetyLevel, - TimelineScorer -> DeciderKey.EnableTimelineScorerSafetyLevel, - TopicsLandingPageTopicRecommendations -> DeciderKey.EnableTopicsLandingPageTopicRecommendationsSafetyLevel, - ExploreRecommendations -> DeciderKey.EnableExploreRecommendationsSafetyLevel, - TimelineInjection -> DeciderKey.EnableTimelineInjectionSafetyLevel, - TimelineMentions -> DeciderKey.EnableTimelineMentionsSafetyLevel, - TimelineModeratedTweetsHydration -> DeciderKey.EnableTimelineModeratedTweetsHydrationSafetyLevel, - TimelineHomeLatest -> DeciderKey.EnableTimelineHomeLatestSafetyLevel, - TimelineLikedBy -> DeciderKey.EnableTimelineLikedBySafetyLevel, - TimelineRetweetedBy -> DeciderKey.EnableTimelineRetweetedBySafetyLevel, - TimelineSuperLikedBy -> DeciderKey.EnableTimelineSuperLikedBySafetyLevel, - TimelineBookmark -> DeciderKey.EnableTimelineBookmarkSafetyLevel, - TimelineMedia -> DeciderKey.EnableTimelineMediaSafetyLevel, - TimelineReactiveBlending -> DeciderKey.EnableTimelineReactiveBlendingSafetyLevel, - TimelineFavorites -> DeciderKey.EnableTimelineFavoritesSafetyLevel, - TimelineFavoritesSelfView -> DeciderKey.EnableSelfViewTimelineFavoritesSafetyLevel, - TimelineLists -> DeciderKey.EnableTimelineListsSafetyLevel, - TimelineProfile -> DeciderKey.EnableTimelineProfileSafetyLevel, - TimelineProfileAll -> DeciderKey.EnableTimelineProfileAllSafetyLevel, - TimelineProfileSpaces -> DeciderKey.EnableTimelineProfileSpacesSafetyLevel, - TimelineProfileSuperFollows -> DeciderKey.EnableTimelineProfileSuperFollowsSafetyLevel, - TimelineFocalTweet -> DeciderKey.EnableTweetTimelineFocalTweetSafetyLevel, - TweetDetailWithInjectionsHydration -> DeciderKey.EnableTweetDetailWithInjectionsHydrationSafetyLevel, - Tombstoning -> DeciderKey.EnableTombstoningSafetyLevel, - TopicRecommendations -> DeciderKey.EnableTopicRecommendationsSafetyLevel, - TrendsRepresentativeTweet -> DeciderKey.EnableTrendsRepresentativeTweetSafetyLevel, - TrustedFriendsUserList -> DeciderKey.EnableTrustedFriendsUserListSafetyLevel, - TwitterDelegateUserList -> DeciderKey.EnableTwitterDelegateUserListSafetyLevel, - TweetDetail -> DeciderKey.EnableTweetDetailSafetyLevel, - TweetDetailNonToo -> DeciderKey.EnableTweetDetailNonTooSafetyLevel, - TweetEngagers -> DeciderKey.EnableTweetEngagersSafetyLevel, - TweetReplyNudge -> DeciderKey.EnableTweetReplyNudgeSafetyLevel, - TweetScopedTimeline -> DeciderKey.EnableTweetScopedTimelineSafetyLevel, - TweetWritesApi -> DeciderKey.EnableTweetWritesApiSafetyLevel, - TwitterArticleCompose -> DeciderKey.EnableTwitterArticleComposeSafetyLevel, - TwitterArticleProfileTab -> DeciderKey.EnableTwitterArticleProfileTabSafetyLevel, - TwitterArticleRead -> DeciderKey.EnableTwitterArticleReadSafetyLevel, - UserProfileHeader -> DeciderKey.EnableUserProfileHeaderSafetyLevel, - UserMilestoneRecommendation -> DeciderKey.EnableUserMilestoneRecommendationSafetyLevel, - UserScopedTimeline -> DeciderKey.EnableUserScopedTimelineSafetyLevel, - UserSearchSrp -> DeciderKey.EnableUserSearchSrpSafetyLevel, - UserSearchTypeahead -> DeciderKey.EnableUserSearchTypeaheadSafetyLevel, - UserSelfViewOnly -> DeciderKey.EnableUserSelfViewOnlySafetyLevel, - UserSettings -> DeciderKey.EnableUserSettingsSafetyLevel, - VideoAds -> DeciderKey.EnableVideoAdsSafetyLevel, - WritePathLimitedActionsEnforcement -> DeciderKey.EnableWritePathLimitedActionsEnforcementSafetyLevel, - ZipbirdConsumerArchives -> DeciderKey.EnableZipbirdConsumerArchivesSafetyLevel, - TweetAward -> DeciderKey.EnableTweetAwardSafetyLevel, - ) - - val BoolToDeciderMap: Map[Param[Boolean], DeciderKey.Value] = Map( - RuleParams.TweetConversationControlEnabledParam -> - DeciderKey.EnableTweetConversationControlRules, - RuleParams.CommunityTweetsEnabledParam -> - DeciderKey.EnableCommunityTweetsControlRules, - RuleParams.DropCommunityTweetWithUndefinedCommunityRuleEnabledParam -> - DeciderKey.EnableDropCommunityTweetWithUndefinedCommunityRule, - TimelineConversationsDownrankingSpecificParams.EnablePSpammyTweetDownrankConvosLowQualityParam -> - DeciderKey.EnablePSpammyTweetDownrankConvosLowQuality, - RuleParams.EnableHighPSpammyTweetScoreSearchTweetLabelDropRuleParam -> - DeciderKey.EnableHighPSpammyTweetScoreSearchTweetLabelDropRule, - TimelineConversationsDownrankingSpecificParams.EnableRitoActionedTweetDownrankConvosLowQualityParam -> - DeciderKey.EnableRitoActionedTweetDownrankConvosLowQuality, - RuleParams.EnableSmyteSpamTweetRuleParam -> - DeciderKey.EnableSmyteSpamTweetRule, - RuleParams.EnableHighSpammyTweetContentScoreSearchLatestTweetLabelDropRuleParam -> - DeciderKey.EnableHighSpammyTweetContentScoreSearchLatestTweetLabelDropRule, - RuleParams.EnableHighSpammyTweetContentScoreSearchTopTweetLabelDropRuleParam -> - DeciderKey.EnableHighSpammyTweetContentScoreSearchTopTweetLabelDropRule, - RuleParams.EnableHighSpammyTweetContentScoreTrendsTopTweetLabelDropRuleParam -> - DeciderKey.EnableHighSpammyTweetContentScoreTrendsTopTweetLabelDropRule, - RuleParams.EnableHighSpammyTweetContentScoreTrendsLatestTweetLabelDropRuleParam -> - DeciderKey.EnableHighSpammyTweetContentScoreTrendsLatestTweetLabelDropRule, - TimelineConversationsDownrankingSpecificParams.EnableHighSpammyTweetContentScoreConvoDownrankAbusiveQualityRuleParam -> - DeciderKey.EnableHighSpammyTweetContentScoreConvoDownrankAbusiveQualityRule, - TimelineConversationsDownrankingSpecificParams.EnableHighCryptospamScoreConvoDownrankAbusiveQualityRuleParam -> - DeciderKey.EnableHighCryptospamScoreConvoDownrankAbusiveQualityRule, - RuleParams.EnableGoreAndViolenceTopicHighRecallTweetLabelRule -> - DeciderKey.EnableGoreAndViolenceTopicHighRecallTweetLabelRule, - RuleParams.EnableLimitRepliesFollowersConversationRule -> - DeciderKey.EnableLimitRepliesFollowersConversationRule, - RuleParams.EnableSearchBasicBlockMuteRulesParam -> DeciderKey.EnableSearchBasicBlockMuteRules, - RuleParams.EnableBlinkBadDownrankingRuleParam -> - DeciderKey.EnableBlinkBadDownrankingRule, - RuleParams.EnableBlinkWorstDownrankingRuleParam -> - DeciderKey.EnableBlinkWorstDownrankingRule, - RuleParams.EnableCopypastaSpamDownrankConvosAbusiveQualityRule -> - DeciderKey.EnableCopypastaSpamDownrankConvosAbusiveQualityRule, - RuleParams.EnableCopypastaSpamSearchDropRule -> - DeciderKey.EnableCopypastaSpamSearchDropRule, - RuleParams.EnableSpammyUserModelTweetDropRuleParam -> - DeciderKey.EnableSpammyUserModelHighPrecisionDropTweetRule, - RuleParams.EnableAvoidNsfwRulesParam -> - DeciderKey.EnableAvoidNsfwRules, - RuleParams.EnableReportedTweetInterstitialRule -> - DeciderKey.EnableReportedTweetInterstitialRule, - RuleParams.EnableReportedTweetInterstitialSearchRule -> - DeciderKey.EnableReportedTweetInterstitialSearchRule, - RuleParams.EnableDropExclusiveTweetContentRule -> - DeciderKey.EnableDropExclusiveTweetContentRule, - RuleParams.EnableDropExclusiveTweetContentRuleFailClosed -> - DeciderKey.EnableDropExclusiveTweetContentRuleFailClosed, - RuleParams.EnableTombstoneExclusiveQtProfileTimelineParam -> - DeciderKey.EnableTombstoneExclusiveQtProfileTimelineParam, - RuleParams.EnableDropAllExclusiveTweetsRuleParam -> DeciderKey.EnableDropAllExclusiveTweetsRule, - RuleParams.EnableDropAllExclusiveTweetsRuleFailClosedParam -> DeciderKey.EnableDropAllExclusiveTweetsRuleFailClosed, - RuleParams.EnableDownrankSpamReplySectioningRuleParam -> - DeciderKey.EnableDownrankSpamReplySectioningRule, - RuleParams.EnableNsfwTextSectioningRuleParam -> - DeciderKey.EnableNsfwTextSectioningRule, - RuleParams.EnableSearchIpiSafeSearchWithoutUserInQueryDropRule -> DeciderKey.EnableSearchIpiSafeSearchWithoutUserInQueryDropRule, - RuleParams.EnableTimelineHomePromotedTweetHealthEnforcementRules -> DeciderKey.EnableTimelineHomePromotedTweetHealthEnforcementRules, - RuleParams.EnableMutedKeywordFilteringSpaceTitleNotificationsRuleParam -> DeciderKey.EnableMutedKeywordFilteringSpaceTitleNotificationsRule, - RuleParams.EnableDropTweetsWithGeoRestrictedMediaRuleParam -> DeciderKey.EnableDropTweetsWithGeoRestrictedMediaRule, - RuleParams.EnableDropAllTrustedFriendsTweetsRuleParam -> DeciderKey.EnableDropAllTrustedFriendsTweetsRule, - RuleParams.EnableDropTrustedFriendsTweetContentRuleParam -> DeciderKey.EnableDropTrustedFriendsTweetContentRule, - RuleParams.EnableDropAllCollabInvitationTweetsRuleParam -> DeciderKey.EnableDropCollabInvitationTweetsRule, - RuleParams.EnableNsfwTextHighPrecisionDropRuleParam -> DeciderKey.EnableNsfwTextHighPrecisionDropRule, - RuleParams.EnableLikelyIvsUserLabelDropRule -> DeciderKey.EnableLikelyIvsUserLabelDropRule, - RuleParams.EnableCardUriRootDomainCardDenylistRule -> DeciderKey.EnableCardUriRootDomainDenylistRule, - RuleParams.EnableCommunityNonMemberPollCardRule -> DeciderKey.EnableCommunityNonMemberPollCardRule, - RuleParams.EnableCommunityNonMemberPollCardRuleFailClosed -> DeciderKey.EnableCommunityNonMemberPollCardRuleFailClosed, - RuleParams.EnableExperimentalNudgeEnabledParam -> DeciderKey.EnableExperimentalNudgeLabelRule, - RuleParams.NsfwHighPrecisionUserLabelAvoidTweetRuleEnabledParam -> DeciderKey.NsfwHighPrecisionUserLabelAvoidTweetRuleEnabledParam, - RuleParams.EnableNewAdAvoidanceRulesParam -> DeciderKey.EnableNewAdAvoidanceRules, - RuleParams.EnableNsfaHighRecallAdAvoidanceParam -> DeciderKey.EnableNsfaHighRecallAdAvoidanceParam, - RuleParams.EnableNsfaKeywordsHighPrecisionAdAvoidanceParam -> DeciderKey.EnableNsfaKeywordsHighPrecisionAdAvoidanceParam, - RuleParams.EnableStaleTweetDropRuleParam -> DeciderKey.EnableStaleTweetDropRuleParam, - RuleParams.EnableStaleTweetDropRuleFailClosedParam -> DeciderKey.EnableStaleTweetDropRuleFailClosedParam, - RuleParams.EnableDeleteStateTweetRulesParam -> DeciderKey.EnableDeleteStateTweetRules, - RuleParams.EnableSpacesSharingNsfwDropRulesParam -> DeciderKey.EnableSpacesSharingNsfwDropRulesParam, - RuleParams.EnableViewerIsSoftUserDropRuleParam -> DeciderKey.EnableViewerIsSoftUserDropRuleParam, - RuleParams.EnablePdnaQuotedTweetTombstoneRuleParam -> DeciderKey.EnablePdnaQuotedTweetTombstoneRule, - RuleParams.EnableSpamQuotedTweetTombstoneRuleParam -> DeciderKey.EnableSpamQuotedTweetTombstoneRule, - RuleParams.EnableNsfwHpQuotedTweetDropRuleParam -> DeciderKey.EnableNsfwHpQuotedTweetDropRule, - RuleParams.EnableNsfwHpQuotedTweetTombstoneRuleParam -> DeciderKey.EnableNsfwHpQuotedTweetTombstoneRule, - RuleParams.EnableInnerQuotedTweetViewerBlocksAuthorInterstitialRuleParam -> DeciderKey.EnableInnerQuotedTweetViewerBlocksAuthorInterstitialRule, - RuleParams.EnableInnerQuotedTweetViewerMutesAuthorInterstitialRuleParam -> DeciderKey.EnableInnerQuotedTweetViewerMutesAuthorInterstitialRule, - RuleParams.EnableToxicReplyFilteringConversationRulesParam -> DeciderKey.VisibilityLibraryEnableToxicReplyFilterConversation, - RuleParams.EnableToxicReplyFilteringNotificationsRulesParam -> DeciderKey.VisibilityLibraryEnableToxicReplyFilterNotifications, - RuleParams.EnableLegacySensitiveMediaHomeTimelineRulesParam -> DeciderKey.EnableLegacySensitiveMediaRulesHomeTimeline, - RuleParams.EnableNewSensitiveMediaSettingsInterstitialsHomeTimelineRulesParam -> DeciderKey.EnableNewSensitiveMediaSettingsInterstitialRulesHomeTimeline, - RuleParams.EnableLegacySensitiveMediaConversationRulesParam -> DeciderKey.EnableLegacySensitiveMediaRulesConversation, - RuleParams.EnableNewSensitiveMediaSettingsInterstitialsConversationRulesParam -> DeciderKey.EnableNewSensitiveMediaSettingsInterstitialRulesConversation, - RuleParams.EnableLegacySensitiveMediaProfileTimelineRulesParam -> DeciderKey.EnableLegacySensitiveMediaRulesProfileTimeline, - RuleParams.EnableNewSensitiveMediaSettingsInterstitialsProfileTimelineRulesParam -> DeciderKey.EnableNewSensitiveMediaSettingsInterstitialRulesProfileTimeline, - RuleParams.EnableLegacySensitiveMediaTweetDetailRulesParam -> DeciderKey.EnableLegacySensitiveMediaRulesTweetDetail, - RuleParams.EnableNewSensitiveMediaSettingsInterstitialsTweetDetailRulesParam -> DeciderKey.EnableNewSensitiveMediaSettingsInterstitialRulesTweetDetail, - RuleParams.EnableLegacySensitiveMediaDirectMessagesRulesParam -> DeciderKey.EnableLegacySensitiveMediaRulesDirectMessages, - RuleParams.EnableAbusiveBehaviorDropRuleParam -> DeciderKey.EnableAbusiveBehaviorDropRule, - RuleParams.EnableAbusiveBehaviorInterstitialRuleParam -> DeciderKey.EnableAbusiveBehaviorInterstitialRule, - RuleParams.EnableAbusiveBehaviorLimitedEngagementsRuleParam -> DeciderKey.EnableAbusiveBehaviorLimitedEngagementsRule, - RuleParams.EnableNotGraduatedDownrankConvosAbusiveQualityRuleParam -> DeciderKey.EnableNotGraduatedDownrankConvosAbusiveQualityRule, - RuleParams.EnableNotGraduatedSearchDropRuleParam -> DeciderKey.EnableNotGraduatedSearchDropRule, - RuleParams.EnableNotGraduatedDropRuleParam -> DeciderKey.EnableNotGraduatedDropRule, - RuleParams.EnableFosnrRuleParam -> DeciderKey.EnableFosnrRules, - RuleParams.EnableAuthorBlocksViewerDropRuleParam -> DeciderKey.EnableAuthorBlocksViewerDropRule - ) - - def config( - deciderGateBuilder: DeciderGateBuilder, - logger: Logger, - statsReceiver: StatsReceiver, - SafetyLevel: SafetyLevel - ): Config = { - - object UserOrGuestOrRequest extends RecipientBuilder { - private val scopedStats = statsReceiver.scope("decider_recipient") - private val userIdDefinedCounter = scopedStats.counter("user_id_defined") - private val userIdNotDefinedCounter = scopedStats.counter("user_id_undefined") - private val guestIdDefinedCounter = scopedStats.counter("guest_id_defined") - private val guestIdNotDefinedCounter = scopedStats.counter("guest_id_undefined") - private val noIdCounter = scopedStats.counter("no_id_defined") - - def apply(requestContext: BaseRequestContext): Option[Recipient] = requestContext match { - case c: WithUserId if c.userId.isDefined => - userIdDefinedCounter.incr() - c.userId.map(SimpleRecipient) - case c: WithGuestId if c.guestId.isDefined => - guestIdDefinedCounter.incr() - c.guestId.map(GuestRecipient) - case c: WithGuestId => - guestIdNotDefinedCounter.incr() - RecipientBuilder.Request(c) - case _: WithUserId => - userIdNotDefinedCounter.incr() - None - case _ => - logger.warning("Request Context with no user or guest id trait found: " + requestContext) - noIdCounter.incr() - None - } - } - - val boolOverrides = BoolToDeciderMap.map { - case (param, deciderKey) => - param.optionalOverrideValue( - DeciderSwitchOverrideValue( - feature = deciderGateBuilder.keyToFeature(deciderKey), - enabledValue = true, - disabledValueOption = Some(false), - recipientBuilder = UserOrGuestOrRequest - ) - ) - }.toSeq - - val safetyLevelOverride = SafetyLevel.enabledParam.optionalOverrideValue( - DeciderSwitchOverrideValue( - feature = deciderGateBuilder.keyToFeature(SafetyLevelToDeciderMap(SafetyLevel)), - enabledValue = true, - recipientBuilder = UserOrGuestOrRequest - ) - ) - - BaseConfigBuilder(boolOverrides :+ safetyLevelOverride).build("VisibilityDeciders") - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityExperimentsConfig.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityExperimentsConfig.docx new file mode 100644 index 000000000..f0d722c2d Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityExperimentsConfig.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityExperimentsConfig.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityExperimentsConfig.scala deleted file mode 100644 index faf25ee2a..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityExperimentsConfig.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.visibility.configapi.configs - -import com.twitter.timelines.configapi.Config -import com.twitter.visibility.configapi.params.RuleParams._ -import com.twitter.visibility.configapi.params.VisibilityExperiments._ -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.SafetyLevel._ - -private[visibility] object VisibilityExperimentsConfig { - import ExperimentsHelper._ - - val TestExperimentConfig: Config = mkABExperimentConfig(TestExperiment, TestHoldbackParam) - - val NotGraduatedUserLabelRuleHoldbackExperimentConfig: Config = - mkABExperimentConfig( - NotGraduatedUserLabelRuleExperiment, - NotGraduatedUserLabelRuleHoldbackExperimentParam - ) - - def config(safetyLevel: SafetyLevel): Seq[Config] = { - - val experimentConfigs = safetyLevel match { - - case Test => - Seq(TestExperimentConfig) - - case _ => Seq(NotGraduatedUserLabelRuleHoldbackExperimentConfig) - } - - experimentConfigs - } - -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityFeatureSwitches.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityFeatureSwitches.docx new file mode 100644 index 000000000..52fa2ca9f Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityFeatureSwitches.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityFeatureSwitches.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityFeatureSwitches.scala deleted file mode 100644 index 2fa33cd57..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/VisibilityFeatureSwitches.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.visibility.configapi.configs - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi._ -import com.twitter.util.Time -import com.twitter.visibility.configapi.params.FSEnumRuleParam -import com.twitter.visibility.configapi.params.FSRuleParams._ - -private[visibility] object VisibilityFeatureSwitches { - - val booleanFsOverrides: Seq[OptionalOverride[Boolean]] = - FeatureSwitchOverrideUtil.getBooleanFSOverrides( - AgeGatingAdultContentExperimentRuleEnabledParam, - CommunityTweetCommunityUnavailableLimitedActionsRulesEnabledParam, - CommunityTweetDropProtectedRuleEnabledParam, - CommunityTweetDropRuleEnabledParam, - CommunityTweetLimitedActionsRulesEnabledParam, - CommunityTweetMemberRemovedLimitedActionsRulesEnabledParam, - CommunityTweetNonMemberLimitedActionsRuleEnabledParam, - NsfwAgeBasedDropRulesHoldbackParam, - SkipTweetDetailLimitedEngagementRuleEnabledParam, - StaleTweetLimitedActionsRulesEnabledParam, - TrustedFriendsTweetLimitedEngagementsRuleEnabledParam, - FosnrFallbackDropRulesEnabledParam, - FosnrRulesEnabledParam - ) - - val doubleFsOverrides: Seq[OptionalOverride[Double]] = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - HighSpammyTweetContentScoreSearchTopProdTweetLabelDropRuleThresholdParam, - HighSpammyTweetContentScoreSearchLatestProdTweetLabelDropRuleThresholdParam, - HighSpammyTweetContentScoreTrendTopTweetLabelDropRuleThresholdParam, - HighSpammyTweetContentScoreTrendLatestTweetLabelDropRuleThresholdParam, - HighSpammyTweetContentScoreConvoDownrankAbusiveQualityThresholdParam, - HighToxicityModelScoreSpaceThresholdParam, - AdAvoidanceHighToxicityModelScoreThresholdParam, - AdAvoidanceReportedTweetModelScoreThresholdParam, - ) - - val timeFsOverrides: Seq[OptionalOverride[Time]] = - FeatureSwitchOverrideUtil.getTimeFromStringFSOverrides() - - val stringSeqFeatureSwitchOverrides: Seq[OptionalOverride[Seq[String]]] = - FeatureSwitchOverrideUtil.getStringSeqFSOverrides( - CountrySpecificNsfwContentGatingCountriesParam, - AgeGatingAdultContentExperimentCountriesParam, - CardUriRootDomainDenyListParam - ) - - val enumFsParams: Seq[FSEnumRuleParam[_ <: Enumeration]] = Seq() - - val mkOptionalEnumFsOverrides: (StatsReceiver, Logger) => Seq[OptionalOverride[_]] = { - (statsReceiver: StatsReceiver, logger: Logger) => - FeatureSwitchOverrideUtil.getEnumFSOverrides( - statsReceiver, - logger, - enumFsParams: _* - ) - } - - def overrides(statsReceiver: StatsReceiver, logger: Logger): Seq[OptionalOverride[_]] = { - val enumOverrides = mkOptionalEnumFsOverrides(statsReceiver, logger) - booleanFsOverrides ++ - doubleFsOverrides ++ - timeFsOverrides ++ - stringSeqFeatureSwitchOverrides ++ - enumOverrides - } - - def config(statsReceiver: StatsReceiver, logger: Logger): BaseConfig = - BaseConfigBuilder(overrides(statsReceiver.scope("features_switches"), logger)) - .build("VisibilityFeatureSwitches") -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/BUILD deleted file mode 100644 index b59bedb48..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "decider", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/BUILD.docx new file mode 100644 index 000000000..6e43b911b Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/VisibilityLibraryDeciderOverrides.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/VisibilityLibraryDeciderOverrides.docx new file mode 100644 index 000000000..4fc5f8413 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/VisibilityLibraryDeciderOverrides.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/VisibilityLibraryDeciderOverrides.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/VisibilityLibraryDeciderOverrides.scala deleted file mode 100644 index 842309be7..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/configs/overrides/VisibilityLibraryDeciderOverrides.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.visibility.configapi.configs.overrides - -import com.twitter.decider.LocalOverrides - -object VisibilityLibraryDeciderOverrides - extends LocalOverrides.Namespace("visibility-library", "") { - - val EnableLocalizedTombstoneOnVisibilityResults = feature( - "visibility_library_enable_localized_tombstones_on_visibility_results") - - val EnableLocalizedInterstitialGenerator: LocalOverrides.Override = - feature("visibility_library_enable_localized_interstitial_generator") - - val EnableInnerQuotedTweetViewerBlocksAuthorInterstitialRule: LocalOverrides.Override = - feature("visibility_library_enable_inner_quoted_tweet_viewer_blocks_author_interstitial_rule") - val EnableInnerQuotedTweetViewerMutesAuthorInterstitialRule: LocalOverrides.Override = - feature("visibility_library_enable_inner_quoted_tweet_viewer_mutes_author_interstitial_rule") - - val EnableBackendLimitedActions: LocalOverrides.Override = - feature("visibility_library_enable_backend_limited_actions") - - val disableLegacyInterstitialFilteredReason: LocalOverrides.Override = feature( - "visibility_library_disable_legacy_interstitial_filtered_reason") -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/BUILD deleted file mode 100644 index b2855c4c6..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "finagle/finagle-stats", - "visibility/common/src/main/scala/com/twitter/visibility/common:model_thresholds", - ], - exports = [ - "visibility/common/src/main/scala/com/twitter/visibility/common:model_thresholds", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/BUILD.docx new file mode 100644 index 000000000..3a80d72e1 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/FSRuleParams.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/FSRuleParams.docx new file mode 100644 index 000000000..797653b50 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/FSRuleParams.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/FSRuleParams.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/FSRuleParams.scala deleted file mode 100644 index 270db45aa..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/FSRuleParams.scala +++ /dev/null @@ -1,213 +0,0 @@ -package com.twitter.visibility.configapi.params - -import com.twitter.timelines.configapi.Bounded -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FeatureName -import com.twitter.timelines.configapi.HasTimeConversion -import com.twitter.timelines.configapi.TimeConversion -import com.twitter.util.Time -import com.twitter.visibility.common.ModelScoreThresholds - -private[visibility] object FeatureSwitchKey extends Enumeration { - type FeatureSwitchKey = String - - final val HighSpammyTweetContentScoreSearchTopProdTweetLabelDropFuleThreshold = - "high_spammy_tweet_content_score_search_top_prod_tweet_label_drop_rule_threshold" - final val HighSpammyTweetContentScoreSearchLatestProdTweetLabelDropRuleThreshold = - "high_spammy_tweet_content_score_search_latest_prod_tweet_label_drop_rule_threshold" - final val HighSpammyTweetContentScoreTrendTopTweetLabelDropRuleThreshold = - "high_spammy_tweet_content_score_trend_top_tweet_label_drop_rule_threshold" - final val HighSpammyTweetContentScoreTrendLatestTweetLabelDropRuleThreshold = - "high_spammy_tweet_content_score_trend_latest_tweet_label_drop_rule_threshold" - final val HighSpammyTweetContentScoreConvoDownrankAbusiveQualityThreshold = - "high_spammy_tweet_content_score_convos_downranking_abusive_quality_threshold" - - final val NsfwAgeBasedDropRulesHoldbackParam = - "nsfw_age_based_drop_rules_holdback" - - final val CommunityTweetDropRuleEnabled = - "community_tweet_drop_rule_enabled" - final val CommunityTweetDropProtectedRuleEnabled = - "community_tweet_drop_protected_rule_enabled" - final val CommunityTweetLimitedActionsRulesEnabled = - "community_tweet_limited_actions_rules_enabled" - final val CommunityTweetMemberRemovedLimitedActionsRulesEnabled = - "community_tweet_member_removed_limited_actions_rules_enabled" - final val CommunityTweetCommunityUnavailableLimitedActionsRulesEnabled = - "community_tweet_community_unavailable_limited_actions_rules_enabled" - final val CommunityTweetNonMemberLimitedActionsRuleEnabled = - "community_tweet_non_member_limited_actions_rule_enabled" - - final val TrustedFriendsTweetLimitedEngagementsRuleEnabled = - "trusted_friends_tweet_limited_engagements_rule_enabled" - - final val CountrySpecificNsfwContentGatingCountries = - "country_specific_nsfw_content_gating_countries" - - final val AgeGatingAdultContentExperimentCountries = - "age_gating_adult_content_experiment_countries" - final val AgeGatingAdultContentExperimentEnabled = - "age_gating_adult_content_experiment_enabled" - - final val HighToxicityModelScoreSpaceThreshold = - "high_toxicity_model_score_space_threshold" - - final val CardUriRootDomainDenyList = "card_uri_root_domain_deny_list" - - final val SkipTweetDetailLimitedEngagementsRuleEnabled = - "skip_tweet_detail_limited_engagements_rule_enabled" - - final val AdAvoidanceHighToxicityModelScoreThreshold = - "ad_avoidance_model_thresholds_high_toxicity_model" - final val AdAvoidanceReportedTweetModelScoreThreshold = - "ad_avoidance_model_thresholds_reported_tweet_model" - - final val StaleTweetLimitedActionsRulesEnabled = - "stale_tweet_limited_actions_rules_enabled" - - final val FosnrFallbackDropRulesEnabled = - "freedom_of_speech_not_reach_fallback_drop_rules_enabled" - final val FosnrRulesEnabled = - "freedom_of_speech_not_reach_rules_enabled" -} - -abstract class FSRuleParam[T](override val name: FeatureName, override val default: T) - extends RuleParam(default) - with FSName - -abstract class FSBoundedRuleParam[T]( - override val name: FeatureName, - override val default: T, - override val min: T, - override val max: T -)( - implicit override val ordering: Ordering[T]) - extends RuleParam(default) - with Bounded[T] - with FSName - -abstract class FSTimeRuleParam[T]( - override val name: FeatureName, - override val default: Time, - override val timeConversion: TimeConversion[T]) - extends RuleParam(default) - with HasTimeConversion[T] - with FSName - -abstract class FSEnumRuleParam[T <: Enumeration]( - override val name: FeatureName, - override val default: T#Value, - override val enum: T) - extends EnumRuleParam(default, enum) - with FSName - -private[visibility] object FSRuleParams { - object HighSpammyTweetContentScoreSearchTopProdTweetLabelDropRuleThresholdParam - extends FSBoundedParam( - FeatureSwitchKey.HighSpammyTweetContentScoreSearchTopProdTweetLabelDropFuleThreshold, - default = ModelScoreThresholds.HighSpammyTweetContentScoreDefaultThreshold, - min = 0, - max = 1) - object HighSpammyTweetContentScoreSearchLatestProdTweetLabelDropRuleThresholdParam - extends FSBoundedParam( - FeatureSwitchKey.HighSpammyTweetContentScoreSearchLatestProdTweetLabelDropRuleThreshold, - default = ModelScoreThresholds.HighSpammyTweetContentScoreDefaultThreshold, - min = 0, - max = 1) - object HighSpammyTweetContentScoreTrendTopTweetLabelDropRuleThresholdParam - extends FSBoundedParam( - FeatureSwitchKey.HighSpammyTweetContentScoreTrendTopTweetLabelDropRuleThreshold, - default = ModelScoreThresholds.HighSpammyTweetContentScoreDefaultThreshold, - min = 0, - max = 1) - object HighSpammyTweetContentScoreTrendLatestTweetLabelDropRuleThresholdParam - extends FSBoundedParam( - FeatureSwitchKey.HighSpammyTweetContentScoreTrendLatestTweetLabelDropRuleThreshold, - default = ModelScoreThresholds.HighSpammyTweetContentScoreDefaultThreshold, - min = 0, - max = 1) - object HighSpammyTweetContentScoreConvoDownrankAbusiveQualityThresholdParam - extends FSBoundedParam( - FeatureSwitchKey.HighSpammyTweetContentScoreConvoDownrankAbusiveQualityThreshold, - default = ModelScoreThresholds.HighSpammyTweetContentScoreDefaultThreshold, - min = 0, - max = 1) - - object CommunityTweetDropRuleEnabledParam - extends FSRuleParam(FeatureSwitchKey.CommunityTweetDropRuleEnabled, true) - - object CommunityTweetDropProtectedRuleEnabledParam - extends FSRuleParam(FeatureSwitchKey.CommunityTweetDropProtectedRuleEnabled, true) - - object CommunityTweetLimitedActionsRulesEnabledParam - extends FSRuleParam(FeatureSwitchKey.CommunityTweetLimitedActionsRulesEnabled, false) - - object CommunityTweetMemberRemovedLimitedActionsRulesEnabledParam - extends FSRuleParam( - FeatureSwitchKey.CommunityTweetMemberRemovedLimitedActionsRulesEnabled, - false) - - object CommunityTweetCommunityUnavailableLimitedActionsRulesEnabledParam - extends FSRuleParam( - FeatureSwitchKey.CommunityTweetCommunityUnavailableLimitedActionsRulesEnabled, - false) - - object CommunityTweetNonMemberLimitedActionsRuleEnabledParam - extends FSRuleParam(FeatureSwitchKey.CommunityTweetNonMemberLimitedActionsRuleEnabled, false) - - object TrustedFriendsTweetLimitedEngagementsRuleEnabledParam - extends FSRuleParam(FeatureSwitchKey.TrustedFriendsTweetLimitedEngagementsRuleEnabled, false) - - object SkipTweetDetailLimitedEngagementRuleEnabledParam - extends FSRuleParam(FeatureSwitchKey.SkipTweetDetailLimitedEngagementsRuleEnabled, false) - - - object NsfwAgeBasedDropRulesHoldbackParam - extends FSRuleParam(FeatureSwitchKey.NsfwAgeBasedDropRulesHoldbackParam, true) - - object CountrySpecificNsfwContentGatingCountriesParam - extends FSRuleParam[Seq[String]]( - FeatureSwitchKey.CountrySpecificNsfwContentGatingCountries, - default = Seq("au")) - - object AgeGatingAdultContentExperimentCountriesParam - extends FSRuleParam[Seq[String]]( - FeatureSwitchKey.AgeGatingAdultContentExperimentCountries, - default = Seq.empty) - object AgeGatingAdultContentExperimentRuleEnabledParam - extends FSRuleParam(FeatureSwitchKey.AgeGatingAdultContentExperimentEnabled, default = false) - - object HighToxicityModelScoreSpaceThresholdParam - extends FSBoundedParam( - FeatureSwitchKey.HighToxicityModelScoreSpaceThreshold, - default = ModelScoreThresholds.HighToxicityModelScoreSpaceDefaultThreshold, - min = 0, - max = 1) - - object CardUriRootDomainDenyListParam - extends FSRuleParam[Seq[String]]( - FeatureSwitchKey.CardUriRootDomainDenyList, - default = Seq.empty) - - object AdAvoidanceHighToxicityModelScoreThresholdParam - extends FSBoundedParam( - FeatureSwitchKey.AdAvoidanceHighToxicityModelScoreThreshold, - default = ModelScoreThresholds.AdAvoidanceHighToxicityModelScoreDefaultThreshold, - min = 0, - max = 1) - - object AdAvoidanceReportedTweetModelScoreThresholdParam - extends FSBoundedParam( - FeatureSwitchKey.AdAvoidanceReportedTweetModelScoreThreshold, - default = ModelScoreThresholds.AdAvoidanceReportedTweetModelScoreDefaultThreshold, - min = 0, - max = 1) - - object StaleTweetLimitedActionsRulesEnabledParam - extends FSRuleParam(FeatureSwitchKey.StaleTweetLimitedActionsRulesEnabled, false) - - object FosnrFallbackDropRulesEnabledParam - extends FSRuleParam(FeatureSwitchKey.FosnrFallbackDropRulesEnabled, false) - object FosnrRulesEnabledParam extends FSRuleParam(FeatureSwitchKey.FosnrRulesEnabled, true) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/GlobalParams.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/GlobalParams.docx new file mode 100644 index 000000000..f74d0ca8f Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/GlobalParams.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/GlobalParams.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/GlobalParams.scala deleted file mode 100644 index c6a960486..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/GlobalParams.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.visibility.configapi.params - -import com.twitter.timelines.configapi.Param - -abstract class GlobalParam[T](override val default: T) extends Param(default) { - override val statName: String = s"GlobalParam/${this.getClass.getSimpleName}" -} - -private[visibility] object GlobalParams { - object EnableFetchingLabelMap extends GlobalParam(false) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/LabelSourceParams.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/LabelSourceParams.docx new file mode 100644 index 000000000..908542deb Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/LabelSourceParams.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/LabelSourceParams.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/LabelSourceParams.scala deleted file mode 100644 index 0f0eaaa2d..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/LabelSourceParams.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.visibility.configapi.params - -import com.twitter.timelines.configapi.Param - -abstract class LabelSourceParam(override val default: Boolean) extends Param(default) { - override val statName: String = s"LabelSourceParam/${this.getClass.getSimpleName}" -} - -private[visibility] object LabelSourceParams { - object FilterLabelsFromBot7174Param extends LabelSourceParam(false) - - object FilterTweetsSmyteAutomationParamA extends LabelSourceParam(false) - object FilterTweetsSmyteAutomationParamB extends LabelSourceParam(false) - object FilterTweetsSmyteAutomationParamAB extends LabelSourceParam(false) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/RuleParams.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/RuleParams.docx new file mode 100644 index 000000000..11edf4b40 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/RuleParams.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/RuleParams.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/RuleParams.scala deleted file mode 100644 index a4e28e690..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/RuleParams.scala +++ /dev/null @@ -1,164 +0,0 @@ -package com.twitter.visibility.configapi.params - -import com.twitter.timelines.configapi.EnumParam -import com.twitter.timelines.configapi.Param - -abstract class RuleParam[T](override val default: T) extends Param(default) { - override val statName: String = s"RuleParam/${this.getClass.getSimpleName}" -} - -abstract class EnumRuleParam[T <: Enumeration](override val default: T#Value, override val enum: T) - extends EnumParam(default, enum) { - override val statName: String = s"RuleParam/${this.getClass.getSimpleName}" -} - -private[visibility] object RuleParams { - object True extends RuleParam(true) - object False extends RuleParam(false) - - object TestHoldbackParam extends RuleParam(true) - - object TweetConversationControlEnabledParam extends RuleParam(default = false) - - object EnableLimitRepliesFollowersConversationRule extends RuleParam(default = false) - - object CommunityTweetsEnabledParam extends RuleParam(default = false) - - object DropCommunityTweetWithUndefinedCommunityRuleEnabledParam extends RuleParam(default = false) - - object EnableHighPSpammyTweetScoreSearchTweetLabelDropRuleParam extends RuleParam(false) - - object EnableSmyteSpamTweetRuleParam extends RuleParam(false) - - object EnableHighSpammyTweetContentScoreSearchLatestTweetLabelDropRuleParam - extends RuleParam(false) - - object EnableHighSpammyTweetContentScoreSearchTopTweetLabelDropRuleParam extends RuleParam(false) - - object NotGraduatedUserLabelRuleHoldbackExperimentParam extends RuleParam(default = false) - - object EnableGoreAndViolenceTopicHighRecallTweetLabelRule extends RuleParam(default = false) - - object EnableBlinkBadDownrankingRuleParam extends RuleParam(false) - object EnableBlinkWorstDownrankingRuleParam extends RuleParam(false) - - object EnableHighSpammyTweetContentScoreTrendsTopTweetLabelDropRuleParam - extends RuleParam(default = false) - - object EnableHighSpammyTweetContentScoreTrendsLatestTweetLabelDropRuleParam - extends RuleParam(default = false) - - object EnableCopypastaSpamDownrankConvosAbusiveQualityRule extends RuleParam(default = false) - object EnableCopypastaSpamSearchDropRule extends RuleParam(default = false) - - object EnableSpammyUserModelTweetDropRuleParam extends RuleParam(default = false) - - object EnableAvoidNsfwRulesParam extends RuleParam(false) - - object EnableReportedTweetInterstitialRule extends RuleParam(default = false) - - object EnableReportedTweetInterstitialSearchRule extends RuleParam(default = false) - - object EnableDropExclusiveTweetContentRule extends RuleParam(default = false) - - object EnableDropExclusiveTweetContentRuleFailClosed extends RuleParam(default = false) - - object EnableTombstoneExclusiveQtProfileTimelineParam extends RuleParam(default = false) - - object EnableDropAllExclusiveTweetsRuleParam extends RuleParam(false) - object EnableDropAllExclusiveTweetsRuleFailClosedParam extends RuleParam(false) - - object EnableDownrankSpamReplySectioningRuleParam extends RuleParam(default = false) - object EnableNsfwTextSectioningRuleParam extends RuleParam(default = false) - - object EnableSearchIpiSafeSearchWithoutUserInQueryDropRule extends RuleParam(false) - - object PromotedTweetHealthEnforcementHoldback extends RuleParam(default = true) - object EnableTimelineHomePromotedTweetHealthEnforcementRules extends RuleParam(default = false) - - object EnableMutedKeywordFilteringSpaceTitleNotificationsRuleParam extends RuleParam(false) - - object EnableDropTweetsWithGeoRestrictedMediaRuleParam extends RuleParam(default = false) - - object EnableDropAllTrustedFriendsTweetsRuleParam extends RuleParam(false) - object EnableDropTrustedFriendsTweetContentRuleParam extends RuleParam(false) - - object EnableDropAllCollabInvitationTweetsRuleParam extends RuleParam(false) - - object EnableNsfwTextHighPrecisionDropRuleParam extends RuleParam(false) - - object EnableLikelyIvsUserLabelDropRule extends RuleParam(false) - - object EnableCardUriRootDomainCardDenylistRule extends RuleParam(false) - object EnableCommunityNonMemberPollCardRule extends RuleParam(false) - object EnableCommunityNonMemberPollCardRuleFailClosed extends RuleParam(false) - - object EnableExperimentalNudgeEnabledParam extends RuleParam(false) - - object NsfwHighPrecisionUserLabelAvoidTweetRuleEnabledParam extends RuleParam(default = false) - - object EnableNewAdAvoidanceRulesParam extends RuleParam(false) - - object EnableNsfaHighRecallAdAvoidanceParam extends RuleParam(false) - - object EnableNsfaKeywordsHighPrecisionAdAvoidanceParam extends RuleParam(false) - - object EnableStaleTweetDropRuleParam extends RuleParam(false) - object EnableStaleTweetDropRuleFailClosedParam extends RuleParam(false) - - object EnableDeleteStateTweetRulesParam extends RuleParam(default = false) - - object EnableSpacesSharingNsfwDropRulesParam extends RuleParam(default = true) - - object EnableViewerIsSoftUserDropRuleParam extends RuleParam(default = false) - - object EnablePdnaQuotedTweetTombstoneRuleParam extends RuleParam(default = true) - object EnableSpamQuotedTweetTombstoneRuleParam extends RuleParam(default = true) - - object EnableNsfwHpQuotedTweetDropRuleParam extends RuleParam(default = false) - object EnableNsfwHpQuotedTweetTombstoneRuleParam extends RuleParam(default = false) - - object EnableInnerQuotedTweetViewerBlocksAuthorInterstitialRuleParam - extends RuleParam(default = false) - object EnableInnerQuotedTweetViewerMutesAuthorInterstitialRuleParam - extends RuleParam(default = false) - - - object EnableNewSensitiveMediaSettingsInterstitialsHomeTimelineRulesParam extends RuleParam(false) - - object EnableNewSensitiveMediaSettingsInterstitialsConversationRulesParam extends RuleParam(false) - - object EnableNewSensitiveMediaSettingsInterstitialsProfileTimelineRulesParam - extends RuleParam(false) - - object EnableNewSensitiveMediaSettingsInterstitialsTweetDetailRulesParam extends RuleParam(false) - - object EnableLegacySensitiveMediaHomeTimelineRulesParam extends RuleParam(true) - - object EnableLegacySensitiveMediaConversationRulesParam extends RuleParam(true) - - object EnableLegacySensitiveMediaProfileTimelineRulesParam extends RuleParam(true) - - object EnableLegacySensitiveMediaTweetDetailRulesParam extends RuleParam(true) - - object EnableLegacySensitiveMediaDirectMessagesRulesParam extends RuleParam(true) - - object EnableToxicReplyFilteringConversationRulesParam extends RuleParam(false) - object EnableToxicReplyFilteringNotificationsRulesParam extends RuleParam(false) - - object EnableSearchQueryMatchesTweetAuthorConditionParam extends RuleParam(default = false) - - object EnableSearchBasicBlockMuteRulesParam extends RuleParam(default = false) - - object EnableAbusiveBehaviorDropRuleParam extends RuleParam(default = false) - object EnableAbusiveBehaviorInterstitialRuleParam extends RuleParam(default = false) - object EnableAbusiveBehaviorLimitedEngagementsRuleParam extends RuleParam(default = false) - - object EnableNotGraduatedDownrankConvosAbusiveQualityRuleParam extends RuleParam(default = false) - object EnableNotGraduatedSearchDropRuleParam extends RuleParam(default = false) - object EnableNotGraduatedDropRuleParam extends RuleParam(default = false) - - object EnableFosnrRuleParam extends RuleParam(default = false) - - object EnableAuthorBlocksViewerDropRuleParam extends RuleParam(default = false) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/SafetyLevelParams.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/SafetyLevelParams.docx new file mode 100644 index 000000000..f92ada76d Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/SafetyLevelParams.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/SafetyLevelParams.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/SafetyLevelParams.scala deleted file mode 100644 index ae54ffd34..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/SafetyLevelParams.scala +++ /dev/null @@ -1,215 +0,0 @@ -package com.twitter.visibility.configapi.params - -import com.twitter.timelines.configapi.Param - -abstract class SafetyLevelParam(override val default: Boolean) extends Param(default) { - override val statName: String = s"SafetyLevelParam/${this.getClass.getSimpleName}" -} - -private[visibility] object SafetyLevelParams { - object EnableAccessInternalPromotedContentSafetyLevelParam extends SafetyLevelParam(false) - object EnableAdsBusinessSettingsSafetyLevelParam extends SafetyLevelParam(false) - object EnableAdsCampaignSafetyLevelParam extends SafetyLevelParam(false) - object EnableAdsManagerSafetyLevelParam extends SafetyLevelParam(false) - object EnableAdsReportingDashboardSafetyLevelParam extends SafetyLevelParam(false) - object EnableAllSubscribedListsSafetyLevelParam extends SafetyLevelParam(false) - object EnableAppealsSafetyLevelParam extends SafetyLevelParam(false) - object EnableArticleTweetTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableBaseQigSafetyLevelParam extends SafetyLevelParam(false) - object EnableBirdwatchNoteAuthorSafetyLevel extends SafetyLevelParam(false) - object EnableBirdwatchNoteTweetsTimelineSafetyLevel extends SafetyLevelParam(false) - object EnableBirdwatchNeedsYourHelpNotificationsSafetyLevel extends SafetyLevelParam(false) - object EnableBlockMuteUsersTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableBrandSafetySafetyLevelParam extends SafetyLevelParam(false) - object EnableCardPollVotingSafetyLevelParam extends SafetyLevelParam(false) - object EnableCardsServiceSafetyLevelParam extends SafetyLevelParam(false) - object EnableCommunitiesSafetyLevelParam extends SafetyLevelParam(false) - object EnableContentControlToolInstallSafetyLevelParam extends SafetyLevelParam(false) - object EnableConversationFocalPrehydrationSafetyLevelParam extends SafetyLevelParam(false) - object EnableConversationFocalTweetSafetyLevelParam extends SafetyLevelParam(false) - object EnableConversationInjectedTweetSafetyLevelParam extends SafetyLevelParam(false) - object EnableConversationReplySafetyLevelParam extends SafetyLevelParam(false) - object EnableCuratedTrendsRepresentativeTweet extends SafetyLevelParam(default = false) - object EnableCurationPolicyViolations extends SafetyLevelParam(default = false) - object EnableDevPlatformGetListTweetsSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESFollowingAndFollowersUserListSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESHomeTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESQuoteTweetTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESRealtimeSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESRealtimeSpamEnrichmentSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESRealtimeTweetFilterSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESRetweetingUsersSafetyLevelParam extends SafetyLevelParam(false) - object EnableDesTweetDetailSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESTweetLikingUsersSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESUserBookmarksSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESUserLikedTweetSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESUserMentionsSafetyLevelParam extends SafetyLevelParam(false) - object EnableDESUserTweetsSafetyLevelParam extends SafetyLevelParam(false) - object EnableDevPlatformComplianceStreamSafetyLevelParam extends SafetyLevelParam(false) - object EnableDirectMessagesSafetyLevelParam extends SafetyLevelParam(false) - object EnableDirectMessagesConversationListSafetyLevelParam extends SafetyLevelParam(false) - object EnableDirectMessagesConversationTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableDirectMessagesInboxSafetyLevelParam extends SafetyLevelParam(false) - object EnableDirectMessagesMutedUsersSafetyLevelParam extends SafetyLevelParam(false) - object EnableDirectMessagesPinnedSafetyLevelParam extends SafetyLevelParam(false) - object EnableDirectMessagesSearchSafetyLevelParam extends SafetyLevelParam(false) - object EnableEditHistoryTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableElevatedQuoteTweetTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableEmbeddedTweetSafetyLevelParam extends SafetyLevelParam(false) - object EnableEmbedsPublicInterestNoticeSafetyLevelParam extends SafetyLevelParam(false) - object EnableEmbedTweetMarkupSafetyLevelParam extends SafetyLevelParam(false) - object EnableWritePathLimitedActionsEnforcementSafetyLevelParam extends SafetyLevelParam(false) - object EnableFilterAllSafetyLevelParam extends SafetyLevelParam(false) - object EnableFilterAllPlaceholderSafetyLevelParam extends SafetyLevelParam(false) - object EnableFilterDefaultSafetyLevelParam extends SafetyLevelParam(false) - object EnableFilterNoneSafetyLevelParam extends SafetyLevelParam(false) - object EnableFollowedTopicsTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableFollowerConnectionsSafetyLevelParam extends SafetyLevelParam(false) - object EnableFollowingAndFollowersUserListSafetyLevelParam extends SafetyLevelParam(false) - object EnableForDevelopmentOnlySafetyLevelParam extends SafetyLevelParam(false) - object EnableFriendsFollowingListSafetyLevelParam extends SafetyLevelParam(false) - object EnableGraphqlDefaultSafetyLevelParam extends SafetyLevelParam(false) - object EnableGryphonDecksAndColumnsSafetyLevelParam extends SafetyLevelParam(false) - object EnableHumanizationNudgeSafetyLevelParam extends SafetyLevelParam(false) - object EnableKitchenSinkDevelopmentSafetyLevelParam extends SafetyLevelParam(false) - object EnableListHeaderSafetyLevelParam extends SafetyLevelParam(false) - object EnableListMembershipsSafetyLevelParam extends SafetyLevelParam(false) - object EnableListOwnershipsSafetyLevelParam extends SafetyLevelParam(false) - object EnableListRecommendationsSafetyLevelParam extends SafetyLevelParam(false) - object EnableListSearchSafetyLevelParam extends SafetyLevelParam(false) - object EnableListSubscriptionsSafetyLevelParam extends SafetyLevelParam(false) - object EnableLivePipelineEngagementCountsSafetyLevelParam extends SafetyLevelParam(false) - object EnableLiveVideoTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableMagicRecsAggressiveSafetyLevelParam extends SafetyLevelParam(false) - object EnableMagicRecsAggressiveV2SafetyLevelParam extends SafetyLevelParam(false) - object EnableMagicRecsSafetyLevelParam extends SafetyLevelParam(false) - object EnableMagicRecsV2SafetyLevelParam extends SafetyLevelParam(false) - object EnableMinimalSafetyLevelParam extends SafetyLevelParam(false) - object EnableModeratedTweetsTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableMomentsSafetyLevelParam extends SafetyLevelParam(false) - object EnableNearbySafetyLevelParam extends SafetyLevelParam(false) - object EnableNewUserExperienceSafetyLevelParam extends SafetyLevelParam(false) - object EnableNotificationsIbisSafetyLevelParam extends SafetyLevelParam(false) - object EnableNotificationsPlatformSafetyLevelParam extends SafetyLevelParam(false) - object EnableNotificationsPlatformPushSafetyLevelParam extends SafetyLevelParam(false) - object EnableNotificationsQigSafetyLevelParam extends SafetyLevelParam(false) - object EnableNotificationsReadSafetyLevelParam extends SafetyLevelParam(false) - object EnableNotificationsTimelineDeviceFollowSafetyLevelParam extends SafetyLevelParam(false) - object EnableNotificationsWriteSafetyLevelParam extends SafetyLevelParam(false) - object EnableNotificationsWriterTweetHydratorSafetyLevelParam extends SafetyLevelParam(false) - object EnableNotificationsWriterV2SafetyLevelParam extends SafetyLevelParam(false) - object EnableQuickPromoteTweetEligibilitySafetyLevelParam extends SafetyLevelParam(false) - object EnableQuoteTweetTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableRecommendationsSafetyLevelParam extends SafetyLevelParam(false) - object EnableRecommendationsWithoutNsfaSafetyLevelParam extends SafetyLevelParam(false) - object EnableRecosVideoSafetyLevelParam extends SafetyLevelParam(false) - object EnableRecosWritePathSafetyLevelParam extends SafetyLevelParam(false) - object EnableRepliesGroupingSafetyLevelParam extends SafetyLevelParam(false) - object EnableReportCenterSafetyLevelParam extends SafetyLevelParam(false) - object EnableReturningUserExperienceFocalTweetSafetyLevelParam extends SafetyLevelParam(false) - object EnableReturningUserExperienceSafetyLevelParam extends SafetyLevelParam(false) - object EnableRevenueSafetyLevelParam extends SafetyLevelParam(false) - object EnableRitoActionedTweetTimelineParam extends SafetyLevelParam(false) - object EnableSafeSearchMinimalSafetyLevelParam extends SafetyLevelParam(false) - object EnableSafeSearchStrictSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchHydrationSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchLatestSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchTopSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchTopQigSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchPhotoSafetyLevelParam extends SafetyLevelParam(false) - object SearchTrendTakeoverPromotedTweetSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchVideoSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchBlenderUserRulesSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchLatestUserRulesSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchPeopleSearchResultPageSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchPeopleTypeaheadSafetyLevelParam extends SafetyLevelParam(false) - object EnableUserSearchSrpSafetyLevelParam extends SafetyLevelParam(false) - object EnableUserSearchTypeaheadSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchMixerSrpMinimalSafetyLevelParam extends SafetyLevelParam(false) - object EnableSearchMixerSrpStrictSafetyLevelParam extends SafetyLevelParam(false) - object EnableShoppingManagerSpyModeSafetyLevelParam extends SafetyLevelParam(false) - object EnableSignalsReactionsSafetyLevelParam extends SafetyLevelParam(false) - object EnableSignalsTweetReactingUsersSafetyLevelParam extends SafetyLevelParam(false) - object EnableSocialProofSafetyLevelParam extends SafetyLevelParam(false) - object EnableSoftInterventionPivotSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpaceFleetlineSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpaceHomeTimelineUprankingSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpaceJoinScreenSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpaceNotificationsSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpacesSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpacesParticipantsSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpaceNotificationSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpacesSellerApplicationStatusSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpacesSharingSafetyLevelParam extends SafetyLevelParam(false) - object EnableSpaceTweetAvatarHomeTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableStickersTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableStratoExtLimitedEngagementsSafetyLevelParam extends SafetyLevelParam(false) - object EnableStreamServicesSafetyLevelParam extends SafetyLevelParam(false) - object EnableSuperFollowerConnectionsSafetyLevelParam extends SafetyLevelParam(false) - object EnableSuperLikeSafetyLevelParam extends SafetyLevelParam(false) - object EnableTestSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineBookmarkSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineConversationsDownrankingSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineContentControlsSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineConversationsDownrankingMinimalSafetyLevelParam - extends SafetyLevelParam(false) - object EnableTimelineConversationsSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineFavoritesSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineFavoritesSelfViewSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineFocalTweetSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineFollowingActivitySafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineHomeCommunitiesSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineHomeHydrationSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineHomeLatestSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineHomePromotedHydrationSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineHomeRecommendationsSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineHomeSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineHomeTopicFollowRecommendationsSafetyLevelParam - extends SafetyLevelParam(false) - object EnableTimelineScorerSafetyLevelParam extends SafetyLevelParam(false) - object EnableTopicsLandingPageTopicRecommendationsSafetyLevelParam extends SafetyLevelParam(false) - object EnableExploreRecommendationsSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineInjectionSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineLikedBySafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineListsSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineMediaSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineMentionsSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineModeratedTweetsHydrationSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineProfileSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineProfileAllSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineProfileSpacesSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineProfileSuperFollowsSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineReactiveBlendingSafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineRetweetedBySafetyLevelParam extends SafetyLevelParam(false) - object EnableTimelineSuperLikedBySafetyLevelParam extends SafetyLevelParam(false) - object EnableTombstoningSafetyLevelParam extends SafetyLevelParam(false) - object EnableTopicRecommendationsSafetyLevelParam extends SafetyLevelParam(false) - object EnableTrendsRepresentativeTweetSafetyLevelParam extends SafetyLevelParam(false) - object EnableTrustedFriendsUserListSafetyLevelParam extends SafetyLevelParam(false) - object EnableTwitterDelegateUserListSafetyLevelParam extends SafetyLevelParam(false) - object EnableTweetDetailSafetyLevelParam extends SafetyLevelParam(false) - object EnableTweetDetailNonTooSafetyLevelParam extends SafetyLevelParam(false) - object EnableTweetDetailWithInjectionsHydrationSafetyLevelParam extends SafetyLevelParam(false) - object EnableTweetEngagersSafetyLevelParam extends SafetyLevelParam(false) - object EnableTweetReplyNudgeParam extends SafetyLevelParam(false) - object EnableTweetScopedTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableTweetWritesApiSafetyLevelParam extends SafetyLevelParam(false) - object EnableTwitterArticleComposeSafetyLevelParam extends SafetyLevelParam(false) - object EnableTwitterArticleProfileTabSafetyLevelParam extends SafetyLevelParam(false) - object EnableTwitterArticleReadSafetyLevelParam extends SafetyLevelParam(false) - object EnableUserProfileHeaderSafetyLevelParam extends SafetyLevelParam(false) - object EnableProfileMixerMediaSafetyLevelParam extends SafetyLevelParam(false) - object EnableProfileMixerFavoritesSafetyLevelParam extends SafetyLevelParam(false) - object EnableUserMilestoneRecommendationSafetyLevelParam extends SafetyLevelParam(false) - object EnableUserScopedTimelineSafetyLevelParam extends SafetyLevelParam(false) - object EnableUserSelfViewOnlySafetyLevelParam extends SafetyLevelParam(false) - object EnableUserSettingsSafetyLevelParam extends SafetyLevelParam(false) - object EnableVideoAdsSafetyLevelParam extends SafetyLevelParam(false) - object EnableZipbirdConsumerArchivesSafetyLevelParam extends SafetyLevelParam(false) - object EnableTweetAwardSafetyLevelParam extends SafetyLevelParam(false) - - object EnableDeprecatedSafetyLevel extends SafetyLevelParam(true) - object EnableQuotedTweetRulesParam extends SafetyLevelParam(true) - object EnableUnsupportedSafetyLevel extends SafetyLevelParam(true) - object EnableUnknownSafetyLevel$ extends SafetyLevelParam(true) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/TimelineConversationsDownrankingSpecificParams.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/TimelineConversationsDownrankingSpecificParams.docx new file mode 100644 index 000000000..06c9ae81f Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/TimelineConversationsDownrankingSpecificParams.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/TimelineConversationsDownrankingSpecificParams.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/TimelineConversationsDownrankingSpecificParams.scala deleted file mode 100644 index eacabbabd..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/TimelineConversationsDownrankingSpecificParams.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.visibility.configapi.params - -private[visibility] object TimelineConversationsDownrankingSpecificParams { - - object EnablePSpammyTweetDownrankConvosLowQualityParam extends RuleParam(false) - - object EnableRitoActionedTweetDownrankConvosLowQualityParam extends RuleParam(false) - - object EnableHighSpammyTweetContentScoreConvoDownrankAbusiveQualityRuleParam - extends RuleParam(false) - - object EnableHighCryptospamScoreConvoDownrankAbusiveQualityRuleParam extends RuleParam(false) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiment.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiment.docx new file mode 100644 index 000000000..05aaf9db4 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiment.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiment.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiment.scala deleted file mode 100644 index 39af78a6b..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiment.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.visibility.configapi.params - -import com.twitter.timelines.configapi.BucketName -import com.twitter.timelines.configapi.Experiment -import com.twitter.timelines.configapi.UseFeatureContext - -object VisibilityExperiment { - val Control = "control" - val Treatment = "treatment" -} - -abstract class VisibilityExperiment(experimentKey: String) - extends Experiment(experimentKey) - with UseFeatureContext { - val TreatmentBucket: String = VisibilityExperiment.Treatment - override def experimentBuckets: Set[BucketName] = Set(TreatmentBucket) - val ControlBucket: String = VisibilityExperiment.Control - override def controlBuckets: Set[BucketName] = Set(ControlBucket) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiments.docx b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiments.docx new file mode 100644 index 000000000..c96866bea Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiments.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiments.scala b/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiments.scala deleted file mode 100644 index b287783e6..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/configapi/params/VisibilityExperiments.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.visibility.configapi.params - -private[visibility] object VisibilityExperiments { - - case object TestExperiment extends VisibilityExperiment("vf_test_ddg_7727") - - object CommonBucketId extends Enumeration { - type CommonBucketId = Value - val Control = Value("control") - val Treatment = Value("treatment") - val None = Value("none") - } - - case object NotGraduatedUserLabelRuleExperiment - extends VisibilityExperiment("not_graduated_user_holdback_16332") -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/engine/BUILD deleted file mode 100644 index 5e3503374..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/engine/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "abdecider/src/main/scala", - "configapi/configapi-core", - "servo/util/src/main/scala", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "visibility/lib/src/main/scala/com/twitter/visibility/builder", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/params", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/models", - "visibility/lib/src/main/scala/com/twitter/visibility/rules", - "visibility/lib/src/main/scala/com/twitter/visibility/rules/providers", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/engine/BUILD.docx new file mode 100644 index 000000000..896d6b0d5 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/engine/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/DeciderableVisibilityRuleEngine.docx b/visibilitylib/src/main/scala/com/twitter/visibility/engine/DeciderableVisibilityRuleEngine.docx new file mode 100644 index 000000000..11f23baf8 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/engine/DeciderableVisibilityRuleEngine.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/DeciderableVisibilityRuleEngine.scala b/visibilitylib/src/main/scala/com/twitter/visibility/engine/DeciderableVisibilityRuleEngine.scala deleted file mode 100644 index cb1119ce3..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/engine/DeciderableVisibilityRuleEngine.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.visibility.engine - -import com.twitter.servo.util.Gate -import com.twitter.spam.rtf.thriftscala.{SafetyLevel => ThriftSafetyLevel} -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.VisibilityResultBuilder -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.rules.EvaluationContext -import com.twitter.visibility.rules.Rule - -trait DeciderableVisibilityRuleEngine { - def apply( - evaluationContext: EvaluationContext, - safetyLevel: SafetyLevel, - visibilityResultBuilder: VisibilityResultBuilder, - enableShortCircuiting: Gate[Unit] = Gate.True, - preprocessedRules: Option[Seq[Rule]] = None - ): Stitch[VisibilityResult] - - def apply( - evaluationContext: EvaluationContext, - thriftSafetyLevel: ThriftSafetyLevel, - visibilityResultBuilder: VisibilityResultBuilder - ): Stitch[VisibilityResult] -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityResultsMetricRecorder.docx b/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityResultsMetricRecorder.docx new file mode 100644 index 000000000..1b92ead5d Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityResultsMetricRecorder.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityResultsMetricRecorder.scala b/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityResultsMetricRecorder.scala deleted file mode 100644 index 97af1d024..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityResultsMetricRecorder.scala +++ /dev/null @@ -1,179 +0,0 @@ -package com.twitter.visibility.engine - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.stats.Verbosity -import com.twitter.servo.util.Gate -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.features.Feature -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.rules.NotEvaluated -import com.twitter.visibility.rules.RuleResult -import com.twitter.visibility.rules.State -import com.twitter.visibility.rules.State.Disabled -import com.twitter.visibility.rules.State.FeatureFailed -import com.twitter.visibility.rules.State.MissingFeature -import com.twitter.visibility.rules.State.RuleFailed -import com.twitter.visibility.rules.Action - - -case class VisibilityResultsMetricRecorder( - statsReceiver: StatsReceiver, - captureDebugStats: Gate[Unit]) { - - private val scopedStatsReceiver = new MemoizingStatsReceiver( - statsReceiver.scope("visibility_rule_engine") - ) - private val actionStats: StatsReceiver = scopedStatsReceiver.scope("by_action") - private val featureFailureReceiver: StatsReceiver = - scopedStatsReceiver.scope("feature_failed") - private val safetyLevelStatsReceiver: StatsReceiver = - scopedStatsReceiver.scope("from_safety_level") - private val ruleStatsReceiver: StatsReceiver = scopedStatsReceiver.scope("for_rule") - private val ruleFailureReceiver: StatsReceiver = - scopedStatsReceiver.scope("rule_failures") - private val failClosedReceiver: StatsReceiver = - scopedStatsReceiver.scope("fail_closed") - private val ruleStatsBySafetyLevelReceiver: StatsReceiver = - scopedStatsReceiver.scope("for_rule_by_safety_level") - - def recordSuccess( - safetyLevel: SafetyLevel, - result: VisibilityResult - ): Unit = { - recordAction(safetyLevel, result.verdict.fullName) - - val isFeatureFailure = result.ruleResultMap.values - .collectFirst { - case RuleResult(_, FeatureFailed(_)) => - ruleFailureReceiver.counter("feature_failed").incr() - true - }.getOrElse(false) - - val isMissingFeature = result.ruleResultMap.values - .collectFirst { - case RuleResult(_, MissingFeature(_)) => - ruleFailureReceiver.counter("missing_feature").incr() - true - }.getOrElse(false) - - val isRuleFailed = result.ruleResultMap.values - .collectFirst { - case RuleResult(_, RuleFailed(_)) => - ruleFailureReceiver.counter("rule_failed").incr() - true - }.getOrElse(false) - - if (isFeatureFailure || isMissingFeature || isRuleFailed) { - ruleFailureReceiver.counter().incr() - } - - if (captureDebugStats()) { - val ruleBySafetyLevelStat = - ruleStatsBySafetyLevelReceiver.scope(safetyLevel.name) - result.ruleResultMap.foreach { - case (rule, ruleResult) => { - ruleBySafetyLevelStat - .scope(rule.name) - .scope("action") - .counter(Verbosity.Debug, ruleResult.action.fullName).incr() - ruleBySafetyLevelStat - .scope(rule.name) - .scope("state") - .counter(Verbosity.Debug, ruleResult.state.name).incr() - } - } - } - } - - def recordFailedFeature( - failedFeature: Feature[_], - exception: Throwable - ): Unit = { - featureFailureReceiver.counter().incr() - - val featureStat = featureFailureReceiver.scope(failedFeature.name) - featureStat.counter().incr() - featureStat.counter(exception.getClass.getName).incr() - } - - def recordAction( - safetyLevel: SafetyLevel, - action: String - ): Unit = { - safetyLevelStatsReceiver.scope(safetyLevel.name).counter(action).incr() - actionStats.counter(action).incr() - } - - def recordUnknownSafetyLevel( - safetyLevel: SafetyLevel - ): Unit = { - safetyLevelStatsReceiver - .scope("unknown_safety_level") - .counter(safetyLevel.name.toLowerCase).incr() - } - - def recordRuleMissingFeatures( - ruleName: String, - missingFeatures: Set[Feature[_]] - ): Unit = { - val ruleStat = ruleStatsReceiver.scope(ruleName) - missingFeatures.foreach { featureId => - ruleStat.scope("missing_feature").counter(featureId.name).incr() - } - ruleStat.scope("action").counter(NotEvaluated.fullName).incr() - ruleStat.scope("state").counter(MissingFeature(missingFeatures).name).incr() - } - - def recordRuleFailedFeatures( - ruleName: String, - failedFeatures: Map[Feature[_], Throwable] - ): Unit = { - val ruleStat = ruleStatsReceiver.scope(ruleName) - - ruleStat.scope("action").counter(NotEvaluated.fullName).incr() - ruleStat.scope("state").counter(FeatureFailed(failedFeatures).name).incr() - } - - def recordFailClosed(rule: String, state: State) { - failClosedReceiver.scope(state.name).counter(rule).incr(); - } - - def recordRuleEvaluation( - ruleName: String, - action: Action, - state: State - ): Unit = { - val ruleStat = ruleStatsReceiver.scope(ruleName) - ruleStat.scope("action").counter(action.fullName).incr() - ruleStat.scope("state").counter(state.name).incr() - } - - - def recordRuleFallbackAction( - ruleName: String - ): Unit = { - val ruleStat = ruleStatsReceiver.scope(ruleName) - ruleStat.counter("fallback_action").incr() - } - - def recordRuleHoldBack( - ruleName: String - ): Unit = { - ruleStatsReceiver.scope(ruleName).counter("heldback").incr() - } - - def recordRuleFailed( - ruleName: String - ): Unit = { - ruleStatsReceiver.scope(ruleName).counter("failed").incr() - } - - def recordDisabledRule( - ruleName: String - ): Unit = recordRuleEvaluation(ruleName, NotEvaluated, Disabled) -} - -object NullVisibilityResultsMetricsRecorder - extends VisibilityResultsMetricRecorder(NullStatsReceiver, Gate.False) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRuleEngine.docx b/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRuleEngine.docx new file mode 100644 index 000000000..7d897e74b Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRuleEngine.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRuleEngine.scala b/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRuleEngine.scala deleted file mode 100644 index d1c33017b..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRuleEngine.scala +++ /dev/null @@ -1,266 +0,0 @@ -package com.twitter.visibility.engine - -import com.twitter.servo.util.Gate -import com.twitter.spam.rtf.thriftscala.{SafetyLevel => ThriftSafetyLevel} -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.VisibilityResultBuilder -import com.twitter.visibility.features._ -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.SafetyLevel.DeprecatedSafetyLevel -import com.twitter.visibility.rules.EvaluationContext -import com.twitter.visibility.rules.State._ -import com.twitter.visibility.rules._ -import com.twitter.visibility.rules.providers.ProvidedEvaluationContext -import com.twitter.visibility.rules.providers.PolicyProvider - -class VisibilityRuleEngine private[VisibilityRuleEngine] ( - rulePreprocessor: VisibilityRulePreprocessor, - metricsRecorder: VisibilityResultsMetricRecorder, - enableComposableActions: Gate[Unit], - enableFailClosed: Gate[Unit], - policyProviderOpt: Option[PolicyProvider] = None) - extends DeciderableVisibilityRuleEngine { - - private[visibility] def apply( - evaluationContext: ProvidedEvaluationContext, - visibilityPolicy: VisibilityPolicy, - visibilityResultBuilder: VisibilityResultBuilder, - enableShortCircuiting: Gate[Unit], - preprocessedRules: Option[Seq[Rule]] - ): Stitch[VisibilityResult] = { - val (resultBuilder, rules) = preprocessedRules match { - case Some(r) => - (visibilityResultBuilder, r) - case None => - rulePreprocessor.evaluate(evaluationContext, visibilityPolicy, visibilityResultBuilder) - } - evaluate(evaluationContext, resultBuilder, rules, enableShortCircuiting) - } - - def apply( - evaluationContext: EvaluationContext, - safetyLevel: SafetyLevel, - visibilityResultBuilder: VisibilityResultBuilder, - enableShortCircuiting: Gate[Unit] = Gate.True, - preprocessedRules: Option[Seq[Rule]] = None - ): Stitch[VisibilityResult] = { - val visibilityPolicy = policyProviderOpt match { - case Some(policyProvider) => - policyProvider.policyForSurface(safetyLevel) - case None => RuleBase.RuleMap(safetyLevel) - } - if (evaluationContext.params(safetyLevel.enabledParam)) { - apply( - ProvidedEvaluationContext.injectRuntimeRulesIntoEvaluationContext( - evaluationContext = evaluationContext, - safetyLevel = Some(safetyLevel), - policyProviderOpt = policyProviderOpt - ), - visibilityPolicy, - visibilityResultBuilder, - enableShortCircuiting, - preprocessedRules - ).onSuccess { result => - metricsRecorder.recordSuccess(safetyLevel, result) - } - .onFailure { _ => - metricsRecorder.recordAction(safetyLevel, "failure") - } - } else { - metricsRecorder.recordAction(safetyLevel, "disabled") - val rules: Seq[Rule] = visibilityPolicy.forContentId(visibilityResultBuilder.contentId) - Stitch.value( - visibilityResultBuilder - .withRuleResultMap(rules.map(r => r -> RuleResult(Allow, Skipped)).toMap) - .withVerdict(verdict = Allow) - .withFinished(finished = true) - .build - ) - } - } - - def apply( - evaluationContext: EvaluationContext, - thriftSafetyLevel: ThriftSafetyLevel, - visibilityResultBuilder: VisibilityResultBuilder - ): Stitch[VisibilityResult] = { - val safetyLevel: SafetyLevel = SafetyLevel.fromThrift(thriftSafetyLevel) - safetyLevel match { - case DeprecatedSafetyLevel => - metricsRecorder.recordUnknownSafetyLevel(safetyLevel) - Stitch.value( - visibilityResultBuilder - .withVerdict(verdict = Allow) - .withFinished(finished = true) - .build - ) - - case thriftSafetyLevel: SafetyLevel => - this( - ProvidedEvaluationContext.injectRuntimeRulesIntoEvaluationContext( - evaluationContext = evaluationContext, - safetyLevel = Some(safetyLevel), - policyProviderOpt = policyProviderOpt - ), - thriftSafetyLevel, - visibilityResultBuilder - ) - } - } - - private[visibility] def evaluateRules( - evaluationContext: ProvidedEvaluationContext, - resolvedFeatureMap: Map[Feature[_], Any], - failedFeatures: Map[Feature[_], Throwable], - resultBuilderWithoutFailedFeatures: VisibilityResultBuilder, - preprocessedRules: Seq[Rule], - enableShortCircuiting: Gate[Unit] - ): VisibilityResultBuilder = { - preprocessedRules - .foldLeft(resultBuilderWithoutFailedFeatures) { (builder, rule) => - builder.ruleResults.get(rule) match { - case Some(RuleResult(_, state)) if state == Evaluated || state == ShortCircuited => - builder - - case _ => - val failedFeatureDependencies: Map[Feature[_], Throwable] = - failedFeatures.filterKeys(key => rule.featureDependencies.contains(key)) - - val shortCircuit = - builder.finished && enableShortCircuiting() && - !(enableComposableActions() && builder.isVerdictComposable()) - - if (failedFeatureDependencies.nonEmpty && rule.fallbackActionBuilder.isEmpty) { - metricsRecorder.recordRuleFailedFeatures(rule.name, failedFeatureDependencies) - builder.withRuleResult( - rule, - RuleResult(NotEvaluated, FeatureFailed(failedFeatureDependencies))) - - } else if (shortCircuit) { - - metricsRecorder.recordRuleEvaluation(rule.name, NotEvaluated, ShortCircuited) - builder.withRuleResult(rule, RuleResult(builder.verdict, ShortCircuited)) - } else { - - if (failedFeatureDependencies.nonEmpty && rule.fallbackActionBuilder.nonEmpty) { - metricsRecorder.recordRuleFallbackAction(rule.name) - } - - - val ruleResult = - rule.evaluate(evaluationContext, resolvedFeatureMap) - metricsRecorder - .recordRuleEvaluation(rule.name, ruleResult.action, ruleResult.state) - val nextBuilder = (ruleResult.action, builder.finished) match { - case (NotEvaluated | Allow, _) => - ruleResult.state match { - case Heldback => - metricsRecorder.recordRuleHoldBack(rule.name) - case RuleFailed(_) => - metricsRecorder.recordRuleFailed(rule.name) - case _ => - } - builder.withRuleResult(rule, ruleResult) - - case (_, true) => - builder - .withRuleResult(rule, ruleResult) - .withSecondaryVerdict(ruleResult.action, rule) - - case _ => - builder - .withRuleResult(rule, ruleResult) - .withVerdict(ruleResult.action, Some(rule)) - .withFinished(true) - } - - nextBuilder - } - } - }.withResolvedFeatureMap(resolvedFeatureMap) - } - - private[visibility] def evaluateFailClosed( - evaluationContext: ProvidedEvaluationContext - ): VisibilityResultBuilder => Stitch[VisibilityResultBuilder] = { builder => - builder.failClosedException(evaluationContext) match { - case Some(e: FailClosedException) if enableFailClosed() => - metricsRecorder.recordFailClosed(e.getRuleName, e.getState); - Stitch.exception(e) - case _ => Stitch.value(builder) - } - } - - private[visibility] def checkMarkFinished( - builder: VisibilityResultBuilder - ): VisibilityResult = { - val allRulesEvaluated: Boolean = builder.ruleResults.values.forall { - case RuleResult(_, state) => - state == Evaluated || state == Disabled || state == Skipped - case _ => - false - } - - if (allRulesEvaluated) { - builder.withFinished(true).build - } else { - builder.build - } - } - - private[visibility] def evaluate( - evaluationContext: ProvidedEvaluationContext, - visibilityResultBuilder: VisibilityResultBuilder, - preprocessedRules: Seq[Rule], - enableShortCircuiting: Gate[Unit] = Gate.True - ): Stitch[VisibilityResult] = { - - val finalBuilder = - FeatureMap.resolve(visibilityResultBuilder.features, evaluationContext.statsReceiver).map { - resolvedFeatureMap => - val (failedFeatureMap, successfulFeatureMap) = resolvedFeatureMap.constantMap.partition({ - case (_, _: FeatureFailedPlaceholderObject) => true - case _ => false - }) - - val failedFeatures: Map[Feature[_], Throwable] = - failedFeatureMap.mapValues({ - case failurePlaceholder: FeatureFailedPlaceholderObject => - failurePlaceholder.throwable - }) - - val resultBuilderWithoutFailedFeatures = - visibilityResultBuilder.withFeatureMap(ResolvedFeatureMap(successfulFeatureMap)) - - evaluateRules( - evaluationContext, - successfulFeatureMap, - failedFeatures, - resultBuilderWithoutFailedFeatures, - preprocessedRules, - enableShortCircuiting - ) - } - - finalBuilder.flatMap(evaluateFailClosed(evaluationContext)).map(checkMarkFinished) - } -} - -object VisibilityRuleEngine { - - def apply( - rulePreprocessor: Option[VisibilityRulePreprocessor] = None, - metricsRecorder: VisibilityResultsMetricRecorder = NullVisibilityResultsMetricsRecorder, - enableComposableActions: Gate[Unit] = Gate.False, - enableFailClosed: Gate[Unit] = Gate.False, - policyProviderOpt: Option[PolicyProvider] = None, - ): VisibilityRuleEngine = { - new VisibilityRuleEngine( - rulePreprocessor.getOrElse(VisibilityRulePreprocessor(metricsRecorder)), - metricsRecorder, - enableComposableActions, - enableFailClosed, - policyProviderOpt = policyProviderOpt) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRulePreprocessor.docx b/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRulePreprocessor.docx new file mode 100644 index 000000000..4732d467a Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRulePreprocessor.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRulePreprocessor.scala b/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRulePreprocessor.scala deleted file mode 100644 index 115c37605..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/engine/VisibilityRulePreprocessor.scala +++ /dev/null @@ -1,156 +0,0 @@ -package com.twitter.visibility.engine - -import com.twitter.abdecider.NullABDecider -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import com.twitter.visibility.builder.VisibilityResultBuilder -import com.twitter.visibility.features._ -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.rules.Rule.DisabledRuleResult -import com.twitter.visibility.rules.Rule.EvaluatedRuleResult -import com.twitter.visibility.rules.State._ -import com.twitter.visibility.rules._ -import com.twitter.visibility.rules.providers.ProvidedEvaluationContext -import com.twitter.visibility.rules.providers.PolicyProvider - -class VisibilityRulePreprocessor private ( - metricsRecorder: VisibilityResultsMetricRecorder, - policyProviderOpt: Option[PolicyProvider] = None) { - - private[engine] def filterEvaluableRules( - evaluationContext: ProvidedEvaluationContext, - resultBuilder: VisibilityResultBuilder, - rules: Seq[Rule] - ): (VisibilityResultBuilder, Seq[Rule]) = { - val (builder, ruleList) = rules.foldLeft((resultBuilder, Seq.empty[Rule])) { - case ((builder, nextPassRules), rule) => - if (evaluationContext.ruleEnabledInContext(rule)) { - val missingFeatures: Set[Feature[_]] = rule.featureDependencies.collect { - case feature: Feature[_] if !builder.featureMap.contains(feature) => feature - } - - if (missingFeatures.isEmpty) { - (builder, nextPassRules :+ rule) - } else { - metricsRecorder.recordRuleMissingFeatures(rule.name, missingFeatures) - ( - builder.withRuleResult( - rule, - RuleResult(NotEvaluated, MissingFeature(missingFeatures)) - ), - nextPassRules - ) - } - } else { - (builder.withRuleResult(rule, DisabledRuleResult), nextPassRules) - } - } - (builder, ruleList) - } - - private[visibility] def preFilterRules( - evaluationContext: ProvidedEvaluationContext, - resolvedFeatureMap: Map[Feature[_], Any], - resultBuilder: VisibilityResultBuilder, - rules: Seq[Rule] - ): (VisibilityResultBuilder, Seq[Rule]) = { - val isResolvedFeatureMap = resultBuilder.featureMap.isInstanceOf[ResolvedFeatureMap] - val (builder, ruleList) = rules.foldLeft((resultBuilder, Seq.empty[Rule])) { - case ((builder, nextPassRules), rule) => - rule.preFilter(evaluationContext, resolvedFeatureMap, NullABDecider) match { - case NeedsFullEvaluation => - (builder, nextPassRules :+ rule) - case NotFiltered => - (builder, nextPassRules :+ rule) - case Filtered if isResolvedFeatureMap => - (builder, nextPassRules :+ rule) - case Filtered => - (builder.withRuleResult(rule, EvaluatedRuleResult), nextPassRules) - } - } - (builder, ruleList) - } - - private[visibility] def evaluate( - evaluationContext: ProvidedEvaluationContext, - safetyLevel: SafetyLevel, - resultBuilder: VisibilityResultBuilder - ): (VisibilityResultBuilder, Seq[Rule]) = { - val visibilityPolicy = policyProviderOpt match { - case Some(policyProvider) => - policyProvider.policyForSurface(safetyLevel) - case None => RuleBase.RuleMap(safetyLevel) - } - - if (evaluationContext.params(safetyLevel.enabledParam)) { - evaluate(evaluationContext, visibilityPolicy, resultBuilder) - } else { - metricsRecorder.recordAction(safetyLevel, "disabled") - - val rules: Seq[Rule] = visibilityPolicy.forContentId(resultBuilder.contentId) - val skippedResultBuilder = resultBuilder - .withRuleResultMap(rules.map(r => r -> RuleResult(Allow, Skipped)).toMap) - .withVerdict(verdict = Allow) - .withFinished(finished = true) - - (skippedResultBuilder, rules) - } - } - - private[visibility] def evaluate( - evaluationContext: ProvidedEvaluationContext, - visibilityPolicy: VisibilityPolicy, - resultBuilder: VisibilityResultBuilder, - ): (VisibilityResultBuilder, Seq[Rule]) = { - - val rules: Seq[Rule] = visibilityPolicy.forContentId(resultBuilder.contentId) - - val (secondPassBuilder, secondPassRules) = - filterEvaluableRules(evaluationContext, resultBuilder, rules) - - val secondPassFeatureMap = secondPassBuilder.featureMap - - val secondPassConstantFeatures: Set[Feature[_]] = RuleBase - .getFeaturesForRules(secondPassRules) - .filter(secondPassFeatureMap.containsConstant(_)) - - val secondPassFeatureValues: Set[(Feature[_], Any)] = secondPassConstantFeatures.map { - feature => - Try(secondPassFeatureMap.getConstant(feature)) match { - case Return(value) => (feature, value) - case Throw(ex) => - metricsRecorder.recordFailedFeature(feature, ex) - (feature, FeatureFailedPlaceholderObject(ex)) - } - } - - val resolvedFeatureMap: Map[Feature[_], Any] = - secondPassFeatureValues.filterNot { - case (_, value) => value.isInstanceOf[FeatureFailedPlaceholderObject] - }.toMap - - val (preFilteredResultBuilder, preFilteredRules) = preFilterRules( - evaluationContext, - resolvedFeatureMap, - secondPassBuilder, - secondPassRules - ) - - val preFilteredFeatureMap = - RuleBase.removeUnusedFeaturesFromFeatureMap( - preFilteredResultBuilder.featureMap, - preFilteredRules) - - (preFilteredResultBuilder.withFeatureMap(preFilteredFeatureMap), preFilteredRules) - } -} - -object VisibilityRulePreprocessor { - def apply( - metricsRecorder: VisibilityResultsMetricRecorder, - policyProviderOpt: Option[PolicyProvider] = None - ): VisibilityRulePreprocessor = { - new VisibilityRulePreprocessor(metricsRecorder, policyProviderOpt) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/AdvancedFilteringFeatures.docx b/visibilitylib/src/main/scala/com/twitter/visibility/features/AdvancedFilteringFeatures.docx new file mode 100644 index 000000000..0106f92a8 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/features/AdvancedFilteringFeatures.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/AdvancedFilteringFeatures.scala b/visibilitylib/src/main/scala/com/twitter/visibility/features/AdvancedFilteringFeatures.scala deleted file mode 100644 index 4e6a33ba9..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/features/AdvancedFilteringFeatures.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.visibility.features - -import com.twitter.gizmoduck.thriftscala.MentionFilter -import com.twitter.util.Duration - -case object ViewerFiltersNoConfirmedEmail extends Feature[Boolean] - -case object ViewerFiltersNoConfirmedPhone extends Feature[Boolean] - -case object ViewerFiltersDefaultProfileImage extends Feature[Boolean] - -case object ViewerFiltersNewUsers extends Feature[Boolean] - -case object ViewerFiltersNotFollowedBy extends Feature[Boolean] - -case object ViewerMentionFilter extends Feature[MentionFilter] - -case object AuthorHasConfirmedEmail extends Feature[Boolean] - -case object AuthorHasVerifiedPhone extends Feature[Boolean] - -case object AuthorHasDefaultProfileImage extends Feature[Boolean] - -case object AuthorAccountAge extends Feature[Duration] diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/features/BUILD deleted file mode 100644 index 3573bb0db..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/features/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/squareup/okhttp:okhttp3", - "finagle/finagle-mux/src/main/scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "visibility/lib/src/main/scala/com/twitter/visibility/models", - "visibility/lib/src/main/scala/com/twitter/visibility/util", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/features/BUILD.docx new file mode 100644 index 000000000..ad09ed5ce Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/features/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/Feature.docx b/visibilitylib/src/main/scala/com/twitter/visibility/features/Feature.docx new file mode 100644 index 000000000..f0f35e594 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/features/Feature.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/Feature.scala b/visibilitylib/src/main/scala/com/twitter/visibility/features/Feature.scala deleted file mode 100644 index 151718814..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/features/Feature.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.visibility.features - -import com.twitter.visibility.util.NamingUtils - -abstract class Feature[T] protected ()(implicit val manifest: Manifest[T]) { - - lazy val name: String = NamingUtils.getFriendlyName(this) - - override lazy val toString: String = - "Feature[%s](name=%s)".format(manifest, getClass.getSimpleName) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/FeatureMap.docx b/visibilitylib/src/main/scala/com/twitter/visibility/features/FeatureMap.docx new file mode 100644 index 000000000..10af879ee Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/features/FeatureMap.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/FeatureMap.scala b/visibilitylib/src/main/scala/com/twitter/visibility/features/FeatureMap.scala deleted file mode 100644 index 1b4ffd182..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/features/FeatureMap.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.visibility.features - -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import scala.language.existentials - -class MissingFeatureException(feature: Feature[_]) extends Exception("Missing value for " + feature) - -case class FeatureFailedException(feature: Feature[_], exception: Throwable) extends Exception - -private[visibility] case class FeatureFailedPlaceholderObject(throwable: Throwable) - -class FeatureMap( - val map: Map[Feature[_], Stitch[_]], - val constantMap: Map[Feature[_], Any]) { - - def contains[T](feature: Feature[T]): Boolean = - constantMap.contains(feature) || map.contains(feature) - - def containsConstant[T](feature: Feature[T]): Boolean = constantMap.contains(feature) - - lazy val size: Int = keys.size - - lazy val keys: Set[Feature[_]] = constantMap.keySet ++ map.keySet - - def get[T](feature: Feature[T]): Stitch[T] = { - map.get(feature) match { - case _ if constantMap.contains(feature) => - Stitch.value(getConstant(feature)) - case Some(x) => - x.asInstanceOf[Stitch[T]] - case _ => - Stitch.exception(new MissingFeatureException(feature)) - } - } - - def getConstant[T](feature: Feature[T]): T = { - constantMap.get(feature) match { - case Some(x) => - x.asInstanceOf[T] - case _ => - throw new MissingFeatureException(feature) - } - } - - def -[T](key: Feature[T]): FeatureMap = new FeatureMap(map - key, constantMap - key) - - override def toString: String = "FeatureMap(%s, %s)".format(map, constantMap) -} - -object FeatureMap { - - def empty: FeatureMap = new FeatureMap(Map.empty, Map.empty) - - def resolve( - featureMap: FeatureMap, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Stitch[ResolvedFeatureMap] = { - val featureMapHydrationStatsReceiver = statsReceiver.scope("feature_map_hydration") - - Stitch - .traverse(featureMap.map.toSeq) { - case (feature, value: Stitch[_]) => - val featureStatsReceiver = featureMapHydrationStatsReceiver.scope(feature.name) - lazy val featureFailureStat = featureStatsReceiver.scope("failures") - val featureStitch: Stitch[(Feature[_], Any)] = value - .map { resolvedValue => - featureStatsReceiver.counter("success").incr() - (feature, resolvedValue) - } - - featureStitch - .handle { - case ffe: FeatureFailedException => - featureFailureStat.counter().incr() - featureFailureStat.counter(ffe.exception.getClass.getName).incr() - (feature, FeatureFailedPlaceholderObject(ffe.exception)) - } - .ensure { - featureStatsReceiver.counter("requests").incr() - } - } - .map { resolvedFeatures: Seq[(Feature[_], Any)] => - new ResolvedFeatureMap(resolvedFeatures.toMap ++ featureMap.constantMap) - } - } - - def rescueFeatureTuple(kv: (Feature[_], Stitch[_])): (Feature[_], Stitch[_]) = { - val (k, v) = kv - - val rescueValue = v.rescue { - case e => - e match { - case cdre: ClientDiscardedRequestException => Stitch.exception(cdre) - case _ => Stitch.exception(FeatureFailedException(k, e)) - } - } - - (k, rescueValue) - } -} - -class ResolvedFeatureMap(private[visibility] val resolvedMap: Map[Feature[_], Any]) - extends FeatureMap(Map.empty, resolvedMap) { - - override def equals(other: Any): Boolean = other match { - case otherResolvedFeatureMap: ResolvedFeatureMap => - this.resolvedMap.equals(otherResolvedFeatureMap.resolvedMap) - case _ => false - } - - override def toString: String = "ResolvedFeatureMap(%s)".format(resolvedMap) -} - -object ResolvedFeatureMap { - def apply(resolvedMap: Map[Feature[_], Any]): ResolvedFeatureMap = { - new ResolvedFeatureMap(resolvedMap) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/Features.docx b/visibilitylib/src/main/scala/com/twitter/visibility/features/Features.docx new file mode 100644 index 000000000..d98abbf9b Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/features/Features.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/features/Features.scala b/visibilitylib/src/main/scala/com/twitter/visibility/features/Features.scala deleted file mode 100644 index ae26dfe78..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/features/Features.scala +++ /dev/null @@ -1,269 +0,0 @@ -package com.twitter.visibility.features - -import com.twitter.contenthealth.toxicreplyfilter.thriftscala.FilterState -import com.twitter.gizmoduck.thriftscala.Label -import com.twitter.search.common.constants.thriftscala.ThriftQuerySource -import com.twitter.tseng.withholding.thriftscala.TakedownReason -import com.twitter.util.Duration -import com.twitter.util.Time -import com.twitter.visibility.models.TweetDeleteReason.TweetDeleteReason -import com.twitter.visibility.models._ - -case object AuthorId extends Feature[Set[Long]] - -case object ViewerId extends Feature[Long] - -case object AuthorIsProtected extends Feature[Boolean] - -case object AuthorIsSuspended extends Feature[Boolean] - -case object AuthorIsUnavailable extends Feature[Boolean] - -case object AuthorIsDeactivated extends Feature[Boolean] - -case object AuthorIsErased extends Feature[Boolean] - -case object AuthorIsOffboarded extends Feature[Boolean] - -case object AuthorIsVerified extends Feature[Boolean] - -case object AuthorIsBlueVerified extends Feature[Boolean] - -case object ViewerIsSuspended extends Feature[Boolean] - -case object ViewerIsDeactivated extends Feature[Boolean] - -case object AuthorFollowsViewer extends Feature[Boolean] - -case object AuthorUserLabels extends Feature[Seq[Label]] - -case object ViewerFollowsAuthorOfViolatingTweet extends Feature[Boolean] - -case object ViewerDoesNotFollowAuthorOfViolatingTweet extends Feature[Boolean] - -case object ViewerFollowsAuthor extends Feature[Boolean] - -case object ViewerBlocksAuthor extends Feature[Boolean] - -case object AuthorBlocksViewer extends Feature[Boolean] - -case object AuthorMutesViewer extends Feature[Boolean] - -case object ViewerMutesAuthor extends Feature[Boolean] - -case object AuthorReportsViewerAsSpam extends Feature[Boolean] - -case object ViewerReportsAuthorAsSpam extends Feature[Boolean] - -case object ViewerReportedTweet extends Feature[Boolean] - -case object ViewerMutesRetweetsFromAuthor extends Feature[Boolean] - -case object ViewerHasUniversalQualityFilterEnabled extends Feature[Boolean] - -case object ViewerIsProtected extends Feature[Boolean] - -case object ViewerIsSoftUser extends Feature[Boolean] - -case object TweetSafetyLabels extends Feature[Seq[TweetSafetyLabel]] - -case object SpaceSafetyLabels extends Feature[Seq[SpaceSafetyLabel]] - -case object MediaSafetyLabels extends Feature[Seq[MediaSafetyLabel]] - -case object TweetTakedownReasons extends Feature[Seq[TakedownReason]] - -case object AuthorTakedownReasons extends Feature[Seq[TakedownReason]] - -case object AuthorIsNsfwUser extends Feature[Boolean] - -case object AuthorIsNsfwAdmin extends Feature[Boolean] - -case object TweetHasNsfwUser extends Feature[Boolean] - -case object TweetHasNsfwAdmin extends Feature[Boolean] - -case object TweetHasMedia extends Feature[Boolean] - -case object CardHasMedia extends Feature[Boolean] - -case object TweetHasCard extends Feature[Boolean] - -case object ViewerMutesKeywordInTweetForHomeTimeline extends Feature[MutedKeyword] - -case object ViewerMutesKeywordInTweetForTweetReplies extends Feature[MutedKeyword] - -case object ViewerMutesKeywordInTweetForNotifications extends Feature[MutedKeyword] - -case object ViewerMutesKeywordInSpaceTitleForNotifications extends Feature[MutedKeyword] - -case object ViewerMutesKeywordInTweetForAllSurfaces extends Feature[MutedKeyword] - -case object ViewerUserLabels extends Feature[Seq[Label]] - -case object RequestCountryCode extends Feature[String] - -case object RequestIsVerifiedCrawler extends Feature[Boolean] - -case object ViewerCountryCode extends Feature[String] - -case object TweetIsSelfReply extends Feature[Boolean] - -case object TweetIsNullcast extends Feature[Boolean] - -case object TweetTimestamp extends Feature[Time] - -case object TweetIsInnerQuotedTweet extends Feature[Boolean] - -case object TweetIsRetweet extends Feature[Boolean] - -case object TweetIsSourceTweet extends Feature[Boolean] - -case object TweetDeleteReason extends Feature[TweetDeleteReason] - -case object TweetReplyToParentTweetDuration extends Feature[Duration] - -case object TweetReplyToRootTweetDuration extends Feature[Duration] - -case object TweetHasCommunityConversationControl extends Feature[Boolean] -case object TweetHasByInvitationConversationControl extends Feature[Boolean] -case object TweetHasFollowersConversationControl extends Feature[Boolean] -case object TweetConversationViewerIsInvited extends Feature[Boolean] -case object TweetConversationViewerIsInvitedViaReplyMention extends Feature[Boolean] -case object TweetConversationViewerIsRootAuthor extends Feature[Boolean] -case object ConversationRootAuthorFollowsViewer extends Feature[Boolean] -case object ViewerFollowsConversationRootAuthor extends Feature[Boolean] - -case object TweetIsExclusiveTweet extends Feature[Boolean] -case object ViewerIsExclusiveTweetRootAuthor extends Feature[Boolean] -case object ViewerSuperFollowsExclusiveTweetRootAuthor extends Feature[Boolean] - -case object TweetIsCommunityTweet extends Feature[Boolean] - -case object CommunityTweetCommunityNotFound extends Feature[Boolean] - -case object CommunityTweetCommunityDeleted extends Feature[Boolean] - -case object CommunityTweetCommunitySuspended extends Feature[Boolean] - -case object CommunityTweetCommunityVisible extends Feature[Boolean] - -case object CommunityTweetIsHidden extends Feature[Boolean] - -case object ViewerIsInternalCommunitiesAdmin extends Feature[Boolean] - -case object ViewerIsCommunityAdmin extends Feature[Boolean] - -case object ViewerIsCommunityModerator extends Feature[Boolean] - -case object ViewerIsCommunityMember extends Feature[Boolean] - -case object CommunityTweetAuthorIsRemoved extends Feature[Boolean] - -case object NotificationIsOnCommunityTweet extends Feature[Boolean] - -case object NotificationIsOnUnmentionedViewer extends Feature[Boolean] - -case object SearchResultsPageNumber extends Feature[Int] - -case object SearchCandidateCount extends Feature[Int] - -case object SearchQuerySource extends Feature[ThriftQuerySource] - -case object SearchQueryHasUser extends Feature[Boolean] - -case object TweetSemanticCoreAnnotations extends Feature[Seq[SemanticCoreAnnotation]] - -case object OuterAuthorId extends Feature[Long] - -case object AuthorBlocksOuterAuthor extends Feature[Boolean] - -case object OuterAuthorFollowsAuthor extends Feature[Boolean] - -case object OuterAuthorIsInnerAuthor extends Feature[Boolean] - -case object TweetIsModerated extends Feature[Boolean] -case object FocalTweetId extends Feature[Long] - -case object TweetId extends Feature[Long] - -case object TweetConversationId extends Feature[Long] -case object TweetParentId extends Feature[Long] -case object ConversationRootAuthorIsVerified extends Feature[Boolean] - -case object ViewerOptInBlocking extends Feature[Boolean] - -case object ViewerOptInFiltering extends Feature[Boolean] - -case object ViewerRoles extends Feature[Seq[String]] { - val EmployeeRole = "employee" -} - -case object TweetMisinformationPolicies extends Feature[Seq[MisinformationPolicy]] - -case object TweetEnglishMisinformationPolicies extends Feature[Seq[MisinformationPolicy]] - -case object HasInnerCircleOfFriendsRelationship extends Feature[Boolean] - -case object ViewerAge extends Feature[UserAge] - -case object HasDmcaMediaFeature extends Feature[Boolean] - -case object MediaGeoRestrictionsAllowList extends Feature[Seq[String]] -case object MediaGeoRestrictionsDenyList extends Feature[Seq[String]] - -case object TweetIsTrustedFriendTweet extends Feature[Boolean] -case object ViewerIsTrustedFriendTweetAuthor extends Feature[Boolean] -case object ViewerIsTrustedFriendOfTweetAuthor extends Feature[Boolean] - -case object DmConversationIsOneToOneConversation extends Feature[Boolean] -case object DmConversationHasEmptyTimeline extends Feature[Boolean] -case object DmConversationHasValidLastReadableEventId extends Feature[Boolean] -case object DmConversationInfoExists extends Feature[Boolean] -case object DmConversationTimelineExists extends Feature[Boolean] -case object ViewerIsDmConversationParticipant extends Feature[Boolean] - -case object DmEventIsMessageCreateEvent extends Feature[Boolean] -case object DmEventIsWelcomeMessageCreateEvent extends Feature[Boolean] -case object DmEventIsLastMessageReadUpdateEvent extends Feature[Boolean] -case object DmEventIsDeleted extends Feature[Boolean] -case object DmEventIsHidden extends Feature[Boolean] -case object ViewerIsDmEventInitiatingUser extends Feature[Boolean] -case object DmEventInOneToOneConversationWithUnavailableUser extends Feature[Boolean] -case object DmEventIsJoinConversationEvent extends Feature[Boolean] -case object DmEventIsConversationCreateEvent extends Feature[Boolean] -case object DmEventInOneToOneConversation extends Feature[Boolean] -case object DmEventIsTrustConversationEvent extends Feature[Boolean] -case object DmEventIsCsFeedbackSubmitted extends Feature[Boolean] -case object DmEventIsCsFeedbackDismissed extends Feature[Boolean] -case object DmEventIsPerspectivalJoinConversationEvent extends Feature[Boolean] - -case object DmEventOccurredBeforeLastClearedEvent extends Feature[Boolean] -case object DmEventOccurredBeforeJoinConversationEvent extends Feature[Boolean] - -case object CardUriHost extends Feature[String] -case object CardIsPoll extends Feature[Boolean] - -case object TweetIsStaleTweet extends Feature[Boolean] - -case object TweetIsEditTweet extends Feature[Boolean] - -case object TweetIsLatestTweet extends Feature[Boolean] - -case object TweetIsInitialTweet extends Feature[Boolean] - -case object TweetIsCollabInvitationTweet extends Feature[Boolean] - -case object ViewerSensitiveMediaSettings extends Feature[UserSensitiveMediaSettings] - - -case object ToxicReplyFilterState extends Feature[FilterState] - - -case object ToxicReplyFilterConversationAuthorIsViewer extends Feature[Boolean] - -case object RawQuery extends Feature[String] - -case object AuthorScreenName extends Feature[String] - -case object TweetIsInternalPromotedContent extends Feature[Boolean] diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/generators/BUILD deleted file mode 100644 index c53b6b59d..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/generators/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/ibm/icu:icu4j", - "configapi/configapi-core", - "decider/src/main/scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-result-scala", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/client", - "twitter-config/yaml", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/common/src/main/scala/com/twitter/visibility/common/actions", - "visibility/common/src/main/scala/com/twitter/visibility/common/user_result", - "visibility/common/src/main/thrift/com/twitter/visibility:action-scala", - "visibility/lib/src/main/scala/com/twitter/visibility/builder", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/configs", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/models", - "visibility/lib/src/main/scala/com/twitter/visibility/rules", - "visibility/results/src/main/scala/com/twitter/visibility/results/richtext", - "visibility/results/src/main/scala/com/twitter/visibility/results/translation", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/generators/BUILD.docx new file mode 100644 index 000000000..6b7af5411 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/generators/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/CountryNameGenerator.docx b/visibilitylib/src/main/scala/com/twitter/visibility/generators/CountryNameGenerator.docx new file mode 100644 index 000000000..f0a174020 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/generators/CountryNameGenerator.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/CountryNameGenerator.scala b/visibilitylib/src/main/scala/com/twitter/visibility/generators/CountryNameGenerator.scala deleted file mode 100644 index 014533a43..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/generators/CountryNameGenerator.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.visibility.generators - -import com.ibm.icu.util.ULocale -import com.twitter.config.yaml.YamlMap -import com.twitter.finagle.stats.StatsReceiver - -object CountryNameGenerator { - - private val AuroraFilesystemPath = "/usr/local/twitter-config/twitter/config/" - - private val ContentBlockingSupportedCountryList = "takedown_countries.yml" - - def providesFromConfigBus(statsReceiver: StatsReceiver): CountryNameGenerator = { - fromFile(AuroraFilesystemPath + ContentBlockingSupportedCountryList, statsReceiver) - } - - def providesWithCustomMap(countryCodeMap: Map[String, String], statsReceiver: StatsReceiver) = { - new CountryNameGenerator(countryCodeMap, statsReceiver) - } - - private def fromFile(fileName: String, statsReceiver: StatsReceiver) = { - val yamlConfig = YamlMap.load(fileName) - val countryCodeMap: Map[String, String] = yamlConfig.keySet.map { countryCode: String => - val normalizedCode = countryCode.toUpperCase - val countryName: Option[String] = - yamlConfig.get(Seq(countryCode, "name")).asInstanceOf[Option[String]] - (normalizedCode, countryName.getOrElse(normalizedCode)) - }.toMap - new CountryNameGenerator(countryCodeMap, statsReceiver) - } -} - -class CountryNameGenerator(countryCodeMap: Map[String, String], statsReceiver: StatsReceiver) { - - private val scopedStatsReceiver = statsReceiver.scope("country_name_generator") - private val foundCountryReceiver = scopedStatsReceiver.counter("found") - private val missingCountryReceiver = scopedStatsReceiver.counter("missing") - - def getCountryName(code: String): String = { - val normalizedCode = code.toUpperCase - countryCodeMap.get(normalizedCode) match { - case Some(retrievedName) => { - foundCountryReceiver.incr() - retrievedName - } - case _ => { - missingCountryReceiver.incr() - val fallbackName = - new ULocale("", normalizedCode).getDisplayCountry(ULocale.forLanguageTag("en")) - - if (fallbackName == "") - normalizedCode - else - fallbackName - } - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/EpitaphToLocalizedMessage.docx b/visibilitylib/src/main/scala/com/twitter/visibility/generators/EpitaphToLocalizedMessage.docx new file mode 100644 index 000000000..884518fd2 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/generators/EpitaphToLocalizedMessage.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/EpitaphToLocalizedMessage.scala b/visibilitylib/src/main/scala/com/twitter/visibility/generators/EpitaphToLocalizedMessage.scala deleted file mode 100644 index af5266848..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/generators/EpitaphToLocalizedMessage.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.visibility.generators - -import com.twitter.visibility.common.actions.LocalizedMessage -import com.twitter.visibility.common.actions.MessageLink -import com.twitter.visibility.results.translation.Translator -import com.twitter.visibility.results.richtext.EpitaphToRichText -import com.twitter.visibility.results.translation.Resource -import com.twitter.visibility.results.translation.LearnMoreLink -import com.twitter.visibility.rules.Epitaph -import com.twitter.visibility.results.richtext.EpitaphToRichText.Copy - -object EpitaphToLocalizedMessage { - def apply( - epitaph: Epitaph, - languageTag: String, - ): LocalizedMessage = { - val copy = - EpitaphToRichText.EpitaphToPolicyMap.getOrElse(epitaph, EpitaphToRichText.FallbackPolicy) - val text = Translator.translate( - copy.resource, - languageTag - ) - localizeWithCopyAndText(copy, languageTag, text) - } - - def apply( - epitaph: Epitaph, - languageTag: String, - applicableCountries: Seq[String], - ): LocalizedMessage = { - val copy = - EpitaphToRichText.EpitaphToPolicyMap.getOrElse(epitaph, EpitaphToRichText.FallbackPolicy) - val text = Translator.translateWithSimplePlaceholderReplacement( - copy.resource, - languageTag, - Map((Resource.ApplicableCountriesPlaceholder -> applicableCountries.mkString(", "))) - ) - localizeWithCopyAndText(copy, languageTag, text) - } - - private def localizeWithCopyAndText( - copy: Copy, - languageTag: String, - text: String - ): LocalizedMessage = { - val learnMore = Translator.translate(LearnMoreLink, languageTag) - - val links = copy.additionalLinks match { - case links if links.nonEmpty => - MessageLink(Resource.LearnMorePlaceholder, learnMore, copy.link) +: - links.map { - case EpitaphToRichText.Link(placeholder, copyResource, link) => - val copyText = Translator.translate(copyResource, languageTag) - MessageLink(placeholder, copyText, link) - } - case _ => - Seq( - MessageLink( - key = Resource.LearnMorePlaceholder, - displayText = learnMore, - uri = copy.link)) - } - - LocalizedMessage(message = text, language = languageTag, links = links) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/InterstitialReasonToLocalizedMessage.docx b/visibilitylib/src/main/scala/com/twitter/visibility/generators/InterstitialReasonToLocalizedMessage.docx new file mode 100644 index 000000000..3e2e526ef Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/generators/InterstitialReasonToLocalizedMessage.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/InterstitialReasonToLocalizedMessage.scala b/visibilitylib/src/main/scala/com/twitter/visibility/generators/InterstitialReasonToLocalizedMessage.scala deleted file mode 100644 index f4e000338..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/generators/InterstitialReasonToLocalizedMessage.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.visibility.generators - -import com.twitter.visibility.common.actions.InterstitialReason -import com.twitter.visibility.common.actions.LocalizedMessage -import com.twitter.visibility.common.actions.MessageLink -import com.twitter.visibility.results.richtext.InterstitialReasonToRichText -import com.twitter.visibility.results.richtext.InterstitialReasonToRichText.InterstitialCopy -import com.twitter.visibility.results.richtext.InterstitialReasonToRichText.InterstitialLink -import com.twitter.visibility.results.translation.LearnMoreLink -import com.twitter.visibility.results.translation.Resource -import com.twitter.visibility.results.translation.Translator - -object InterstitialReasonToLocalizedMessage { - def apply( - reason: InterstitialReason, - languageTag: String, - ): Option[LocalizedMessage] = { - InterstitialReasonToRichText.reasonToCopy(reason).map { copy => - val text = Translator.translate( - copy.resource, - languageTag - ) - localizeWithCopyAndText(copy, languageTag, text) - } - } - - private def localizeWithCopyAndText( - copy: InterstitialCopy, - languageTag: String, - text: String - ): LocalizedMessage = { - val learnMore = Translator.translate(LearnMoreLink, languageTag) - - val learnMoreLinkOpt = - copy.link.map { link => - MessageLink(key = Resource.LearnMorePlaceholder, displayText = learnMore, uri = link) - } - val additionalLinks = copy.additionalLinks.map { - case InterstitialLink(placeholder, copyResource, link) => - val copyText = Translator.translate(copyResource, languageTag) - MessageLink(key = placeholder, displayText = copyText, uri = link) - } - - val links = learnMoreLinkOpt.toSeq ++ additionalLinks - LocalizedMessage(message = text, language = languageTag, links = links) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/LocalizedInterstitialGenerator.docx b/visibilitylib/src/main/scala/com/twitter/visibility/generators/LocalizedInterstitialGenerator.docx new file mode 100644 index 000000000..6470fa101 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/generators/LocalizedInterstitialGenerator.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/LocalizedInterstitialGenerator.scala b/visibilitylib/src/main/scala/com/twitter/visibility/generators/LocalizedInterstitialGenerator.scala deleted file mode 100644 index 6d381642e..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/generators/LocalizedInterstitialGenerator.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.visibility.generators - -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.common.actions.LocalizedMessage -import com.twitter.visibility.common.actions.MessageLink -import com.twitter.visibility.configapi.configs.VisibilityDeciderGates -import com.twitter.visibility.results.richtext.PublicInterestReasonToRichText -import com.twitter.visibility.results.translation.LearnMoreLink -import com.twitter.visibility.results.translation.Resource -import com.twitter.visibility.results.translation.SafetyResultReasonToResource -import com.twitter.visibility.results.translation.Translator -import com.twitter.visibility.rules.EmergencyDynamicInterstitial -import com.twitter.visibility.rules.Interstitial -import com.twitter.visibility.rules.InterstitialLimitedEngagements -import com.twitter.visibility.rules.PublicInterest -import com.twitter.visibility.rules.Reason -import com.twitter.visibility.rules.TweetInterstitial - -object LocalizedInterstitialGenerator { - def apply( - visibilityDecider: Decider, - baseStatsReceiver: StatsReceiver, - ): LocalizedInterstitialGenerator = { - new LocalizedInterstitialGenerator(visibilityDecider, baseStatsReceiver) - } -} - -class LocalizedInterstitialGenerator private ( - val visibilityDecider: Decider, - val baseStatsReceiver: StatsReceiver) { - - private val visibilityDeciderGates = VisibilityDeciderGates(visibilityDecider) - private val localizationStatsReceiver = baseStatsReceiver.scope("interstitial_localization") - private val publicInterestInterstitialStats = - localizationStatsReceiver.scope("public_interest_copy") - private val emergencyDynamicInterstitialStats = - localizationStatsReceiver.scope("emergency_dynamic_copy") - private val regularInterstitialStats = localizationStatsReceiver.scope("interstitial_copy") - - def apply(visibilityResult: VisibilityResult, languageTag: String): VisibilityResult = { - if (!visibilityDeciderGates.enableLocalizedInterstitialGenerator()) { - return visibilityResult - } - - visibilityResult.verdict match { - case ipi: InterstitialLimitedEngagements if PublicInterest.Reasons.contains(ipi.reason) => - visibilityResult.copy( - verdict = ipi.copy( - localizedMessage = Some(localizePublicInterestCopyInResult(ipi, languageTag)) - )) - case edi: EmergencyDynamicInterstitial => - visibilityResult.copy( - verdict = EmergencyDynamicInterstitial( - edi.copy, - edi.linkOpt, - Some(localizeEmergencyDynamicCopyInResult(edi, languageTag)) - )) - case interstitial: Interstitial => - visibilityResult.copy( - verdict = interstitial.copy( - localizedMessage = localizeInterstitialCopyInResult(interstitial, languageTag) - )) - case tweetInterstitial: TweetInterstitial if tweetInterstitial.interstitial.isDefined => - tweetInterstitial.interstitial.get match { - case ipi: InterstitialLimitedEngagements if PublicInterest.Reasons.contains(ipi.reason) => - visibilityResult.copy( - verdict = tweetInterstitial.copy( - interstitial = Some( - ipi.copy( - localizedMessage = Some(localizePublicInterestCopyInResult(ipi, languageTag)) - )) - )) - case edi: EmergencyDynamicInterstitial => - visibilityResult.copy( - verdict = tweetInterstitial.copy( - interstitial = Some( - EmergencyDynamicInterstitial( - edi.copy, - edi.linkOpt, - Some(localizeEmergencyDynamicCopyInResult(edi, languageTag)) - )) - )) - case interstitial: Interstitial => - visibilityResult.copy( - verdict = tweetInterstitial.copy( - interstitial = Some( - interstitial.copy( - localizedMessage = localizeInterstitialCopyInResult(interstitial, languageTag) - )) - )) - case _ => visibilityResult - } - case _ => visibilityResult - } - } - - private def localizeEmergencyDynamicCopyInResult( - edi: EmergencyDynamicInterstitial, - languageTag: String - ): LocalizedMessage = { - val text = edi.linkOpt - .map(_ => s"${edi.copy} {${Resource.LearnMorePlaceholder}}") - .getOrElse(edi.copy) - - val messageLinks = edi.linkOpt - .map { link => - val learnMoreText = Translator.translate(LearnMoreLink, languageTag) - Seq(MessageLink(Resource.LearnMorePlaceholder, learnMoreText, link)) - }.getOrElse(Seq.empty) - - emergencyDynamicInterstitialStats.counter("localized").incr() - LocalizedMessage(text, languageTag, messageLinks) - } - - private def localizePublicInterestCopyInResult( - ipi: InterstitialLimitedEngagements, - languageTag: String - ): LocalizedMessage = { - val safetyResultReason = PublicInterest.ReasonToSafetyResultReason(ipi.reason) - val text = Translator.translate( - SafetyResultReasonToResource.resource(safetyResultReason), - languageTag, - ) - - val learnMoreLink = PublicInterestReasonToRichText.toLearnMoreLink(safetyResultReason) - val learnMoreText = Translator.translate(LearnMoreLink, languageTag) - val messageLinks = Seq(MessageLink(Resource.LearnMorePlaceholder, learnMoreText, learnMoreLink)) - - publicInterestInterstitialStats.counter("localized").incr() - LocalizedMessage(text, languageTag, messageLinks) - } - - private def localizeInterstitialCopyInResult( - interstitial: Interstitial, - languageTag: String - ): Option[LocalizedMessage] = { - val localizedMessageOpt = Reason - .toInterstitialReason(interstitial.reason) - .flatMap(InterstitialReasonToLocalizedMessage(_, languageTag)) - - if (localizedMessageOpt.isDefined) { - regularInterstitialStats.counter("localized").incr() - localizedMessageOpt - } else { - regularInterstitialStats.counter("empty").incr() - None - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/TombstoneGenerator.docx b/visibilitylib/src/main/scala/com/twitter/visibility/generators/TombstoneGenerator.docx new file mode 100644 index 000000000..9f86cba03 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/generators/TombstoneGenerator.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/generators/TombstoneGenerator.scala b/visibilitylib/src/main/scala/com/twitter/visibility/generators/TombstoneGenerator.scala deleted file mode 100644 index 9d52cc217..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/generators/TombstoneGenerator.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.visibility.generators - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.common.actions.TombstoneReason -import com.twitter.visibility.configapi.VisibilityParams -import com.twitter.visibility.rules.Epitaph -import com.twitter.visibility.rules.LocalizedTombstone -import com.twitter.visibility.rules.Tombstone - -object TombstoneGenerator { - def apply( - visibilityParams: VisibilityParams, - countryNameGenerator: CountryNameGenerator, - statsReceiver: StatsReceiver - ): TombstoneGenerator = { - new TombstoneGenerator(visibilityParams, countryNameGenerator, statsReceiver) - } -} - -class TombstoneGenerator( - paramsFactory: VisibilityParams, - countryNameGenerator: CountryNameGenerator, - baseStatsReceiver: StatsReceiver) { - - private[this] val statsReceiver = new MemoizingStatsReceiver( - baseStatsReceiver.scope("tombstone_generator")) - private[this] val deletedReceiver = statsReceiver.scope("deleted_state") - private[this] val authorStateReceiver = statsReceiver.scope("tweet_author_state") - private[this] val visResultReceiver = statsReceiver.scope("visibility_result") - - def apply( - result: VisibilityResult, - language: String - ): VisibilityResult = { - - result.verdict match { - case tombstone: Tombstone => - val epitaph = tombstone.epitaph - visResultReceiver.scope("tombstone").counter(epitaph.name.toLowerCase()) - - val overriddenLanguage = epitaph match { - case Epitaph.LegalDemandsWithheldMedia | Epitaph.LocalLawsWithheldMedia => "en" - case _ => language - } - - tombstone.applicableCountryCodes match { - case Some(countryCodes) => { - val countryNames = countryCodes.map(countryNameGenerator.getCountryName(_)) - - result.copy(verdict = LocalizedTombstone( - reason = epitaphToTombstoneReason(epitaph), - message = EpitaphToLocalizedMessage(epitaph, overriddenLanguage, countryNames))) - } - case _ => { - result.copy(verdict = LocalizedTombstone( - reason = epitaphToTombstoneReason(epitaph), - message = EpitaphToLocalizedMessage(epitaph, overriddenLanguage))) - } - } - case _ => - result - } - } - - private def epitaphToTombstoneReason(epitaph: Epitaph): TombstoneReason = { - epitaph match { - case Epitaph.Deleted => TombstoneReason.Deleted - case Epitaph.Bounced => TombstoneReason.Bounced - case Epitaph.BounceDeleted => TombstoneReason.BounceDeleted - case Epitaph.Protected => TombstoneReason.ProtectedAuthor - case Epitaph.Suspended => TombstoneReason.SuspendedAuthor - case Epitaph.BlockedBy => TombstoneReason.AuthorBlocksViewer - case Epitaph.SuperFollowsContent => TombstoneReason.ExclusiveTweet - case Epitaph.Underage => TombstoneReason.NsfwViewerIsUnderage - case Epitaph.NoStatedAge => TombstoneReason.NsfwViewerHasNoStatedAge - case Epitaph.LoggedOutAge => TombstoneReason.NsfwLoggedOut - case Epitaph.Deactivated => TombstoneReason.DeactivatedAuthor - case Epitaph.CommunityTweetHidden => TombstoneReason.CommunityTweetHidden - case Epitaph.CommunityTweetCommunityIsSuspended => - TombstoneReason.CommunityTweetCommunityIsSuspended - case Epitaph.DevelopmentOnly => TombstoneReason.DevelopmentOnly - case Epitaph.AdultMedia => TombstoneReason.AdultMedia - case Epitaph.ViolentMedia => TombstoneReason.ViolentMedia - case Epitaph.OtherSensitiveMedia => TombstoneReason.OtherSensitiveMedia - case Epitaph.DmcaWithheldMedia => TombstoneReason.DmcaWithheldMedia - case Epitaph.LegalDemandsWithheldMedia => TombstoneReason.LegalDemandsWithheldMedia - case Epitaph.LocalLawsWithheldMedia => TombstoneReason.LocalLawsWithheldMedia - case Epitaph.ToxicReplyFiltered => TombstoneReason.ReplyFiltered - case _ => TombstoneReason.Unspecified - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BUILD deleted file mode 100644 index 545aee9e5..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/src/java/com/twitter/logpipeline/client:logpipeline-event-publisher-thin", - "decider/src/main/scala", - "mediaservices/media-util/src/main/scala", - "servo/decider/src/main/scala", - "src/thrift/com/twitter/escherbird:media-annotation-structs-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/client", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/media", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/tweets", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/configs", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/blender", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/tweets", - "visibility/lib/src/main/thrift/com/twitter/visibility/logging:vf-logging-scala", - ], - exports = [ - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/lib/src/main/scala/com/twitter/visibility", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BUILD.docx new file mode 100644 index 000000000..36788f1ac Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityLibrary.docx new file mode 100644 index 000000000..a65f6557b Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityLibrary.scala deleted file mode 100644 index c83756818..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityLibrary.scala +++ /dev/null @@ -1,416 +0,0 @@ -package com.twitter.visibility.interfaces.blender - -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.mediaservices.media_util.GenericMediaKey -import com.twitter.servo.util.Gate -import com.twitter.stitch.Stitch -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.util.Stopwatch -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.VerdictLogger -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.media.MediaFeatures -import com.twitter.visibility.builder.media.StratoMediaLabelMaps -import com.twitter.visibility.builder.tweets._ -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.builder.users.RelationshipFeatures -import com.twitter.visibility.builder.users.ViewerFeatures -import com.twitter.visibility.common.MediaSafetyLabelMapSource -import com.twitter.visibility.common.MisinformationPolicySource -import com.twitter.visibility.common.SafetyLabelMapSource -import com.twitter.visibility.common.TrustedFriendsSource -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.rules.ComposableActions.ComposableActionsWithInterstitial -import com.twitter.visibility.configapi.configs.VisibilityDeciderGates -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.features.TweetIsInnerQuotedTweet -import com.twitter.visibility.features.TweetIsRetweet -import com.twitter.visibility.features.TweetIsSourceTweet -import com.twitter.visibility.logging.thriftscala.VFLibType -import com.twitter.visibility.models.ContentId -import com.twitter.visibility.models.ContentId.BlenderTweetId -import com.twitter.visibility.models.ContentId.TweetId -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.SafetyLevel.toThrift -import com.twitter.visibility.rules.Action -import com.twitter.visibility.rules.Allow -import com.twitter.visibility.rules.Drop -import com.twitter.visibility.rules.Interstitial -import com.twitter.visibility.rules.TweetInterstitial - -object TweetType extends Enumeration { - type TweetType = Value - val ORIGINAL, SOURCE, QUOTED = Value -} -import com.twitter.visibility.interfaces.blender.TweetType._ - -object BlenderVisibilityLibrary { - def buildWithStratoClient( - visibilityLibrary: VisibilityLibrary, - decider: Decider, - stratoClient: StratoClient, - userSource: UserSource, - userRelationshipSource: UserRelationshipSource - ): BlenderVisibilityLibrary = new BlenderVisibilityLibrary( - visibilityLibrary, - decider, - stratoClient, - userSource, - userRelationshipSource, - None - ) - - def buildWithSafetyLabelMapSource( - visibilityLibrary: VisibilityLibrary, - decider: Decider, - stratoClient: StratoClient, - userSource: UserSource, - userRelationshipSource: UserRelationshipSource, - safetyLabelMapSource: SafetyLabelMapSource - ): BlenderVisibilityLibrary = new BlenderVisibilityLibrary( - visibilityLibrary, - decider, - stratoClient, - userSource, - userRelationshipSource, - Some(safetyLabelMapSource) - ) - - def createVerdictLogger( - enableVerdictLogger: Gate[Unit], - decider: Decider, - statsReceiver: StatsReceiver - ): VerdictLogger = { - if (enableVerdictLogger()) { - VerdictLogger(statsReceiver, decider) - } else { - VerdictLogger.Empty - } - } - - def scribeVisibilityVerdict( - result: CombinedVisibilityResult, - enableVerdictScribing: Gate[Unit], - verdictLogger: VerdictLogger, - viewerId: Option[Long], - safetyLevel: SafetyLevel - ): Unit = if (enableVerdictScribing()) { - verdictLogger.scribeVerdict( - visibilityResult = result.tweetVisibilityResult, - viewerId = viewerId, - safetyLevel = toThrift(safetyLevel), - vfLibType = VFLibType.BlenderVisibilityLibrary) - - result.quotedTweetVisibilityResult.map(quotedTweetVisibilityResult => - verdictLogger.scribeVerdict( - visibilityResult = quotedTweetVisibilityResult, - viewerId = viewerId, - safetyLevel = toThrift(safetyLevel), - vfLibType = VFLibType.BlenderVisibilityLibrary)) - } -} - -class BlenderVisibilityLibrary( - visibilityLibrary: VisibilityLibrary, - decider: Decider, - stratoClient: StratoClient, - userSource: UserSource, - userRelationshipSource: UserRelationshipSource, - safetyLabelMapSourceOption: Option[SafetyLabelMapSource]) { - - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val stratoClientStatsReceiver = visibilityLibrary.statsReceiver.scope("strato") - val vfEngineCounter = libraryStatsReceiver.counter("vf_engine_requests") - val bvlRequestCounter = libraryStatsReceiver.counter("bvl_requests") - val vfLatencyOverallStat = libraryStatsReceiver.stat("vf_latency_overall") - val vfLatencyStitchBuildStat = libraryStatsReceiver.stat("vf_latency_stitch_build") - val vfLatencyStitchRunStat = libraryStatsReceiver.stat("vf_latency_stitch_run") - val visibilityDeciderGates = VisibilityDeciderGates(decider) - val verdictLogger = BlenderVisibilityLibrary.createVerdictLogger( - visibilityDeciderGates.enableVerdictLoggerBVL, - decider, - libraryStatsReceiver) - - val tweetLabels = safetyLabelMapSourceOption match { - case Some(safetyLabelMapSource) => - new StratoTweetLabelMaps(safetyLabelMapSource) - case None => - new StratoTweetLabelMaps( - SafetyLabelMapSource.fromStrato(stratoClient, stratoClientStatsReceiver)) - } - - val mediaLabelMaps = new StratoMediaLabelMaps( - MediaSafetyLabelMapSource.fromStrato(stratoClient, stratoClientStatsReceiver)) - - val tweetFeatures = new TweetFeatures(tweetLabels, libraryStatsReceiver) - val blenderContextFeatures = new BlenderContextFeatures(libraryStatsReceiver) - val authorFeatures = new AuthorFeatures(userSource, libraryStatsReceiver) - val viewerFeatures = new ViewerFeatures(userSource, libraryStatsReceiver) - val relationshipFeatures = - new RelationshipFeatures(userRelationshipSource, libraryStatsReceiver) - val fonsrRelationshipFeatures = - new FosnrRelationshipFeatures( - tweetLabels = tweetLabels, - userRelationshipSource = userRelationshipSource, - statsReceiver = libraryStatsReceiver) - val misinfoPolicySource = - MisinformationPolicySource.fromStrato(stratoClient, stratoClientStatsReceiver) - val misinfoPolicyFeatures = - new MisinformationPolicyFeatures(misinfoPolicySource, stratoClientStatsReceiver) - val exclusiveTweetFeatures = - new ExclusiveTweetFeatures(userRelationshipSource, libraryStatsReceiver) - val mediaFeatures = new MediaFeatures(mediaLabelMaps, libraryStatsReceiver) - val trustedFriendsTweetFeatures = new TrustedFriendsFeatures( - trustedFriendsSource = TrustedFriendsSource.fromStrato(stratoClient, stratoClientStatsReceiver)) - val editTweetFeatures = new EditTweetFeatures(libraryStatsReceiver) - - def getCombinedVisibilityResult( - bvRequest: BlenderVisibilityRequest - ): Stitch[CombinedVisibilityResult] = { - val elapsed = Stopwatch.start() - bvlRequestCounter.incr() - - val ( - requestTweetVisibilityResult, - quotedTweetVisibilityResultOption, - sourceTweetVisibilityResultOption - ) = getAllVisibilityResults(bvRequest: BlenderVisibilityRequest) - - val response: Stitch[CombinedVisibilityResult] = { - ( - requestTweetVisibilityResult, - quotedTweetVisibilityResultOption, - sourceTweetVisibilityResultOption) match { - case (requestTweetVisResult, Some(quotedTweetVisResult), Some(sourceTweetVisResult)) => { - Stitch - .join( - requestTweetVisResult, - quotedTweetVisResult, - sourceTweetVisResult - ).map { - case (requestTweetVisResult, quotedTweetVisResult, sourceTweetVisResult) => { - requestTweetVisResult.verdict match { - case Allow => - CombinedVisibilityResult(sourceTweetVisResult, Some(quotedTweetVisResult)) - case _ => - CombinedVisibilityResult(requestTweetVisResult, Some(quotedTweetVisResult)) - } - } - } - } - - case (requestTweetVisResult, None, Some(sourceTweetVisResult)) => { - Stitch - .join( - requestTweetVisResult, - sourceTweetVisResult - ).map { - case (requestTweetVisResult, sourceTweetVisResult) => { - requestTweetVisResult.verdict match { - case Allow => - CombinedVisibilityResult(sourceTweetVisResult, None) - case _ => - CombinedVisibilityResult(requestTweetVisResult, None) - } - } - } - } - - case (requestTweetVisResult, Some(quotedTweetVisResult), None) => { - Stitch - .join( - requestTweetVisResult, - quotedTweetVisResult - ).map { - case (requestTweetVisResult, quotedTweetVisResult) => { - CombinedVisibilityResult(requestTweetVisResult, Some(quotedTweetVisResult)) - } - } - } - - case (requestTweetVisResult, None, None) => { - requestTweetVisResult.map { - CombinedVisibilityResult(_, None) - } - } - } - } - val runStitchStartMs = elapsed().inMilliseconds - val buildStitchStatMs = elapsed().inMilliseconds - vfLatencyStitchBuildStat.add(buildStitchStatMs) - - response - .onSuccess(_ => { - val overallMs = elapsed().inMilliseconds - vfLatencyOverallStat.add(overallMs) - val stitchRunMs = elapsed().inMilliseconds - runStitchStartMs - vfLatencyStitchRunStat.add(stitchRunMs) - }) - .onSuccess( - BlenderVisibilityLibrary.scribeVisibilityVerdict( - _, - visibilityDeciderGates.enableVerdictScribingBVL, - verdictLogger, - bvRequest.viewerContext.userId, - bvRequest.safetyLevel)) - } - - def getContentId(viewerId: Option[Long], authorId: Long, tweet: Tweet): ContentId = { - if (viewerId.contains(authorId)) - TweetId(tweet.id) - else BlenderTweetId(tweet.id) - } - - def getAllVisibilityResults(bvRequest: BlenderVisibilityRequest): ( - Stitch[VisibilityResult], - Option[Stitch[VisibilityResult]], - Option[Stitch[VisibilityResult]] - ) = { - val tweetContentId = getContentId( - viewerId = bvRequest.viewerContext.userId, - authorId = bvRequest.tweet.coreData.get.userId, - tweet = bvRequest.tweet) - - val tweetFeatureMap = - buildFeatureMap(bvRequest, bvRequest.tweet, ORIGINAL) - vfEngineCounter.incr() - val requestTweetVisibilityResult = visibilityLibrary - .runRuleEngine( - tweetContentId, - tweetFeatureMap, - bvRequest.viewerContext, - bvRequest.safetyLevel - ).map(handleComposableVisibilityResult) - - val quotedTweetVisibilityResultOption = bvRequest.quotedTweet.map(quotedTweet => { - val quotedTweetContentId = getContentId( - viewerId = bvRequest.viewerContext.userId, - authorId = quotedTweet.coreData.get.userId, - tweet = quotedTweet) - - val quotedInnerTweetFeatureMap = - buildFeatureMap(bvRequest, quotedTweet, QUOTED) - vfEngineCounter.incr() - visibilityLibrary - .runRuleEngine( - quotedTweetContentId, - quotedInnerTweetFeatureMap, - bvRequest.viewerContext, - bvRequest.safetyLevel - ) - .map(handleComposableVisibilityResult) - .map(handleInnerQuotedTweetVisibilityResult) - }) - - val sourceTweetVisibilityResultOption = bvRequest.retweetSourceTweet.map(sourceTweet => { - val sourceTweetContentId = getContentId( - viewerId = bvRequest.viewerContext.userId, - authorId = sourceTweet.coreData.get.userId, - tweet = sourceTweet) - - val sourceTweetFeatureMap = - buildFeatureMap(bvRequest, sourceTweet, SOURCE) - vfEngineCounter.incr() - visibilityLibrary - .runRuleEngine( - sourceTweetContentId, - sourceTweetFeatureMap, - bvRequest.viewerContext, - bvRequest.safetyLevel - ) - .map(handleComposableVisibilityResult) - }) - - ( - requestTweetVisibilityResult, - quotedTweetVisibilityResultOption, - sourceTweetVisibilityResultOption) - } - - def buildFeatureMap( - bvRequest: BlenderVisibilityRequest, - tweet: Tweet, - tweetType: TweetType - ): FeatureMap = { - val authorId = tweet.coreData.get.userId - val viewerId = bvRequest.viewerContext.userId - val isRetweet = if (tweetType.equals(ORIGINAL)) bvRequest.isRetweet else false - val isSourceTweet = tweetType.equals(SOURCE) - val isQuotedTweet = tweetType.equals(QUOTED) - val tweetMediaKeys: Seq[GenericMediaKey] = tweet.media - .getOrElse(Seq.empty) - .flatMap(_.mediaKey.map(GenericMediaKey.apply)) - - visibilityLibrary.featureMapBuilder( - Seq( - viewerFeatures - .forViewerBlenderContext(bvRequest.blenderVFRequestContext, bvRequest.viewerContext), - relationshipFeatures.forAuthorId(authorId, viewerId), - fonsrRelationshipFeatures - .forTweetAndAuthorId(tweet = tweet, authorId = authorId, viewerId = viewerId), - tweetFeatures.forTweet(tweet), - mediaFeatures.forMediaKeys(tweetMediaKeys), - authorFeatures.forAuthorId(authorId), - blenderContextFeatures.forBlenderContext(bvRequest.blenderVFRequestContext), - _.withConstantFeature(TweetIsRetweet, isRetweet), - misinfoPolicyFeatures.forTweet(tweet, bvRequest.viewerContext), - exclusiveTweetFeatures.forTweet(tweet, bvRequest.viewerContext), - trustedFriendsTweetFeatures.forTweet(tweet, viewerId), - editTweetFeatures.forTweet(tweet), - _.withConstantFeature(TweetIsInnerQuotedTweet, isQuotedTweet), - _.withConstantFeature(TweetIsSourceTweet, isSourceTweet), - ) - ) - } - - def handleComposableVisibilityResult(result: VisibilityResult): VisibilityResult = { - if (result.secondaryVerdicts.nonEmpty) { - result.copy(verdict = composeActions(result.verdict, result.secondaryVerdicts)) - } else { - result - } - } - - private def composeActions(primary: Action, secondary: Seq[Action]): Action = { - if (primary.isComposable && secondary.nonEmpty) { - val actions = Seq[Action] { primary } ++ secondary - val interstitialOpt = Action.getFirstInterstitial(actions: _*) - val softInterventionOpt = Action.getFirstSoftIntervention(actions: _*) - val limitedEngagementsOpt = Action.getFirstLimitedEngagements(actions: _*) - val avoidOpt = Action.getFirstAvoid(actions: _*) - - val numActions = - Seq[Option[_]](interstitialOpt, softInterventionOpt, limitedEngagementsOpt, avoidOpt) - .count(_.isDefined) - if (numActions > 1) { - TweetInterstitial( - interstitialOpt, - softInterventionOpt, - limitedEngagementsOpt, - None, - avoidOpt - ) - } else { - primary - } - } else { - primary - } - } - - def handleInnerQuotedTweetVisibilityResult( - result: VisibilityResult - ): VisibilityResult = { - val newVerdict: Action = - result.verdict match { - case interstitial: Interstitial => Drop(interstitial.reason) - case ComposableActionsWithInterstitial(tweetInterstitial) => Drop(tweetInterstitial.reason) - case verdict => verdict - } - - result.copy(verdict = newVerdict) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityRequest.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityRequest.docx new file mode 100644 index 000000000..1ea000bea Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityRequest.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityRequest.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityRequest.scala deleted file mode 100644 index aa6e604a5..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/BlenderVisibilityRequest.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.visibility.interfaces.blender - -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext -import com.twitter.visibility.interfaces.common.blender.BlenderVFRequestContext - -case class BlenderVisibilityRequest( - tweet: Tweet, - quotedTweet: Option[Tweet], - retweetSourceTweet: Option[Tweet] = None, - isRetweet: Boolean, - safetyLevel: SafetyLevel, - viewerContext: ViewerContext, - blenderVFRequestContext: BlenderVFRequestContext) { - - def getTweetID: Long = tweet.id - - def hasQuotedTweet: Boolean = { - quotedTweet.nonEmpty - } - def hasSourceTweet: Boolean = { - retweetSourceTweet.nonEmpty - } - - def getQuotedTweetId: Long = { - quotedTweet match { - case Some(qTweet) => - qTweet.id - case None => - -1 - } - } - def getSourceTweetId: Long = { - retweetSourceTweet match { - case Some(sourceTweet) => - sourceTweet.id - case None => - -1 - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/CombinedVisibilityResult.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/CombinedVisibilityResult.docx new file mode 100644 index 000000000..cbd1aec46 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/CombinedVisibilityResult.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/CombinedVisibilityResult.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/CombinedVisibilityResult.scala deleted file mode 100644 index 6868b67ec..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/blender/CombinedVisibilityResult.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.visibility.interfaces.blender - -import com.twitter.visibility.builder.VisibilityResult - -case class CombinedVisibilityResult( - tweetVisibilityResult: VisibilityResult, - quotedTweetVisibilityResult: Option[VisibilityResult]) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/BUILD deleted file mode 100644 index b2b0f998f..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "appsec/sanitization-lib/src/main/scala", - "src/thrift/com/twitter/expandodo:cards-scala", - "stitch/stitch-core", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/tweets", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/BUILD.docx new file mode 100644 index 000000000..8226d4168 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibrary.docx new file mode 100644 index 000000000..c61a3b42b Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibrary.scala deleted file mode 100644 index 575356901..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibrary.scala +++ /dev/null @@ -1,187 +0,0 @@ -package com.twitter.visibility.interfaces.cards - -import com.twitter.appsec.sanitization.URLSafety -import com.twitter.decider.Decider -import com.twitter.servo.util.Gate -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.{thriftscala => tweetypiethrift} -import com.twitter.util.Stopwatch -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.tweets.CommunityTweetFeatures -import com.twitter.visibility.builder.tweets.CommunityTweetFeaturesV2 -import com.twitter.visibility.builder.tweets.NilTweetLabelMaps -import com.twitter.visibility.builder.tweets.TweetFeatures -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.builder.users.RelationshipFeatures -import com.twitter.visibility.builder.users.ViewerFeatures -import com.twitter.visibility.common.CommunitiesSource -import com.twitter.visibility.common.UserId -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.configapi.configs.VisibilityDeciderGates -import com.twitter.visibility.features.CardIsPoll -import com.twitter.visibility.features.CardUriHost -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.models.ContentId.CardId -import com.twitter.visibility.models.ViewerContext - -object CardVisibilityLibrary { - type Type = CardVisibilityRequest => Stitch[VisibilityResult] - - private[this] def getAuthorFeatures( - authorIdOpt: Option[Long], - authorFeatures: AuthorFeatures - ): FeatureMapBuilder => FeatureMapBuilder = { - authorIdOpt match { - case Some(authorId) => authorFeatures.forAuthorId(authorId) - case _ => authorFeatures.forNoAuthor() - } - } - - private[this] def getCardUriFeatures( - cardUri: String, - enableCardVisibilityLibraryCardUriParsing: Boolean, - trackCardUriHost: Option[String] => Unit - ): FeatureMapBuilder => FeatureMapBuilder = { - if (enableCardVisibilityLibraryCardUriParsing) { - val safeCardUriHost = URLSafety.getHostSafe(cardUri) - trackCardUriHost(safeCardUriHost) - - _.withConstantFeature(CardUriHost, safeCardUriHost) - } else { - identity - } - } - - private[this] def getRelationshipFeatures( - authorIdOpt: Option[Long], - viewerIdOpt: Option[Long], - relationshipFeatures: RelationshipFeatures - ): FeatureMapBuilder => FeatureMapBuilder = { - authorIdOpt match { - case Some(authorId) => relationshipFeatures.forAuthorId(authorId, viewerIdOpt) - case _ => relationshipFeatures.forNoAuthor() - } - } - - private[this] def getTweetFeatures( - tweetOpt: Option[tweetypiethrift.Tweet], - tweetFeatures: TweetFeatures - ): FeatureMapBuilder => FeatureMapBuilder = { - tweetOpt match { - case Some(tweet) => tweetFeatures.forTweet(tweet) - case _ => identity - } - } - - private[this] def getCommunityFeatures( - tweetOpt: Option[tweetypiethrift.Tweet], - viewerContext: ViewerContext, - communityTweetFeatures: CommunityTweetFeatures - ): FeatureMapBuilder => FeatureMapBuilder = { - tweetOpt match { - case Some(tweet) => communityTweetFeatures.forTweet(tweet, viewerContext) - case _ => identity - } - } - - def apply( - visibilityLibrary: VisibilityLibrary, - userSource: UserSource = UserSource.empty, - userRelationshipSource: UserRelationshipSource = UserRelationshipSource.empty, - communitiesSource: CommunitiesSource = CommunitiesSource.empty, - enableVfParityTest: Gate[Unit] = Gate.False, - enableVfFeatureHydration: Gate[Unit] = Gate.False, - decider: Decider - ): Type = { - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val vfLatencyOverallStat = libraryStatsReceiver.stat("vf_latency_overall") - val vfLatencyStitchBuildStat = libraryStatsReceiver.stat("vf_latency_stitch_build") - val vfLatencyStitchRunStat = libraryStatsReceiver.stat("vf_latency_stitch_run") - val cardUriStats = libraryStatsReceiver.scope("card_uri") - val visibilityDeciderGates = VisibilityDeciderGates(decider) - - val authorFeatures = new AuthorFeatures(userSource, libraryStatsReceiver) - val viewerFeatures = new ViewerFeatures(userSource, libraryStatsReceiver) - val tweetFeatures = new TweetFeatures(NilTweetLabelMaps, libraryStatsReceiver) - val communityTweetFeatures = new CommunityTweetFeaturesV2( - communitiesSource = communitiesSource, - ) - val relationshipFeatures = - new RelationshipFeatures(userRelationshipSource, libraryStatsReceiver) - val parityTest = new CardVisibilityLibraryParityTest(libraryStatsReceiver) - - { r: CardVisibilityRequest => - val elapsed = Stopwatch.start() - var runStitchStartMs = 0L - - val viewerId: Option[UserId] = r.viewerContext.userId - - val featureMap = - visibilityLibrary - .featureMapBuilder( - Seq( - viewerFeatures.forViewerId(viewerId), - getAuthorFeatures(r.authorId, authorFeatures), - getCardUriFeatures( - cardUri = r.cardUri, - enableCardVisibilityLibraryCardUriParsing = - visibilityDeciderGates.enableCardVisibilityLibraryCardUriParsing(), - trackCardUriHost = { safeCardUriHost: Option[String] => - if (safeCardUriHost.isEmpty) { - cardUriStats.counter("empty").incr() - } - } - ), - getCommunityFeatures(r.tweetOpt, r.viewerContext, communityTweetFeatures), - getRelationshipFeatures(r.authorId, r.viewerContext.userId, relationshipFeatures), - getTweetFeatures(r.tweetOpt, tweetFeatures), - _.withConstantFeature(CardIsPoll, r.isPollCardType) - ) - ) - - val response = visibilityLibrary - .runRuleEngine( - CardId(r.cardUri), - featureMap, - r.viewerContext, - r.safetyLevel - ) - .onSuccess(_ => { - val overallStatMs = elapsed().inMilliseconds - vfLatencyOverallStat.add(overallStatMs) - val runStitchEndMs = elapsed().inMilliseconds - vfLatencyStitchRunStat.add(runStitchEndMs - runStitchStartMs) - }) - - runStitchStartMs = elapsed().inMilliseconds - val buildStitchStatMs = elapsed().inMilliseconds - vfLatencyStitchBuildStat.add(buildStitchStatMs) - - lazy val hydratedFeatureResponse: Stitch[VisibilityResult] = - FeatureMap.resolve(featureMap, libraryStatsReceiver).flatMap { resolvedFeatureMap => - visibilityLibrary.runRuleEngine( - CardId(r.cardUri), - resolvedFeatureMap, - r.viewerContext, - r.safetyLevel - ) - } - - val isVfParityTestEnabled = enableVfParityTest() - val isVfFeatureHydrationEnabled = enableVfFeatureHydration() - - if (!isVfParityTestEnabled && !isVfFeatureHydrationEnabled) { - response - } else if (isVfParityTestEnabled && !isVfFeatureHydrationEnabled) { - response.applyEffect { resp => - Stitch.async(parityTest.runParityTest(hydratedFeatureResponse, resp)) - } - } else { - hydratedFeatureResponse - } - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibraryParityTest.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibraryParityTest.docx new file mode 100644 index 000000000..92155f8bd Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibraryParityTest.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibraryParityTest.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibraryParityTest.scala deleted file mode 100644 index 4dc3f6baf..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityLibraryParityTest.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.visibility.interfaces.cards - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.visibility.builder.VisibilityResult - -class CardVisibilityLibraryParityTest(statsReceiver: StatsReceiver) { - private val parityTestScope = statsReceiver.scope("card_visibility_library_parity") - private val requests = parityTestScope.counter("requests") - private val equal = parityTestScope.counter("equal") - private val incorrect = parityTestScope.counter("incorrect") - private val failures = parityTestScope.counter("failures") - - def runParityTest( - preHydratedFeatureVisibilityResult: Stitch[VisibilityResult], - resp: VisibilityResult - ): Stitch[Unit] = { - requests.incr() - - preHydratedFeatureVisibilityResult - .flatMap { parityResponse => - if (parityResponse.verdict == resp.verdict) { - equal.incr() - } else { - incorrect.incr() - } - - Stitch.Done - }.rescue { - case t: Throwable => - failures.incr() - Stitch.Done - }.unit - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityRequest.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityRequest.docx new file mode 100644 index 000000000..3e33119ed Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityRequest.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityRequest.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityRequest.scala deleted file mode 100644 index c3cef91d2..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/cards/CardVisibilityRequest.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.visibility.interfaces.cards - -import com.twitter.tweetypie.{thriftscala => tweetypiethrift} -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext - -case class CardVisibilityRequest( - cardUri: String, - authorId: Option[Long], - tweetOpt: Option[tweetypiethrift.Tweet], - isPollCardType: Boolean, - safetyLevel: SafetyLevel, - viewerContext: ViewerContext) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/BUILD.bazel b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/BUILD.bazel deleted file mode 100644 index e8d9d1b25..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "src/scala/com/twitter/search/blender/services/strato", - "src/thrift/com/twitter/spam/rtf:safety-label-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/catalog", - "strato/src/main/scala/com/twitter/strato/client", - "strato/src/main/scala/com/twitter/strato/data", - "strato/src/main/scala/com/twitter/strato/thrift", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/BUILD.docx new file mode 100644 index 000000000..d3bce17a1 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BUILD deleted file mode 100644 index c0f745c39..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/scala/com/twitter/search/blender/services/strato", - "src/thrift/com/twitter/search/common:constants-scala", - "stitch/stitch-core", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BUILD.docx new file mode 100644 index 000000000..e787e50d5 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BlenderVFRequestContext.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BlenderVFRequestContext.docx new file mode 100644 index 000000000..a70e86704 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BlenderVFRequestContext.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BlenderVFRequestContext.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BlenderVFRequestContext.scala deleted file mode 100644 index 05e98c17d..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/blender/BlenderVFRequestContext.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.visibility.interfaces.common.blender - -import com.twitter.search.blender.services.strato.UserSearchSafetySettings -import com.twitter.search.common.constants.thriftscala.ThriftQuerySource - -case class BlenderVFRequestContext( - resultsPageNumber: Int, - candidateCount: Int, - querySourceOption: Option[ThriftQuerySource], - userSearchSafetySettings: UserSearchSafetySettings, - queryHasUser: Boolean = false) { - - def this( - resultsPageNumber: Int, - candidateCount: Int, - querySourceOption: Option[ThriftQuerySource], - userSearchSafetySettings: UserSearchSafetySettings - ) = this(resultsPageNumber, candidateCount, querySourceOption, userSearchSafetySettings, false) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/BUILD deleted file mode 100644 index c0f745c39..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/scala/com/twitter/search/blender/services/strato", - "src/thrift/com/twitter/search/common:constants-scala", - "stitch/stitch-core", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/BUILD.docx new file mode 100644 index 000000000..e787e50d5 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/SearchVFRequestContext.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/SearchVFRequestContext.docx new file mode 100644 index 000000000..22b21ddd3 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/SearchVFRequestContext.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/SearchVFRequestContext.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/SearchVFRequestContext.scala deleted file mode 100644 index ef06b0b3b..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/search/SearchVFRequestContext.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.visibility.interfaces.common.search - -import com.twitter.search.blender.services.strato.UserSearchSafetySettings -import com.twitter.search.common.constants.thriftscala.ThriftQuerySource - -case class SearchVFRequestContext( - resultsPageNumber: Int, - candidateCount: Int, - querySourceOption: Option[ThriftQuerySource], - userSearchSafetySettings: UserSearchSafetySettings, - queryHasUser: Boolean = false) { - - def this( - resultsPageNumber: Int, - candidateCount: Int, - querySourceOption: Option[ThriftQuerySource], - userSearchSafetySettings: UserSearchSafetySettings - ) = this(resultsPageNumber, candidateCount, querySourceOption, userSearchSafetySettings, false) -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/BUILD deleted file mode 100644 index 64400fae2..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/scala/com/twitter/search/blender/services/strato", - "src/thrift/com/twitter/spam/rtf:safety-label-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/catalog", - "strato/src/main/scala/com/twitter/strato/client", - "strato/src/main/scala/com/twitter/strato/data", - "strato/src/main/scala/com/twitter/strato/thrift", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/BUILD.docx new file mode 100644 index 000000000..5203df9f0 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelFetcher.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelFetcher.docx new file mode 100644 index 000000000..65de5334e Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelFetcher.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelFetcher.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelFetcher.scala deleted file mode 100644 index 324620892..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelFetcher.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.visibility.interfaces.common.tweets - -import com.twitter.spam.rtf.thriftscala.SafetyLabel -import com.twitter.spam.rtf.thriftscala.SafetyLabelType -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.strato.thrift.ScroogeConvImplicits._ -import com.twitter.util.Memoize - -object StratoSafetyLabelFetcher { - def apply(client: StratoClient): SafetyLabelFetcherType = { - val getFetcher: SafetyLabelType => Fetcher[Long, Unit, SafetyLabel] = - Memoize((safetyLabelType: SafetyLabelType) => - client.fetcher[Long, SafetyLabel](s"visibility/${safetyLabelType.name}.Tweet")) - - (tweetId, safetyLabelType) => getFetcher(safetyLabelType).fetch(tweetId).map(_.v) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelMapFetcher.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelMapFetcher.docx new file mode 100644 index 000000000..6ef9607e3 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelMapFetcher.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelMapFetcher.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelMapFetcher.scala deleted file mode 100644 index a76ad92a9..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/StratoSafetyLabelMapFetcher.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.visibility.interfaces.common.tweets - -import com.twitter.spam.rtf.thriftscala.SafetyLabelMap -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.strato.thrift.ScroogeConvImplicits._ - -object StratoSafetyLabelMapFetcher { - val column = "visibility/baseTweetSafetyLabelMap" - - def apply(client: StratoClient): SafetyLabelMapFetcherType = { - val fetcher: Fetcher[Long, Unit, SafetyLabelMap] = - client.fetcher[Long, SafetyLabelMap](column) - - tweetId => fetcher.fetch(tweetId).map(_.v) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/package.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/package.docx new file mode 100644 index 000000000..cb18d752d Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/package.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/package.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/package.scala deleted file mode 100644 index 82a8033d6..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/common/tweets/package.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.visibility.interfaces.common - -import com.twitter.search.blender.services.strato.UserSearchSafetySettings -import com.twitter.spam.rtf.thriftscala.SafetyLabel -import com.twitter.spam.rtf.thriftscala.SafetyLabelMap -import com.twitter.spam.rtf.thriftscala.SafetyLabelType -import com.twitter.stitch.Stitch - -package object tweets { - type SafetyLabelFetcherType = (Long, SafetyLabelType) => Stitch[Option[SafetyLabel]] - type SafetyLabelMapFetcherType = Long => Stitch[Option[SafetyLabelMap]] - type UserSearchSafetySettingsFetcherType = Long => Stitch[UserSearchSafetySettings] -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/AdAvoidanceLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/AdAvoidanceLibrary.docx new file mode 100644 index 000000000..e417460fd Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/AdAvoidanceLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/AdAvoidanceLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/AdAvoidanceLibrary.scala deleted file mode 100644 index 067fa833f..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/AdAvoidanceLibrary.scala +++ /dev/null @@ -1,158 +0,0 @@ -package com.twitter.visibility.interfaces.conversations - -import com.google.common.annotations.VisibleForTesting -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.GetTweetFieldsResult -import com.twitter.tweetypie.thriftscala.TweetFieldsResultFound -import com.twitter.tweetypie.thriftscala.TweetFieldsResultState -import com.twitter.util.Stopwatch -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.common.filtered_reason.FilteredReasonHelper -import com.twitter.visibility.models.ViewerContext -import com.twitter.visibility.rules.Interstitial -import com.twitter.visibility.rules.Tombstone - -case class AdAvoidanceRequest( - conversationId: Long, - focalTweetId: Long, - tweets: Seq[(GetTweetFieldsResult, Option[SafetyLevel])], - authorMap: Map[ - Long, - User - ], - moderatedTweetIds: Seq[Long], - viewerContext: ViewerContext, - useRichText: Boolean = true) - -case class AdAvoidanceResponse(dropAd: Map[Long, Boolean]) - -object AdAvoidanceLibrary { - type Type = - AdAvoidanceRequest => Stitch[AdAvoidanceResponse] - - private def shouldAvoid( - result: TweetFieldsResultState, - tombstoneOpt: Option[VfTombstone], - statsReceiver: StatsReceiver - ): Boolean = { - shouldAvoid(result, statsReceiver) || shouldAvoid(tombstoneOpt, statsReceiver) - } - - private def shouldAvoid( - result: TweetFieldsResultState, - statsReceiver: StatsReceiver - ): Boolean = { - result match { - case TweetFieldsResultState.Found(TweetFieldsResultFound(_, _, Some(filteredReason))) - if FilteredReasonHelper.isAvoid(filteredReason) => - statsReceiver.counter("avoid").incr() - true - case _ => false - } - } - - private def shouldAvoid( - tombstoneOpt: Option[VfTombstone], - statsReceiver: StatsReceiver, - ): Boolean = { - tombstoneOpt - .map(_.action).collect { - case Tombstone(epitaph, _) => - statsReceiver.scope("tombstone").counter(epitaph.name).incr() - true - case interstitial: Interstitial => - statsReceiver.scope("interstitial").counter(interstitial.reason.name).incr() - true - case _ => false - }.getOrElse(false) - } - - private def runTombstoneVisLib( - request: AdAvoidanceRequest, - tombstoneVisibilityLibrary: TombstoneVisibilityLibrary, - ): Stitch[TombstoneVisibilityResponse] = { - val tombstoneRequest = TombstoneVisibilityRequest( - conversationId = request.conversationId, - focalTweetId = request.focalTweetId, - tweets = request.tweets, - authorMap = request.authorMap, - moderatedTweetIds = request.moderatedTweetIds, - viewerContext = request.viewerContext, - useRichText = request.useRichText - ) - - tombstoneVisibilityLibrary(tombstoneRequest) - } - - def buildTweetAdAvoidanceMap(tweets: Seq[GetTweetFieldsResult]): Map[Long, Boolean] = tweets - .map(tweet => { - val shouldAvoid = tweet.tweetResult match { - case TweetFieldsResultState.Found(TweetFieldsResultFound(_, _, Some(filteredReason))) => - FilteredReasonHelper.isAvoid(filteredReason) - case _ => false - } - - tweet.tweetId -> shouldAvoid - }).toMap - - def apply(visibilityLibrary: VisibilityLibrary, decider: Decider): Type = { - val tvl = - TombstoneVisibilityLibrary(visibilityLibrary, visibilityLibrary.statsReceiver, decider) - buildLibrary(tvl, visibilityLibrary.statsReceiver) - } - - @VisibleForTesting - def buildLibrary( - tvl: TombstoneVisibilityLibrary, - libraryStatsReceiver: StatsReceiver - ): AdAvoidanceLibrary.Type = { - - val statsReceiver = libraryStatsReceiver.scope("AdAvoidanceLibrary") - val reasonsStatsReceiver = statsReceiver.scope("reasons") - val latencyStatsReceiver = statsReceiver.scope("latency") - val vfLatencyOverallStat = latencyStatsReceiver.stat("vf_latency_overall") - val vfLatencyStitchBuildStat = latencyStatsReceiver.stat("vf_latency_stitch_build") - val vfLatencyStitchRunStat = latencyStatsReceiver.stat("vf_latency_stitch_run") - - request: AdAvoidanceRequest => { - val elapsed = Stopwatch.start() - - var runStitchStartMs = 0L - - val tombstoneResponse: Stitch[TombstoneVisibilityResponse] = - runTombstoneVisLib(request, tvl) - - val response = tombstoneResponse - .map({ response: TombstoneVisibilityResponse => - statsReceiver.counter("requests").incr(request.tweets.size) - - val dropResults: Seq[(Long, Boolean)] = request.tweets.map(tweetAndSafetyLevel => { - val tweet = tweetAndSafetyLevel._1 - tweet.tweetId -> - shouldAvoid( - tweet.tweetResult, - response.tweetVerdicts.get(tweet.tweetId), - reasonsStatsReceiver) - }) - - AdAvoidanceResponse(dropAd = dropResults.toMap) - }) - .onSuccess(_ => { - val overallStatMs = elapsed().inMilliseconds - vfLatencyOverallStat.add(overallStatMs) - val runStitchEndMs = elapsed().inMilliseconds - vfLatencyStitchRunStat.add(runStitchEndMs - runStitchStartMs) - }) - - runStitchStartMs = elapsed().inMilliseconds - val buildStitchStatMs = elapsed().inMilliseconds - vfLatencyStitchBuildStat.add(buildStitchStatMs) - - response - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/BUILD deleted file mode 100644 index eae843def..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/BUILD +++ /dev/null @@ -1,46 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "3rdparty/jvm/com/ibm/icu:icu4j", - "decider/src/main/scala", - "servo/decider/src/main/scala", - "servo/repo", - "src/thrift/com/twitter/context:twitter-context-scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "translation/src/main/scala", - "twitter-config/yaml", - "twitter-context/src/main/scala", - "urt/richtext/src/main/scala/com/twitter/urt/richtext", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/common/src/main/scala/com/twitter/visibility/common/actions", - "visibility/common/src/main/scala/com/twitter/visibility/common/filtered_reason", - "visibility/common/src/main/thrift/com/twitter/visibility:action-scala", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/tweets", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/configs", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/params", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/tweets", - "visibility/lib/src/main/thrift/com/twitter/visibility/logging:vf-logging-scala", - "visibility/results/src/main/scala/com/twitter/visibility/results/richtext", - "visibility/results/src/main/scala/com/twitter/visibility/results/translation", - "visibility/results/src/main/scala/com/twitter/visibility/results/urt", - ], - exports = [ - "visibility/common/src/main/thrift/com/twitter/visibility:action-scala", - "visibility/lib/src/main/scala/com/twitter/visibility", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/BUILD.docx new file mode 100644 index 000000000..897238825 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityLibrary.docx new file mode 100644 index 000000000..2fdc76a30 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityLibrary.scala deleted file mode 100644 index cefe6d762..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityLibrary.scala +++ /dev/null @@ -1,260 +0,0 @@ -package com.twitter.visibility.interfaces.conversations - -import com.twitter.decider.Decider -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.Label -import com.twitter.servo.repository.KeyValueResult -import com.twitter.servo.util.Gate -import com.twitter.spam.rtf.thriftscala.SafetyLabel -import com.twitter.spam.rtf.thriftscala.SafetyLabelType -import com.twitter.spam.rtf.thriftscala.SafetyLabelValue -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import com.twitter.util.Return -import com.twitter.util.Stopwatch -import com.twitter.util.Try -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.tweets.TweetIdFeatures -import com.twitter.visibility.builder.FeatureMapBuilder -import com.twitter.visibility.builder.VerdictLogger -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.tweets.FosnrPefetchedLabelsRelationshipFeatures -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.configapi.configs.VisibilityDeciderGates -import com.twitter.visibility.features.AuthorUserLabels -import com.twitter.visibility.features.ConversationRootAuthorIsVerified -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.features.HasInnerCircleOfFriendsRelationship -import com.twitter.visibility.features.TweetConversationId -import com.twitter.visibility.features.TweetParentId -import com.twitter.visibility.logging.thriftscala.VFLibType -import com.twitter.visibility.models.ContentId.TweetId -import com.twitter.visibility.models.SafetyLevel.TimelineConversationsDownranking -import com.twitter.visibility.models.SafetyLevel.TimelineConversationsDownrankingMinimal -import com.twitter.visibility.models.SafetyLevel.toThrift -import com.twitter.visibility.models.ContentId -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.TweetSafetyLabel -import com.twitter.visibility.models.UnitOfDiversion - -object TimelineConversationsVisibilityLibrary { - type Type = - TimelineConversationsVisibilityRequest => Stitch[TimelineConversationsVisibilityResponse] - - def apply( - visibilityLibrary: VisibilityLibrary, - batchSafetyLabelRepository: BatchSafetyLabelRepository, - decider: Decider, - userRelationshipSource: UserRelationshipSource = UserRelationshipSource.empty, - userSource: UserSource = UserSource.empty - ): Type = { - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val tweetIdFeatures = new TweetIdFeatures( - statsReceiver = libraryStatsReceiver, - enableStitchProfiling = Gate.False - ) - val tweetIdFeaturesMinimal = new TweetIdFeatures( - statsReceiver = libraryStatsReceiver, - enableStitchProfiling = Gate.False - ) - val vfLatencyOverallStat = libraryStatsReceiver.stat("vf_latency_overall") - val vfLatencyStitchBuildStat = libraryStatsReceiver.stat("vf_latency_stitch_build") - val vfLatencyStitchRunStat = libraryStatsReceiver.stat("vf_latency_stitch_run") - - val visibilityDeciderGates = VisibilityDeciderGates(decider) - val verdictLogger = - createVerdictLogger( - visibilityDeciderGates.enableVerdictLoggerTCVL, - decider, - libraryStatsReceiver) - - request: TimelineConversationsVisibilityRequest => - val elapsed = Stopwatch.start() - var runStitchStartMs = 0L - - val future = request.prefetchedSafetyLabels match { - case Some(labels) => Future.value(labels) - case _ => - batchSafetyLabelRepository((request.conversationId, request.tweetIds)) - } - - val fosnrPefetchedLabelsRelationshipFeatures = - new FosnrPefetchedLabelsRelationshipFeatures( - userRelationshipSource = userRelationshipSource, - statsReceiver = libraryStatsReceiver) - - val authorFeatures = new AuthorFeatures(userSource, libraryStatsReceiver) - - Stitch.callFuture(future).flatMap { - kvr: KeyValueResult[Long, scala.collection.Map[SafetyLabelType, SafetyLabel]] => - val featureMapProvider: (ContentId, SafetyLevel) => FeatureMap = { - case (TweetId(tweetId), safetyLevel) => - val constantTweetSafetyLabels: Seq[TweetSafetyLabel] = - kvr.found.getOrElse(tweetId, Map.empty).toSeq.map { - case (safetyLabelType, safetyLabel) => - TweetSafetyLabel.fromThrift(SafetyLabelValue(safetyLabelType, safetyLabel)) - } - - val replyAuthor = request.tweetAuthors.flatMap { - _(tweetId) match { - case Return(Some(userId)) => Some(userId) - case _ => None - } - } - - val fosnrPefetchedLabelsRelationshipFeatureConf = replyAuthor match { - case Some(authorId) if visibilityLibrary.isReleaseCandidateEnabled => - fosnrPefetchedLabelsRelationshipFeatures - .forTweetWithSafetyLabelsAndAuthorId( - safetyLabels = constantTweetSafetyLabels, - authorId = authorId, - viewerId = request.viewerContext.userId) - case _ => fosnrPefetchedLabelsRelationshipFeatures.forNonFosnr() - } - - val authorFeatureConf = replyAuthor match { - case Some(authorId) if visibilityLibrary.isReleaseCandidateEnabled => - authorFeatures.forAuthorId(authorId) - case _ => authorFeatures.forNoAuthor() - } - - val baseBuilderArguments = (safetyLevel match { - case TimelineConversationsDownranking => - Seq(tweetIdFeatures.forTweetId(tweetId, constantTweetSafetyLabels)) - case TimelineConversationsDownrankingMinimal => - Seq(tweetIdFeaturesMinimal.forTweetId(tweetId, constantTweetSafetyLabels)) - case _ => Nil - }) :+ fosnrPefetchedLabelsRelationshipFeatureConf :+ authorFeatureConf - - val tweetAuthorUserLabels: Option[Seq[Label]] = - request.prefetchedTweetAuthorUserLabels.flatMap { - _.apply(tweetId) match { - case Return(Some(labelMap)) => - Some(labelMap.values.toSeq) - case _ => - None - } - } - - val hasInnerCircleOfFriendsRelationship: Boolean = - request.innerCircleOfFriendsRelationships match { - case Some(keyValueResult) => - keyValueResult(tweetId) match { - case Return(Some(true)) => true - case _ => false - } - case None => false - } - - val builderArguments: Seq[FeatureMapBuilder => FeatureMapBuilder] = - tweetAuthorUserLabels match { - case Some(labels) => - baseBuilderArguments :+ { (fmb: FeatureMapBuilder) => - fmb.withConstantFeature(AuthorUserLabels, labels) - } - - case None => - baseBuilderArguments :+ { (fmb: FeatureMapBuilder) => - fmb.withConstantFeature(AuthorUserLabels, Seq.empty) - } - case _ => - baseBuilderArguments - } - - val tweetParentIdOpt: Option[Long] = - request.tweetParentIdMap.flatMap(tweetParentIdMap => tweetParentIdMap(tweetId)) - - visibilityLibrary.featureMapBuilder(builderArguments :+ { (fmb: FeatureMapBuilder) => - fmb.withConstantFeature( - HasInnerCircleOfFriendsRelationship, - hasInnerCircleOfFriendsRelationship) - fmb.withConstantFeature(TweetConversationId, request.conversationId) - fmb.withConstantFeature(TweetParentId, tweetParentIdOpt) - fmb.withConstantFeature( - ConversationRootAuthorIsVerified, - request.rootAuthorIsVerified) - }) - case _ => - visibilityLibrary.featureMapBuilder(Nil) - } - val safetyLevel = - if (request.minimalSectioningOnly) TimelineConversationsDownrankingMinimal - else TimelineConversationsDownranking - - val evaluationContextBuilder = visibilityLibrary - .evaluationContextBuilder(request.viewerContext) - .withUnitOfDiversion(UnitOfDiversion.ConversationId(request.conversationId)) - - visibilityLibrary - .runRuleEngineBatch( - request.tweetIds.map(TweetId), - featureMapProvider, - evaluationContextBuilder, - safetyLevel - ) - .map { results: Seq[Try[VisibilityResult]] => - val (succeededRequests, _) = results.partition(_.exists(_.finished)) - val visibilityResultMap = succeededRequests.flatMap { - case Return(result) => - scribeVisibilityVerdict( - result, - visibilityDeciderGates.enableVerdictScribingTCVL, - verdictLogger, - request.viewerContext.userId, - safetyLevel) - result.contentId match { - case TweetId(id) => Some((id, result)) - case _ => None - } - case _ => None - }.toMap - val failedTweetIds = request.tweetIds diff visibilityResultMap.keys.toSeq - val response = TimelineConversationsVisibilityResponse( - visibilityResults = visibilityResultMap, - failedTweetIds = failedTweetIds - ) - - runStitchStartMs = elapsed().inMilliseconds - val buildStitchStatMs = elapsed().inMilliseconds - vfLatencyStitchBuildStat.add(buildStitchStatMs) - - response - } - .onSuccess(_ => { - val overallStatMs = elapsed().inMilliseconds - vfLatencyOverallStat.add(overallStatMs) - val runStitchEndMs = elapsed().inMilliseconds - vfLatencyStitchRunStat.add(runStitchEndMs - runStitchStartMs) - }) - } - } - - def scribeVisibilityVerdict( - visibilityResult: VisibilityResult, - enableVerdictScribing: Gate[Unit], - verdictLogger: VerdictLogger, - viewerId: Option[Long], - safetyLevel: SafetyLevel - ): Unit = if (enableVerdictScribing()) { - verdictLogger.scribeVerdict( - visibilityResult = visibilityResult, - viewerId = viewerId, - safetyLevel = toThrift(safetyLevel), - vfLibType = VFLibType.TimelineConversationsVisibilityLibrary) - } - - def createVerdictLogger( - enableVerdictLogger: Gate[Unit], - decider: Decider, - statsReceiver: StatsReceiver - ): VerdictLogger = { - if (enableVerdictLogger()) { - VerdictLogger(statsReceiver, decider) - } else { - VerdictLogger.Empty - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityRequest.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityRequest.docx new file mode 100644 index 000000000..92317e3a8 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityRequest.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityRequest.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityRequest.scala deleted file mode 100644 index 217296f8b..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityRequest.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.visibility.interfaces.conversations - -import com.twitter.gizmoduck.thriftscala.Label -import com.twitter.gizmoduck.thriftscala.LabelValue -import com.twitter.servo.repository.KeyValueResult -import com.twitter.spam.rtf.thriftscala.SafetyLabel -import com.twitter.spam.rtf.thriftscala.SafetyLabelType -import com.twitter.visibility.models.ViewerContext - -case class TimelineConversationsVisibilityRequest( - conversationId: Long, - tweetIds: Seq[Long], - viewerContext: ViewerContext, - minimalSectioningOnly: Boolean = false, - prefetchedSafetyLabels: Option[KeyValueResult[Long, Map[SafetyLabelType, SafetyLabel]]] = None, - prefetchedTweetAuthorUserLabels: Option[KeyValueResult[Long, Map[LabelValue, Label]]] = None, - innerCircleOfFriendsRelationships: Option[KeyValueResult[Long, Boolean]] = None, - tweetParentIdMap: Option[Map[Long, Option[Long]]] = None, - rootAuthorIsVerified: Boolean = false, - tweetAuthors: Option[KeyValueResult[Long, Long]] = None) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityResponse.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityResponse.docx new file mode 100644 index 000000000..1b9f1eeab Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityResponse.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityResponse.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityResponse.scala deleted file mode 100644 index f086c792c..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TimelineConversationsVisibilityResponse.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.visibility.interfaces.conversations - -import com.twitter.visibility.builder.VisibilityResult - -case class TimelineConversationsVisibilityResponse( - visibilityResults: Map[Long, VisibilityResult], - failedTweetIds: Seq[Long]) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/Tombstone.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/Tombstone.docx new file mode 100644 index 000000000..2a687f812 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/Tombstone.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/Tombstone.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/Tombstone.scala deleted file mode 100644 index 1d3274ed2..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/Tombstone.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.visibility.interfaces.conversations - -import com.twitter.timelines.render.thriftscala.TombstoneDisplayType -import com.twitter.timelines.render.thriftscala.TombstoneInfo -import com.twitter.visibility.rules._ - -case class VfTombstone( - tombstoneId: Long, - includeTweet: Boolean, - action: Action, - tombstoneInfo: Option[TombstoneInfo] = None, - tombstoneDisplayType: TombstoneDisplayType = TombstoneDisplayType.Inline, - truncateDescendantsWhenFocal: Boolean = false) { - - val isTruncatable: Boolean = action match { - case Interstitial(Reason.ViewerBlocksAuthor, _, _) => true - case Interstitial(Reason.ViewerHardMutedAuthor, _, _) => true - case Interstitial(Reason.MutedKeyword, _, _) => true - case Tombstone(Epitaph.NotFound, _) => true - case Tombstone(Epitaph.Unavailable, _) => true - case Tombstone(Epitaph.Suspended, _) => true - case Tombstone(Epitaph.Protected, _) => true - case Tombstone(Epitaph.Deactivated, _) => true - case Tombstone(Epitaph.BlockedBy, _) => true - case Tombstone(Epitaph.Moderated, _) => true - case Tombstone(Epitaph.Deleted, _) => true - case Tombstone(Epitaph.Underage, _) => true - case Tombstone(Epitaph.NoStatedAge, _) => true - case Tombstone(Epitaph.LoggedOutAge, _) => true - case Tombstone(Epitaph.SuperFollowsContent, _) => true - case Tombstone(Epitaph.CommunityTweetHidden, _) => true - case _: LocalizedTombstone => true - case _ => false - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TombstoneVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TombstoneVisibilityLibrary.docx new file mode 100644 index 000000000..709ff2401 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TombstoneVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TombstoneVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TombstoneVisibilityLibrary.scala deleted file mode 100644 index 3f228670d..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/TombstoneVisibilityLibrary.scala +++ /dev/null @@ -1,633 +0,0 @@ -package com.twitter.visibility.interfaces.conversations - -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.spam.rtf.thriftscala.FilteredReason -import com.twitter.spam.rtf.thriftscala.FilteredReason.UnspecifiedReason -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.spam.rtf.thriftscala.SafetyResult -import com.twitter.stitch.Stitch -import com.twitter.timelines.render.thriftscala.RichText -import com.twitter.timelines.render.thriftscala.TombstoneDisplayType -import com.twitter.timelines.render.thriftscala.TombstoneInfo -import com.twitter.tweetypie.thriftscala.GetTweetFieldsResult -import com.twitter.tweetypie.thriftscala.TweetFieldsResultFailed -import com.twitter.tweetypie.thriftscala.TweetFieldsResultFiltered -import com.twitter.tweetypie.thriftscala.TweetFieldsResultFound -import com.twitter.tweetypie.thriftscala.TweetFieldsResultNotFound -import com.twitter.tweetypie.thriftscala.TweetFieldsResultState -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.tweets.ModerationFeatures -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.builder.users.RelationshipFeatures -import com.twitter.visibility.builder.users.ViewerFeatures -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.common.actions.InterstitialReason -import com.twitter.visibility.common.actions.LimitedEngagementReason -import com.twitter.visibility.common.actions.TombstoneReason -import com.twitter.visibility.common.actions.converter.scala.InterstitialReasonConverter -import com.twitter.visibility.common.actions.converter.scala.LocalizedMessageConverter -import com.twitter.visibility.common.actions.converter.scala.TombstoneReasonConverter -import com.twitter.visibility.common.filtered_reason.FilteredReasonHelper -import com.twitter.visibility.configapi.configs.VisibilityDeciderGates -import com.twitter.visibility.features.FocalTweetId -import com.twitter.visibility.features.TweetId -import com.twitter.visibility.models.ContentId -import com.twitter.visibility.models.SafetyLevel.Tombstoning -import com.twitter.visibility.models.ViewerContext -import com.twitter.visibility.results.richtext.EpitaphToRichText -import com.twitter.visibility.results.richtext.LocalizedMessageToRichText -import com.twitter.visibility.results.urt.ReasonToUrtParser -import com.twitter.visibility.results.urt.SafetyResultToUrtParser -import com.twitter.visibility.rules._ -import com.twitter.visibility.{thriftscala => t} - -case class TombstoneVisibilityRequest( - conversationId: Long, - focalTweetId: Long, - tweets: Seq[(GetTweetFieldsResult, Option[SafetyLevel])], - authorMap: Map[ - Long, - User - ], - moderatedTweetIds: Seq[Long], - viewerContext: ViewerContext, - useRichText: Boolean = true) - -case class TombstoneVisibilityResponse(tweetVerdicts: Map[Long, VfTombstone]) - -case class TombstoneVisibilityLibrary( - visibilityLibrary: VisibilityLibrary, - statsReceiver: StatsReceiver, - decider: Decider) { - - private case class TombstoneType( - tweetId: Long, - tombstoneId: Long, - action: Action) { - - lazy val isInnerTombstone: Boolean = tweetId != tombstoneId - - lazy val tombstoneDisplayType: TombstoneDisplayType = action match { - case _: InterstitialLimitedEngagements | _: EmergencyDynamicInterstitial => - TombstoneDisplayType.NonCompliant - case _ => TombstoneDisplayType.Inline - } - } - - val En: String = "en" - val View: String = "View" - val relationshipFeatures = - new RelationshipFeatures( - statsReceiver) - val visibilityDeciderGates = VisibilityDeciderGates(decider) - - - def toAction( - filteredReason: FilteredReason, - actionStatsReceiver: StatsReceiver - ): Option[Action] = { - - val enableLocalizedInterstitials = - visibilityDeciderGates.enableConvosLocalizedInterstitial() - val enableLegacyInterstitials = - visibilityDeciderGates.enableConvosLegacyInterstitial() - - val tombstoneStatsReceiver = actionStatsReceiver.scope("tombstone") - val interstitialLocalStatsReceiver = - actionStatsReceiver.scope("interstitial").scope("localized") - val interstitialLegacyStatsReceiver = - actionStatsReceiver.scope("interstitial").scope("legacy") - - filteredReason match { - case _ if FilteredReasonHelper.isTombstone(filteredReason) => - createLocalizedTombstone(filteredReason, tombstoneStatsReceiver) match { - case tombstoneOpt @ Some(LocalizedTombstone(_, _)) => tombstoneOpt - case _ => - createTombstone(Epitaph.Unavailable, tombstoneStatsReceiver, Some("emptyTombstone")) - } - - case _ - if enableLocalizedInterstitials && - FilteredReasonHelper.isLocalizedSuppressedReasonInterstitial(filteredReason) => - FilteredReasonHelper.getLocalizedSuppressedReasonInterstitial(filteredReason) match { - case Some(t.Interstitial(reasonOpt, Some(message))) => - InterstitialReasonConverter.fromThrift(reasonOpt).map { interstitialReason => - interstitialLocalStatsReceiver.counter("interstitial").incr() - Interstitial( - Reason.fromInterstitialReason(interstitialReason), - Some(LocalizedMessageConverter.fromThrift(message))) - } - - case _ => None - } - - case _ if FilteredReasonHelper.containNsfwMedia(filteredReason) => - None - - case _ if FilteredReasonHelper.possiblyUndesirable(filteredReason) => - None - - case _ if FilteredReasonHelper.reportedTweet(filteredReason) => - filteredReason match { - case FilteredReason.ReportedTweet(true) => - interstitialLegacyStatsReceiver.counter("fr_reported").incr() - Some(Interstitial(Reason.ViewerReportedAuthor)) - - case FilteredReason.SafetyResult(safetyResult: SafetyResult) - if enableLegacyInterstitials => - val safetyResultReported = InterstitialReasonConverter - .fromAction(safetyResult.action).collect { - case InterstitialReason.ViewerReportedTweet => true - case InterstitialReason.ViewerReportedAuthor => true - }.getOrElse(false) - - if (safetyResultReported) { - interstitialLegacyStatsReceiver.counter("reported_author").incr() - Some(Interstitial(Reason.ViewerReportedAuthor)) - } else None - - case _ => None - } - - case _ if FilteredReasonHelper.tweetMatchesViewerMutedKeyword(filteredReason) => - filteredReason match { - case FilteredReason.TweetMatchesViewerMutedKeyword(_) => - interstitialLegacyStatsReceiver.counter("fr_muted_keyword").incr() - Some(Interstitial(Reason.MutedKeyword)) - - case FilteredReason.SafetyResult(safetyResult: SafetyResult) - if enableLegacyInterstitials => - val safetyResultMutedKeyword = InterstitialReasonConverter - .fromAction(safetyResult.action).collect { - case _: InterstitialReason.MatchesMutedKeyword => true - }.getOrElse(false) - - if (safetyResultMutedKeyword) { - interstitialLegacyStatsReceiver.counter("muted_keyword").incr() - Some(Interstitial(Reason.MutedKeyword)) - } else None - - case _ => None - } - - case _ => - None - } - } - - def toAction( - tfrs: TweetFieldsResultState, - actionStatsReceiver: StatsReceiver - ): Option[Action] = { - - val enableLocalizedInterstitials = visibilityDeciderGates.enableConvosLocalizedInterstitial() - val enableLegacyInterstitials = visibilityDeciderGates.enableConvosLegacyInterstitial() - - val tombstoneStatsReceiver = actionStatsReceiver.scope("tombstone") - val interstitialLocalStatsReceiver = - actionStatsReceiver.scope("interstitial").scope("localized") - val interstitialLegacyStatsReceiver = - actionStatsReceiver.scope("interstitial").scope("legacy") - - tfrs match { - - case TweetFieldsResultState.NotFound(TweetFieldsResultNotFound(_, _, Some(filteredReason))) - if FilteredReasonHelper.isTombstone(filteredReason) => - createLocalizedTombstone(filteredReason, tombstoneStatsReceiver) - - case TweetFieldsResultState.NotFound(tfr: TweetFieldsResultNotFound) if tfr.deleted => - createTombstone(Epitaph.Deleted, tombstoneStatsReceiver) - - case TweetFieldsResultState.NotFound(_: TweetFieldsResultNotFound) => - createTombstone(Epitaph.NotFound, tombstoneStatsReceiver) - - case TweetFieldsResultState.Failed(TweetFieldsResultFailed(_, _, _)) => - createTombstone(Epitaph.Unavailable, tombstoneStatsReceiver, Some("failed")) - - case TweetFieldsResultState.Filtered(TweetFieldsResultFiltered(UnspecifiedReason(true))) => - createTombstone(Epitaph.Unavailable, tombstoneStatsReceiver, Some("filtered")) - - case TweetFieldsResultState.Filtered(TweetFieldsResultFiltered(filteredReason)) => - toAction(filteredReason, actionStatsReceiver) - - case TweetFieldsResultState.Found(TweetFieldsResultFound(_, _, Some(filteredReason))) - if enableLocalizedInterstitials && - FilteredReasonHelper.isSuppressedReasonPublicInterestInterstial(filteredReason) => - interstitialLocalStatsReceiver.counter("ipi").incr() - FilteredReasonHelper - .getSafetyResult(filteredReason) - .flatMap(_.reason) - .flatMap(PublicInterest.SafetyResultReasonToReason.get) match { - case Some(safetyResultReason) => - FilteredReasonHelper - .getSuppressedReasonPublicInterestInterstial(filteredReason) - .map(edi => edi.localizedMessage) - .map(tlm => LocalizedMessageConverter.fromThrift(tlm)) - .map(lm => - InterstitialLimitedEngagements( - safetyResultReason, - Some(LimitedEngagementReason.NonCompliant), - lm)) - case _ => None - } - - case TweetFieldsResultState.Found(TweetFieldsResultFound(_, _, Some(filteredReason))) - if enableLegacyInterstitials && - FilteredReasonHelper.isSuppressedReasonPublicInterestInterstial(filteredReason) => - interstitialLegacyStatsReceiver.counter("ipi").incr() - FilteredReasonHelper - .getSafetyResult(filteredReason) - .flatMap(_.reason) - .flatMap(PublicInterest.SafetyResultReasonToReason.get) - .map(InterstitialLimitedEngagements(_, Some(LimitedEngagementReason.NonCompliant))) - - case TweetFieldsResultState.Found(TweetFieldsResultFound(_, _, Some(filteredReason))) - if enableLocalizedInterstitials && - FilteredReasonHelper.isLocalizedSuppressedReasonEmergencyDynamicInterstitial( - filteredReason) => - interstitialLocalStatsReceiver.counter("edi").incr() - FilteredReasonHelper - .getSuppressedReasonEmergencyDynamicInterstitial(filteredReason) - .map(e => - EmergencyDynamicInterstitial( - e.copy, - e.link, - LocalizedMessageConverter.fromThrift(e.localizedMessage))) - - case TweetFieldsResultState.Found(TweetFieldsResultFound(_, _, Some(filteredReason))) - if enableLegacyInterstitials && - FilteredReasonHelper.isSuppressedReasonEmergencyDynamicInterstitial(filteredReason) => - interstitialLegacyStatsReceiver.counter("edi").incr() - FilteredReasonHelper - .getSuppressedReasonEmergencyDynamicInterstitial(filteredReason) - .map(e => EmergencyDynamicInterstitial(e.copy, e.link)) - - case TweetFieldsResultState.Found(TweetFieldsResultFound(tweet, _, _)) - if tweet.perspective.exists(_.reported) => - interstitialLegacyStatsReceiver.counter("reported").incr() - Some(Interstitial(Reason.ViewerReportedAuthor)) - - case TweetFieldsResultState.Found( - TweetFieldsResultFound(_, _, Some(UnspecifiedReason(true)))) => - None - - case TweetFieldsResultState.Found(TweetFieldsResultFound(_, _, Some(filteredReason))) => - toAction(filteredReason, actionStatsReceiver) - - case _ => - None - } - } - - private[conversations] def shouldTruncateDescendantsWhenFocal(action: Action): Boolean = - action match { - case _: InterstitialLimitedEngagements | _: EmergencyDynamicInterstitial => - true - case Tombstone(Epitaph.Bounced, _) | Tombstone(Epitaph.BounceDeleted, _) => - true - case LocalizedTombstone(TombstoneReason.Bounced, _) | - LocalizedTombstone(TombstoneReason.BounceDeleted, _) => - true - case LimitedEngagements(LimitedEngagementReason.NonCompliant, _) => - true - case _ => false - } - - def apply(request: TombstoneVisibilityRequest): Stitch[TombstoneVisibilityResponse] = { - - val moderationFeatures = new ModerationFeatures( - moderationSource = request.moderatedTweetIds.contains, - statsReceiver = statsReceiver - ) - - val userSource = UserSource.fromFunction({ - case (userId, _) => - request.authorMap - .get(userId) - .map(Stitch.value).getOrElse(Stitch.NotFound) - }) - - val authorFeatures = new AuthorFeatures(userSource, statsReceiver) - val viewerFeatures = new ViewerFeatures(userSource, statsReceiver) - - val languageTag = request.viewerContext.requestCountryCode.getOrElse(En) - val firstRound: Seq[(GetTweetFieldsResult, Option[TombstoneType])] = request.tweets.map { - case (gtfr, safetyLevel) => - val actionStats = statsReceiver - .scope("action") - .scope(safetyLevel.map(_.toString().toLowerCase()).getOrElse("unknown_safety_level")) - toAction(gtfr.tweetResult, actionStats) match { - case Some(action) => - (gtfr, Some(TombstoneType(gtfr.tweetId, gtfr.tweetId, action))) - - case None => - val quotedTweetId: Option[Long] = gtfr.tweetResult match { - case TweetFieldsResultState.Found(TweetFieldsResultFound(tweet, _, _)) => - tweet.quotedTweet.map(_.tweetId) - case _ => None - } - - (quotedTweetId, gtfr.quotedTweetResult) match { - case (Some(quotedTweetId), Some(tfrs)) => - val qtActionStats = actionStats.scope("quoted") - toAction(tfrs, qtActionStats) match { - case None => - (gtfr, None) - - case Some(action) => - (gtfr, Some(TombstoneType(gtfr.tweetId, quotedTweetId, action))) - } - - case _ => - (gtfr, None) - } - } - } - - val (firstRoundActions, secondRoundInput) = firstRound.partition { - case (_, Some(tombstoneType)) => - !tombstoneType.isInnerTombstone - case (_, None) => false - } - - def invokeVisibilityLibrary(tweetId: Long, author: User): Stitch[Action] = { - visibilityLibrary - .runRuleEngine( - ContentId.TweetId(tweetId), - visibilityLibrary.featureMapBuilder( - Seq( - viewerFeatures.forViewerContext(request.viewerContext), - moderationFeatures.forTweetId(tweetId), - authorFeatures.forAuthor(author), - relationshipFeatures - .forAuthor(author, request.viewerContext.userId), - _.withConstantFeature(TweetId, tweetId), - _.withConstantFeature(FocalTweetId, request.focalTweetId) - ) - ), - request.viewerContext, - Tombstoning - ).map(_.verdict) - } - - val secondRoundActions: Stitch[Seq[(GetTweetFieldsResult, Option[TombstoneType])]] = - Stitch.traverse(secondRoundInput) { - case (gtfr: GetTweetFieldsResult, firstRoundTombstone: Option[TombstoneType]) => - val secondRoundTombstone: Stitch[Option[TombstoneType]] = gtfr.tweetResult match { - case TweetFieldsResultState.Found(TweetFieldsResultFound(tweet, _, _)) => - val tweetId = tweet.id - - tweet.coreData - .flatMap { coreData => request.authorMap.get(coreData.userId) } match { - case Some(author) => - invokeVisibilityLibrary(tweetId, author).flatMap { - case Allow => - val quotedTweetId = tweet.quotedTweet.map(_.tweetId) - val quotedTweetAuthor = tweet.quotedTweet.flatMap { qt => - request.authorMap.get(qt.userId) - } - - (quotedTweetId, quotedTweetAuthor) match { - case (Some(quotedTweetId), Some(quotedTweetAuthor)) => - invokeVisibilityLibrary(quotedTweetId, quotedTweetAuthor).flatMap { - case Allow => - Stitch.None - - case reason => - Stitch.value(Some(TombstoneType(tweetId, quotedTweetId, reason))) - } - - case _ => - Stitch.None - } - - case reason => - Stitch.value(Some(TombstoneType(tweetId, tweetId, reason))) - } - - case None => - Stitch.None - } - - case _ => - Stitch.None - } - - secondRoundTombstone.map { opt => opt.orElse(firstRoundTombstone) }.map { opt => - (gtfr, opt) - } - } - - secondRoundActions.map { secondRound => - val tombstones: Seq[(Long, VfTombstone)] = (firstRoundActions ++ secondRound).flatMap { - case (gtfr, tombstoneTypeOpt) => { - - val nonCompliantLimitedEngagementsOpt = gtfr.tweetResult match { - case TweetFieldsResultState.Found(TweetFieldsResultFound(_, _, Some(filteredReason))) - if FilteredReasonHelper.isLimitedEngagementsNonCompliant(filteredReason) => - Some(LimitedEngagements(LimitedEngagementReason.NonCompliant)) - case _ => None - } - - (tombstoneTypeOpt, nonCompliantLimitedEngagementsOpt) match { - case (Some(tombstoneType), nonCompliantOpt) => - val tombstoneId = tombstoneType.tombstoneId - val action = tombstoneType.action - val textOpt: Option[RichText] = action match { - - case InterstitialLimitedEngagements(_, _, Some(localizedMessage), _) => - Some(LocalizedMessageToRichText(localizedMessage)) - case ipi: InterstitialLimitedEngagements => - Some( - SafetyResultToUrtParser.fromSafetyResultToRichText( - SafetyResult( - Some(PublicInterest.ReasonToSafetyResultReason(ipi.reason)), - ipi.toActionThrift() - ), - languageTag - ) - ) - - case EmergencyDynamicInterstitial(_, _, Some(localizedMessage), _) => - Some(LocalizedMessageToRichText(localizedMessage)) - case edi: EmergencyDynamicInterstitial => - Some( - SafetyResultToUrtParser.fromSafetyResultToRichText( - SafetyResult( - None, - edi.toActionThrift() - ), - languageTag - ) - ) - - case Tombstone(epitaph, _) => - if (request.useRichText) - Some(EpitaphToRichText(epitaph, languageTag)) - else - Some(EpitaphToRichText(Epitaph.UnavailableWithoutLink, languageTag)) - - case LocalizedTombstone(_, message) => - if (request.useRichText) - Some(LocalizedMessageToRichText(LocalizedMessageConverter.toThrift(message))) - else - Some(EpitaphToRichText(Epitaph.UnavailableWithoutLink, languageTag)) - - case Interstitial(_, Some(localizedMessage), _) => - Some(LocalizedMessageToRichText.apply(localizedMessage)) - - case interstitial: Interstitial => - ReasonToUrtParser.fromReasonToRichText(interstitial.reason, languageTag) - - case _ => - None - } - - val isRoot: Boolean = gtfr.tweetId == request.conversationId - val isOuter: Boolean = tombstoneId == request.conversationId - val revealTextOpt: Option[RichText] = action match { - case _: InterstitialLimitedEngagements | _: EmergencyDynamicInterstitial - if isRoot && isOuter => - None - - case _: Interstitial | _: InterstitialLimitedEngagements | - _: EmergencyDynamicInterstitial => - Some(ReasonToUrtParser.getRichRevealText(languageTag)) - - case _ => - None - } - - val includeTweet = action match { - case _: Interstitial | _: InterstitialLimitedEngagements | - _: EmergencyDynamicInterstitial => - true - case _ => false - } - - val truncateForAction: Boolean = - shouldTruncateDescendantsWhenFocal(action) - val truncateForNonCompliant: Boolean = - nonCompliantOpt - .map(shouldTruncateDescendantsWhenFocal).getOrElse(false) - val truncateDescendants: Boolean = - truncateForAction || truncateForNonCompliant - - val tombstone = textOpt match { - case Some(_) if request.useRichText => - VfTombstone( - includeTweet = includeTweet, - action = action, - tombstoneInfo = Some( - TombstoneInfo( - cta = None, - revealText = None, - richText = textOpt, - richRevealText = revealTextOpt - ) - ), - tombstoneDisplayType = tombstoneType.tombstoneDisplayType, - truncateDescendantsWhenFocal = truncateDescendants - ) - case Some(_) => - VfTombstone( - includeTweet = includeTweet, - action = action, - tombstoneInfo = Some( - TombstoneInfo( - text = textOpt - .map(richText => richText.text).getOrElse( - "" - cta = None, - revealText = revealTextOpt.map(_.text), - richText = None, - richRevealText = None - ) - ), - tombstoneDisplayType = tombstoneType.tombstoneDisplayType, - truncateDescendantsWhenFocal = truncateDescendants - ) - - case None => - VfTombstone( - includeTweet = false, - action = action, - tombstoneInfo = Some( - TombstoneInfo( - cta = None, - revealText = None, - richText = Some(EpitaphToRichText(Epitaph.Unavailable, languageTag)), - richRevealText = None - ) - ), - tombstoneDisplayType = tombstoneType.tombstoneDisplayType, - truncateDescendantsWhenFocal = truncateDescendants - ) - } - - Some((gtfr.tweetId, tombstone)) - - case (None, Some(limitedEngagements)) - if shouldTruncateDescendantsWhenFocal(limitedEngagements) => - val tombstone = VfTombstone( - tombstoneId = gtfr.tweetId, - includeTweet = true, - action = limitedEngagements, - tombstoneInfo = None, - tombstoneDisplayType = TombstoneDisplayType.NonCompliant, - truncateDescendantsWhenFocal = true - ) - Some((gtfr.tweetId, tombstone)) - - case _ => - None - } - } - } - - TombstoneVisibilityResponse( - tweetVerdicts = tombstones.toMap - ) - } - } - - private def createLocalizedTombstone( - filteredReason: FilteredReason, - tombstoneStats: StatsReceiver, - ): Option[LocalizedTombstone] = { - - val tombstoneOpt = FilteredReasonHelper.getTombstone(filteredReason) - tombstoneOpt match { - case Some(t.Tombstone(reasonOpt, Some(message))) => - TombstoneReasonConverter.fromThrift(reasonOpt).map { localReason => - tombstoneStats - .scope("localized").counter(localReason.toString().toLowerCase()).incr() - LocalizedTombstone(localReason, LocalizedMessageConverter.fromThrift(message)) - } - - case _ => None - } - } - - private def createTombstone( - epitaph: Epitaph, - tombstoneStats: StatsReceiver, - extraCounterOpt: Option[String] = None - ): Option[Action] = { - tombstoneStats - .scope("legacy") - .counter(epitaph.toString().toLowerCase()) - .incr() - extraCounterOpt.map { extraCounter => - tombstoneStats - .scope("legacy") - .scope(epitaph.toString().toLowerCase()) - .counter(extraCounter) - .incr() - } - Some(Tombstone(epitaph)) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/package.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/package.docx new file mode 100644 index 000000000..ad460354a Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/package.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/package.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/package.scala deleted file mode 100644 index 4064fc33b..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/conversations/package.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.visibility.interfaces - -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.spam.rtf.thriftscala.SafetyLabel -import com.twitter.spam.rtf.thriftscala.SafetyLabelType -import scala.collection.Map - -package object conversations { - type BatchSafetyLabelRepository = - KeyValueRepository[(Long, Seq[Long]), Long, Map[SafetyLabelType, SafetyLabel]] -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/BUILD deleted file mode 100644 index 2d2dfacf8..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "src/thrift/com/twitter/tweetypie:events-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/common/src/main/thrift/com/twitter/visibility:action-scala", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/tweets", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/tweets", - ], - exports = [ - "visibility/lib/src/main/scala/com/twitter/visibility", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/BUILD.docx new file mode 100644 index 000000000..0277e9686 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESRealtimeVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESRealtimeVisibilityLibrary.docx new file mode 100644 index 000000000..a99015acf Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESRealtimeVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESRealtimeVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESRealtimeVisibilityLibrary.scala deleted file mode 100644 index dd6cc68de..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESRealtimeVisibilityLibrary.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.visibility.interfaces.des - -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.tweets.CommunityTweetFeaturesV2 -import com.twitter.visibility.builder.tweets.EditTweetFeatures -import com.twitter.visibility.builder.tweets.ExclusiveTweetFeatures -import com.twitter.visibility.builder.tweets.NilTweetLabelMaps -import com.twitter.visibility.builder.tweets.TrustedFriendsFeatures -import com.twitter.visibility.builder.tweets.TweetFeatures -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.builder.users.ViewerFeatures -import com.twitter.visibility.common.CommunitiesSource -import com.twitter.visibility.common.TrustedFriendsSource -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.models.ContentId -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext -import com.twitter.visibility.rules.Allow -import com.twitter.visibility.{thriftscala => vfthrift} - -case class DESRealtimeVisibilityRequest(tweet: Tweet, author: User, viewer: Option[User]) - -object DESRealtimeVisibilityLibrary { - type Type = DESRealtimeVisibilityRequest => Stitch[vfthrift.Action] - - private[this] val safetyLevel = SafetyLevel.DesRealtime - - def apply(visibilityLibrary: VisibilityLibrary): Type = { - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val vfEngineCounter = libraryStatsReceiver.counter("vf_engine_requests") - - val tweetFeatures = new TweetFeatures(NilTweetLabelMaps, libraryStatsReceiver) - - val authorFeatures = new AuthorFeatures(UserSource.empty, libraryStatsReceiver) - val viewerFeatures = new ViewerFeatures(UserSource.empty, libraryStatsReceiver) - val communityTweetFeatures = new CommunityTweetFeaturesV2(CommunitiesSource.empty) - val exclusiveTweetFeatures = - new ExclusiveTweetFeatures(UserRelationshipSource.empty, libraryStatsReceiver) - val trustedFriendsTweetFeatures = new TrustedFriendsFeatures(TrustedFriendsSource.empty) - val editTweetFeatures = new EditTweetFeatures(libraryStatsReceiver) - - { request: DESRealtimeVisibilityRequest => - vfEngineCounter.incr() - - val tweet = request.tweet - val author = request.author - val viewer = request.viewer - val viewerContext = ViewerContext.fromContext - - val featureMap = - visibilityLibrary.featureMapBuilder( - Seq( - tweetFeatures.forTweetWithoutSafetyLabels(tweet), - authorFeatures.forAuthorNoDefaults(author), - viewerFeatures.forViewerNoDefaults(viewer), - communityTweetFeatures.forTweetOnly(tweet), - exclusiveTweetFeatures.forTweetOnly(tweet), - trustedFriendsTweetFeatures.forTweetOnly(tweet), - editTweetFeatures.forTweet(tweet), - ) - ) - - val tweetResult = visibilityLibrary.runRuleEngine( - ContentId.TweetId(tweet.id), - featureMap, - viewerContext, - safetyLevel - ) - val authorResult = visibilityLibrary.runRuleEngine( - ContentId.UserId(author.id), - featureMap, - viewerContext, - safetyLevel - ) - - Stitch.join(tweetResult, authorResult).map { - case (tweetResult, authorResult) => mergeResults(tweetResult, authorResult) - } - } - } - - def mergeResults( - tweetResult: VisibilityResult, - authorResult: VisibilityResult, - ): vfthrift.Action = { - Set(tweetResult.verdict, authorResult.verdict) - .find { - case Allow => false - case _ => true - } - .map(_.toActionThrift()) - .getOrElse(Allow.toActionThrift()) - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESVisibilityLibrary.docx new file mode 100644 index 000000000..0d9684c49 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESVisibilityLibrary.scala deleted file mode 100644 index b3297c67c..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/des/DESVisibilityLibrary.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.visibility.interfaces.des - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.tweets.StratoTweetLabelMaps -import com.twitter.visibility.builder.tweets.TweetFeatures -import com.twitter.visibility.common.SafetyLabelMapSource -import com.twitter.visibility.features.AuthorId -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.interfaces.common.tweets.SafetyLabelMapFetcherType -import com.twitter.visibility.models.ContentId.TweetId -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext - -case class DESVisibilityRequest( - tweet: Tweet, - visibilitySurface: SafetyLevel, - viewerContext: ViewerContext) - -object DESVisibilityLibrary { - type Type = DESVisibilityRequest => Stitch[VisibilityResult] - - def apply( - visibilityLibrary: VisibilityLibrary, - getLabelMap: SafetyLabelMapFetcherType, - enableShimFeatureHydration: Any => Boolean = _ => false - ): Type = { - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val vfEngineCounter = libraryStatsReceiver.counter("vf_engine_requests") - - val tweetLabelMap = new StratoTweetLabelMaps( - SafetyLabelMapSource.fromSafetyLabelMapFetcher(getLabelMap)) - val tweetFeatures = new TweetFeatures(tweetLabelMap, libraryStatsReceiver) - - { request: DESVisibilityRequest => - vfEngineCounter.incr() - - val contentId = TweetId(request.tweet.id) - val authorId = coreData.userId - - val featureMap = - visibilityLibrary.featureMapBuilder( - Seq( - tweetFeatures.forTweet(request.tweet), - _.withConstantFeature(AuthorId, Set(authorId)) - ) - ) - - val isShimFeatureHydrationEnabled = enableShimFeatureHydration() - - if (isShimFeatureHydrationEnabled) { - FeatureMap.resolve(featureMap, libraryStatsReceiver).flatMap { resolvedFeatureMap => - visibilityLibrary.runRuleEngine( - contentId, - resolvedFeatureMap, - request.viewerContext, - request.visibilitySurface - ) - } - } else { - visibilityLibrary.runRuleEngine( - contentId, - featureMap, - request.viewerContext, - request.visibilitySurface - ) - } - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/BUILD deleted file mode 100644 index 5a3dcb977..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "decider/src/main/scala", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/catalog", - "strato/src/main/scala/com/twitter/strato/client", - "strato/src/main/scala/com/twitter/strato/data", - "strato/src/main/scala/com/twitter/strato/thrift", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/common/src/main/scala/com/twitter/visibility/common/dm_sources", - "visibility/common/src/main/scala/com/twitter/visibility/common/stitch", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/dms", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/configs", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/rules/providers", - "visibility/lib/src/main/scala/com/twitter/visibility/rules/utils", - "visibility/lib/src/main/scala/com/twitter/visibility/util", - "visibility/lib/src/main/thrift/com/twitter/visibility/safety_label_store:safety-label-store-scala", - ], - exports = [ - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/lib/src/main/scala/com/twitter/visibility", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/BUILD.docx new file mode 100644 index 000000000..35cf3dbac Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityLibrary.docx new file mode 100644 index 000000000..265e1d075 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityLibrary.scala deleted file mode 100644 index 23d547c3e..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityLibrary.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.visibility.interfaces.dms - -import com.twitter.servo.util.Gate -import com.twitter.stitch.Stitch -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.dms.DmConversationFeatures -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.common.dm_sources.DmConversationSource -import com.twitter.visibility.common.stitch.StitchHelpers -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.models.ContentId.DmConversationId -import com.twitter.visibility.rules.Drop -import com.twitter.visibility.rules.EvaluationContext -import com.twitter.visibility.rules.Reason -import com.twitter.visibility.rules.RuleBase -import com.twitter.visibility.rules.providers.ProvidedEvaluationContext -import com.twitter.visibility.rules.utils.ShimUtils - -object DmConversationVisibilityLibrary { - type Type = DmConversationVisibilityRequest => Stitch[VisibilityResult] - - def apply( - visibilityLibrary: VisibilityLibrary, - stratoClient: StratoClient, - userSource: UserSource, - enableVfFeatureHydrationInShim: Gate[Unit] = Gate.False - ): Type = { - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val stratoClientStatsReceiver = visibilityLibrary.statsReceiver.scope("strato") - val vfLatencyStatsReceiver = visibilityLibrary.statsReceiver.scope("vf_latency") - val vfEngineCounter = libraryStatsReceiver.counter("vf_engine_requests") - - val dmConversationSource = - DmConversationSource.fromStrato(stratoClient, stratoClientStatsReceiver) - val authorFeatures = new AuthorFeatures(userSource, libraryStatsReceiver) - val dmConversationFeatures = new DmConversationFeatures(dmConversationSource, authorFeatures) - - { req: DmConversationVisibilityRequest => - val dmConversationId = req.dmConversationId - val contentId = DmConversationId(dmConversationId) - val safetyLevel = req.safetyLevel - - if (!RuleBase.hasDmConversationRules(safetyLevel)) { - Stitch.value(VisibilityResult(contentId = contentId, verdict = Drop(Reason.Unspecified))) - } else { - vfEngineCounter.incr() - - val viewerContext = req.viewerContext - val viewerId = viewerContext.userId - val isVfFeatureHydrationEnabled: Boolean = - enableVfFeatureHydrationInShim() - - val featureMap = visibilityLibrary.featureMapBuilder( - Seq(dmConversationFeatures.forDmConversationId(dmConversationId, viewerId))) - - val resp = if (isVfFeatureHydrationEnabled) { - val evaluationContext = ProvidedEvaluationContext.injectRuntimeRulesIntoEvaluationContext( - evaluationContext = EvaluationContext( - safetyLevel, - visibilityLibrary.getParams(viewerContext, safetyLevel), - visibilityLibrary.statsReceiver) - ) - - val preFilteredFeatureMap = - ShimUtils.preFilterFeatureMap(featureMap, safetyLevel, contentId, evaluationContext) - - FeatureMap.resolve(preFilteredFeatureMap, libraryStatsReceiver).flatMap { - resolvedFeatureMap => - visibilityLibrary - .runRuleEngine( - contentId, - resolvedFeatureMap, - viewerContext, - safetyLevel - ) - } - } else { - visibilityLibrary - .runRuleEngine( - contentId, - featureMap, - viewerContext, - safetyLevel - ) - } - - StitchHelpers.profileStitch(resp, Seq(vfLatencyStatsReceiver)) - } - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityRequest.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityRequest.docx new file mode 100644 index 000000000..fbd5ff64b Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityRequest.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityRequest.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityRequest.scala deleted file mode 100644 index 0b3eac66c..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmConversationVisibilityRequest.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.visibility.interfaces.dms - -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext - -case class DmConversationVisibilityRequest( - dmConversationId: String, - safetyLevel: SafetyLevel, - viewerContext: ViewerContext) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityLibrary.docx new file mode 100644 index 000000000..79726bd86 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityLibrary.scala deleted file mode 100644 index d66539459..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityLibrary.scala +++ /dev/null @@ -1,80 +0,0 @@ -package com.twitter.visibility.interfaces.dms - -import com.twitter.stitch.Stitch -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.dms.DmConversationFeatures -import com.twitter.visibility.builder.dms.DmEventFeatures -import com.twitter.visibility.builder.dms.InvalidDmEventFeatureException -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.common.dm_sources.DmConversationSource -import com.twitter.visibility.common.dm_sources.DmEventSource -import com.twitter.visibility.common.stitch.StitchHelpers -import com.twitter.visibility.models.ContentId.DmEventId -import com.twitter.visibility.rules.Drop -import com.twitter.visibility.rules.Reason -import com.twitter.visibility.rules.RuleBase - -object DmEventVisibilityLibrary { - type Type = DmEventVisibilityRequest => Stitch[VisibilityResult] - - def apply( - visibilityLibrary: VisibilityLibrary, - stratoClient: StratoClient, - userSource: UserSource - ): Type = { - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val stratoClientStatsReceiver = visibilityLibrary.statsReceiver.scope("strato") - val vfLatencyStatsReceiver = visibilityLibrary.statsReceiver.scope("vf_latency") - val vfEngineCounter = libraryStatsReceiver.counter("vf_engine_requests") - val dmConversationSource = { - DmConversationSource.fromStrato(stratoClient, stratoClientStatsReceiver) - } - val dmEventSource = { - DmEventSource.fromStrato(stratoClient, stratoClientStatsReceiver) - } - val authorFeatures = new AuthorFeatures(userSource, libraryStatsReceiver) - val dmConversationFeatures = new DmConversationFeatures(dmConversationSource, authorFeatures) - val dmEventFeatures = - new DmEventFeatures( - dmEventSource, - dmConversationSource, - authorFeatures, - dmConversationFeatures, - libraryStatsReceiver) - - { req: DmEventVisibilityRequest => - val dmEventId = req.dmEventId - val contentId = DmEventId(dmEventId) - val safetyLevel = req.safetyLevel - - if (!RuleBase.hasDmEventRules(safetyLevel)) { - Stitch.value(VisibilityResult(contentId = contentId, verdict = Drop(Reason.Unspecified))) - } else { - vfEngineCounter.incr() - - val viewerContext = req.viewerContext - val viewerIdOpt = viewerContext.userId - - viewerIdOpt match { - case Some(viewerId) => - val featureMap = visibilityLibrary.featureMapBuilder( - Seq(dmEventFeatures.forDmEventId(dmEventId, viewerId))) - - val resp = visibilityLibrary - .runRuleEngine( - contentId, - featureMap, - viewerContext, - safetyLevel - ) - StitchHelpers.profileStitch(resp, Seq(vfLatencyStatsReceiver)) - - case None => Stitch.exception(InvalidDmEventFeatureException("Viewer id is missing")) - } - } - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityRequest.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityRequest.docx new file mode 100644 index 000000000..52a052563 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityRequest.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityRequest.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityRequest.scala deleted file mode 100644 index c7e9c65af..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmEventVisibilityRequest.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.visibility.interfaces.dms - -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext - -case class DmEventVisibilityRequest( - dmEventId: Long, - safetyLevel: SafetyLevel, - viewerContext: ViewerContext) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmVisibilityLibrary.docx new file mode 100644 index 000000000..63bc20f70 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmVisibilityLibrary.scala deleted file mode 100644 index d6a7d3ce5..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/DmVisibilityLibrary.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.visibility.interfaces.dms - -import com.twitter.servo.util.Gate -import com.twitter.stitch.Stitch -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.common.DmId -import com.twitter.visibility.common.UserId -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.models.ContentId.{DmId => DmContentId} -import com.twitter.visibility.models.SafetyLevel.DirectMessages -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext -import com.twitter.visibility.rules.Drop -import com.twitter.visibility.rules.Reason.DeactivatedAuthor -import com.twitter.visibility.rules.Reason.ErasedAuthor -import com.twitter.visibility.rules.Reason.Nsfw - -object DmVisibilityLibrary { - type Type = DmVisibilityRequest => Stitch[DmVisibilityResponse] - - case class DmVisibilityRequest( - dmId: DmId, - dmAuthorUserId: UserId, - viewerContext: ViewerContext) - - case class DmVisibilityResponse(isMessageNsfw: Boolean) - - val DefaultSafetyLevel: SafetyLevel = DirectMessages - - def apply( - visibilityLibrary: VisibilityLibrary, - stratoClient: StratoClient, - userSource: UserSource, - enableVfFeatureHydrationInShim: Gate[Unit] = Gate.False - ): Type = { - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val vfEngineCounter = libraryStatsReceiver.counter("vf_engine_requests") - - val authorFeatures = new AuthorFeatures(userSource, libraryStatsReceiver) - - { r: DmVisibilityRequest => - vfEngineCounter.incr() - - val contentId = DmContentId(r.dmId) - val dmAuthorUserId = r.dmAuthorUserId - val isVfFeatureHydrationEnabled = enableVfFeatureHydrationInShim() - - val featureMap = - visibilityLibrary.featureMapBuilder( - Seq(authorFeatures.forAuthorId(dmAuthorUserId)) - ) - - val resp = if (isVfFeatureHydrationEnabled) { - FeatureMap.resolve(featureMap, libraryStatsReceiver).flatMap { resolvedFeatureMap => - visibilityLibrary.runRuleEngine( - contentId, - resolvedFeatureMap, - r.viewerContext, - DefaultSafetyLevel - ) - } - } else { - visibilityLibrary - .runRuleEngine( - contentId, - featureMap, - r.viewerContext, - DefaultSafetyLevel - ) - } - - resp.map(buildResponse) - } - } - - private[this] def buildResponse(visibilityResult: VisibilityResult) = - visibilityResult.verdict match { - case Drop(Nsfw | ErasedAuthor | DeactivatedAuthor, _) => - DmVisibilityResponse(isMessageNsfw = true) - case _ => - DmVisibilityResponse(isMessageNsfw = false) - } - -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/package.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/package.docx new file mode 100644 index 000000000..136beda3a Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/package.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/package.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/package.scala deleted file mode 100644 index 0b8f4b410..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/dms/package.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.visibility.interfaces - -import com.twitter.stitch.Stitch -import com.twitter.visibility.common.DmId -import com.twitter.visibility.safety_label_store.thriftscala.DmSafetyLabelMap - -package object dms { - type DmSafetyLabelMapFetcherType = DmId => Stitch[Option[DmSafetyLabelMap]] - - val DmSafetyLabelMapFetcherStratoColumn = - "visibility/safety-label-store/vflib/dm/safetyLabelMap.Dm" -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/BUILD.bazel b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/BUILD.bazel deleted file mode 100644 index 1fe0d3dd8..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "mediaservices/media-util", - "stitch/stitch-core", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/media", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/rules/providers", - "visibility/lib/src/main/scala/com/twitter/visibility/rules/utils", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/BUILD.docx new file mode 100644 index 000000000..a2c0dfd3d Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityLibrary.docx new file mode 100644 index 000000000..e6bb778e6 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityLibrary.scala deleted file mode 100644 index 79f25d3ea..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityLibrary.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.visibility.interfaces.media - -import com.twitter.stitch.Stitch -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.util.Stopwatch -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.users.ViewerFeatures -import com.twitter.visibility.builder.media.MediaFeatures -import com.twitter.visibility.builder.media.MediaMetadataFeatures -import com.twitter.visibility.builder.media.StratoMediaLabelMaps -import com.twitter.visibility.common.MediaMetadataSource -import com.twitter.visibility.common.MediaSafetyLabelMapSource -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.generators.TombstoneGenerator -import com.twitter.visibility.models.ContentId.MediaId -import com.twitter.visibility.rules.EvaluationContext -import com.twitter.visibility.rules.providers.ProvidedEvaluationContext -import com.twitter.visibility.rules.utils.ShimUtils - -object MediaVisibilityLibrary { - type Type = MediaVisibilityRequest => Stitch[VisibilityResult] - - def apply( - visibilityLibrary: VisibilityLibrary, - userSource: UserSource, - tombstoneGenerator: TombstoneGenerator, - stratoClient: StratoClient, - ): Type = { - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val vfEngineCounter = libraryStatsReceiver.counter("vf_engine_requests") - val vfLatencyOverallStat = libraryStatsReceiver.stat("vf_latency_overall") - val vfLatencyStitchRunStat = libraryStatsReceiver.stat("vf_latency_stitch_run") - - val stratoClientStatsReceiver = libraryStatsReceiver.scope("strato") - - val mediaMetadataFeatures = new MediaMetadataFeatures( - MediaMetadataSource.fromStrato(stratoClient, stratoClientStatsReceiver), - libraryStatsReceiver) - - val mediaLabelMaps = new StratoMediaLabelMaps( - MediaSafetyLabelMapSource.fromStrato(stratoClient, stratoClientStatsReceiver)) - val mediaFeatures = new MediaFeatures(mediaLabelMaps, libraryStatsReceiver) - - val viewerFeatures = new ViewerFeatures(userSource, libraryStatsReceiver) - - { r: MediaVisibilityRequest => - vfEngineCounter.incr() - - val contentId = MediaId(r.mediaKey.toStringKey) - val languageCode = r.viewerContext.requestLanguageCode.getOrElse("en") - - val featureMap = visibilityLibrary.featureMapBuilder( - Seq( - viewerFeatures.forViewerContext(r.viewerContext), - mediaFeatures.forGenericMediaKey(r.mediaKey), - mediaMetadataFeatures.forGenericMediaKey(r.mediaKey), - ) - ) - - val evaluationContext = ProvidedEvaluationContext.injectRuntimeRulesIntoEvaluationContext( - evaluationContext = EvaluationContext( - r.safetyLevel, - visibilityLibrary.getParams(r.viewerContext, r.safetyLevel), - visibilityLibrary.statsReceiver) - ) - - val preFilteredFeatureMap = - ShimUtils.preFilterFeatureMap(featureMap, r.safetyLevel, contentId, evaluationContext) - - val elapsed = Stopwatch.start() - FeatureMap.resolve(preFilteredFeatureMap, libraryStatsReceiver).flatMap { - resolvedFeatureMap => - vfLatencyStitchRunStat.add(elapsed().inMilliseconds) - - visibilityLibrary - .runRuleEngine( - contentId, - resolvedFeatureMap, - r.viewerContext, - r.safetyLevel - ) - .map(tombstoneGenerator(_, languageCode)) - .onSuccess(_ => vfLatencyOverallStat.add(elapsed().inMilliseconds)) - } - } - } -} diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityRequest.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityRequest.docx new file mode 100644 index 000000000..9c3729635 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityRequest.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityRequest.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityRequest.scala deleted file mode 100644 index 0f6b78f77..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/media/MediaVisibilityRequest.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.visibility.interfaces.media - -import com.twitter.mediaservices.media_util.GenericMediaKey -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext - -case class MediaVisibilityRequest( - mediaKey: GenericMediaKey, - safetyLevel: SafetyLevel, - viewerContext: ViewerContext) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/BUILD b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/BUILD deleted file mode 100644 index 37b3f5dba..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/BUILD +++ /dev/null @@ -1,31 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/model:alias", - "notificationservice/common/src/main/scala/com/twitter/notificationservice/model/notification", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-label-scala", - "stitch/stitch-core", - "stitch/stitch-gizmoduck/src/main/scala", - "stitch/stitch-socialgraph", - "stitch/stitch-socialgraph/src/main/scala", - "visibility/common/src/main/scala/com/twitter/visibility/common", - "visibility/common/src/main/thrift/com/twitter/visibility:action-scala", - "visibility/lib/src/main/resources/config", - "visibility/lib/src/main/scala/com/twitter/visibility", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/tweets", - "visibility/lib/src/main/scala/com/twitter/visibility/builder/users", - "visibility/lib/src/main/scala/com/twitter/visibility/configapi/params", - "visibility/lib/src/main/scala/com/twitter/visibility/features", - "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/common/tweets", - ], - exports = [ - "visibility/lib/src/main/scala/com/twitter/visibility", - ], -) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/BUILD.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/BUILD.docx new file mode 100644 index 000000000..0eab9b82c Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/BUILD.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationVFRequest.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationVFRequest.docx new file mode 100644 index 000000000..d399ad924 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationVFRequest.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationVFRequest.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationVFRequest.scala deleted file mode 100644 index edf92aeab..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationVFRequest.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.visibility.interfaces.notifications - -import com.twitter.visibility.models.ContentId -import com.twitter.visibility.models.SafetyLevel - -case class NotificationVFRequest( - recipientId: Long, - subject: ContentId.UserId, - safetyLevel: SafetyLevel) diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsFilteringResponse.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsFilteringResponse.docx new file mode 100644 index 000000000..a76f97ad8 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsFilteringResponse.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsFilteringResponse.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsFilteringResponse.scala deleted file mode 100644 index bc500cce6..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsFilteringResponse.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.visibility.interfaces.notifications - -import com.twitter.visibility.features.Feature -import com.twitter.visibility.rules.Action -import scala.collection.immutable.Set - -sealed trait NotificationsFilteringResponse - -case object Allow extends NotificationsFilteringResponse - -case class Filtered(action: Action) extends NotificationsFilteringResponse - -case class Failed(features: Set[Feature[_]]) extends NotificationsFilteringResponse diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformFilteringResponse.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformFilteringResponse.docx new file mode 100644 index 000000000..15de4a9c8 Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformFilteringResponse.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformFilteringResponse.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformFilteringResponse.scala deleted file mode 100644 index 0a5624e49..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformFilteringResponse.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.visibility.interfaces.notifications - -import com.twitter.visibility.features.Feature -import com.twitter.visibility.rules.Action - -trait NotificationsPlatformFilteringResponse - -case object AllowVerdict extends NotificationsPlatformFilteringResponse - -case class FilteredVerdict(action: Action) extends NotificationsPlatformFilteringResponse - -case class FailedVerdict(featuresMap: Map[Feature[_], String]) - extends NotificationsPlatformFilteringResponse diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformVisibilityLibrary.docx b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformVisibilityLibrary.docx new file mode 100644 index 000000000..309d7a05f Binary files /dev/null and b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformVisibilityLibrary.docx differ diff --git a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformVisibilityLibrary.scala b/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformVisibilityLibrary.scala deleted file mode 100644 index bdd2f59f1..000000000 --- a/visibilitylib/src/main/scala/com/twitter/visibility/interfaces/notifications/NotificationsPlatformVisibilityLibrary.scala +++ /dev/null @@ -1,157 +0,0 @@ -package com.twitter.visibility.interfaces.notifications - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.Gate -import com.twitter.stitch.Stitch -import com.twitter.util.Throwables -import com.twitter.visibility.VisibilityLibrary -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.builder.tweets.CommunityNotificationFeatures -import com.twitter.visibility.builder.tweets.UnmentionNotificationFeatures -import com.twitter.visibility.builder.users.AuthorDeviceFeatures -import com.twitter.visibility.builder.users.AuthorFeatures -import com.twitter.visibility.builder.users.RelationshipFeatures -import com.twitter.visibility.builder.users.ViewerAdvancedFilteringFeatures -import com.twitter.visibility.builder.users.ViewerFeatures -import com.twitter.visibility.common.UserDeviceSource -import com.twitter.visibility.common.UserRelationshipSource -import com.twitter.visibility.common.UserSource -import com.twitter.visibility.features.AuthorUserLabels -import com.twitter.visibility.features.Feature -import com.twitter.visibility.features.FeatureMap -import com.twitter.visibility.models.ViewerContext -import com.twitter.visibility.rules.State.FeatureFailed -import com.twitter.visibility.rules.State.MissingFeature -import com.twitter.visibility.rules.Action -import com.twitter.visibility.rules.RuleResult -import com.twitter.visibility.rules.{Allow => AllowAction} - -object NotificationsPlatformVisibilityLibrary { - type NotificationsPlatformVFType = - NotificationVFRequest => Stitch[NotificationsPlatformFilteringResponse] - - private val AllowResponse: Stitch[NotificationsPlatformFilteringResponse] = - Stitch.value(AllowVerdict) - - def apply( - userSource: UserSource, - userRelationshipSource: UserRelationshipSource, - userDeviceSource: UserDeviceSource, - visibilityLibrary: VisibilityLibrary, - enableShimFeatureHydration: Gate[Unit] = Gate.False - ): NotificationsPlatformVFType = { - val libraryStatsReceiver = visibilityLibrary.statsReceiver - val vfEngineCounter = libraryStatsReceiver.counter("vf_engine_requests") - - val authorFeatures = new AuthorFeatures(userSource, libraryStatsReceiver) - val authorDeviceFeatures = new AuthorDeviceFeatures(userDeviceSource, libraryStatsReceiver) - val viewerFeatures = new ViewerFeatures(userSource, libraryStatsReceiver) - - val viewerAdvancedFilteringFeatures = - new ViewerAdvancedFilteringFeatures(userSource, libraryStatsReceiver) - val relationshipFeatures = - new RelationshipFeatures(userRelationshipSource, libraryStatsReceiver) - - val isShimFeatureHydrationEnabled = enableShimFeatureHydration() - - def runRuleEngine(candidate: NotificationVFRequest): Stitch[VisibilityResult] = { - val featureMap = - visibilityLibrary.featureMapBuilder( - Seq( - viewerFeatures.forViewerId(Some(candidate.recipientId)), - viewerAdvancedFilteringFeatures.forViewerId(Some(candidate.recipientId)), - authorFeatures.forAuthorId(candidate.subject.id), - authorDeviceFeatures.forAuthorId(candidate.subject.id), - relationshipFeatures.forAuthorId(candidate.subject.id, Some(candidate.recipientId)), - CommunityNotificationFeatures.ForNonCommunityTweetNotification, - UnmentionNotificationFeatures.ForNonUnmentionNotificationFeatures - ) - ) - - vfEngineCounter.incr() - - if (isShimFeatureHydrationEnabled) { - FeatureMap.resolve(featureMap, libraryStatsReceiver).flatMap { resolvedFeatureMap => - visibilityLibrary.runRuleEngine( - contentId = candidate.subject, - featureMap = resolvedFeatureMap, - viewerContext = - ViewerContext.fromContextWithViewerIdFallback(Some(candidate.recipientId)), - safetyLevel = candidate.safetyLevel - ) - } - } else { - visibilityLibrary.runRuleEngine( - contentId = candidate.subject, - featureMap = featureMap, - viewerContext = - ViewerContext.fromContextWithViewerIdFallback(Some(candidate.recipientId)), - safetyLevel = candidate.safetyLevel - ) - } - } - - { - case candidate: NotificationVFRequest => - runRuleEngine(candidate).flatMap(failCloseForFailures(_, libraryStatsReceiver)) - case _ => - AllowResponse - } - } - - private def failCloseForFailures( - visibilityResult: VisibilityResult, - stats: StatsReceiver - ): Stitch[NotificationsPlatformFilteringResponse] = { - lazy val vfEngineSuccess = stats.counter("vf_engine_success") - lazy val vfEngineFailures = stats.counter("vf_engine_failures") - lazy val vfEngineFailuresMissing = stats.scope("vf_engine_failures").counter("missing") - lazy val vfEngineFailuresFailed = stats.scope("vf_engine_failures").counter("failed") - lazy val vfEngineFiltered = stats.counter("vf_engine_filtered") - - val isFailedOrMissingFeature: RuleResult => Boolean = { - case RuleResult(_, FeatureFailed(features)) => - !(features.contains(AuthorUserLabels) && features.size == 1) - case RuleResult(_, MissingFeature(_)) => true - case _ => false - } - - val failedRuleResults = - visibilityResult.ruleResultMap.values.filter(isFailedOrMissingFeature(_)) - - val (failedFeatures, missingFeatures) = failedRuleResults.partition { - case RuleResult(_, FeatureFailed(_)) => true - case RuleResult(_, MissingFeature(_)) => false - case _ => false - } - - val failedOrMissingFeatures: Map[Feature[_], String] = failedRuleResults - .collect { - case RuleResult(_, FeatureFailed(features)) => - features.map { - case (feature: Feature[_], throwable: Throwable) => - feature -> Throwables.mkString(throwable).mkString(" -> ") - }.toSet - case RuleResult(_, MissingFeature(features)) => features.map(_ -> "Feature missing.") - }.flatten.toMap - - visibilityResult.verdict match { - case AllowAction if failedOrMissingFeatures.isEmpty => - vfEngineSuccess.incr() - AllowResponse - case AllowAction if failedOrMissingFeatures.nonEmpty => - vfEngineFailures.incr() - if (missingFeatures.nonEmpty) { - vfEngineFailuresMissing.incr() - } - if (failedFeatures.nonEmpty) { - vfEngineFailuresFailed.incr() - } - - Stitch.value(FailedVerdict(failedOrMissingFeatures)) - case action: Action => - vfEngineFiltered.incr() - Stitch.value(FilteredVerdict(action)) - } - } -}