diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ItemTypeFilterPredicates.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ItemTypeFilterPredicates.docx new file mode 100644 index 000000000..61f0289c1 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ItemTypeFilterPredicates.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ItemTypeFilterPredicates.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ItemTypeFilterPredicates.scala deleted file mode 100644 index 6fb43b09c..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ItemTypeFilterPredicates.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.unified_user_actions.adapter.client_event - -import com.twitter.clientapp.thriftscala.ItemType - -object ItemTypeFilterPredicates { - private val TweetItemTypes = Set[ItemType](ItemType.Tweet, ItemType.QuotedTweet) - private val TopicItemTypes = Set[ItemType](ItemType.Tweet, ItemType.QuotedTweet, ItemType.Topic) - private val ProfileItemTypes = Set[ItemType](ItemType.User) - private val TypeaheadResultItemTypes = Set[ItemType](ItemType.Search, ItemType.User) - private val SearchResultsPageFeedbackSubmitItemTypes = - Set[ItemType](ItemType.Tweet, ItemType.RelevancePrompt) - - /** - * DDG lambda metrics count Tweets based on the `itemType` - * Reference code - https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/scala/com/twitter/experiments/lambda/shared/Timelines.scala?L156 - * Since enums `PROMOTED_TWEET` and `POPULAR_TWEET` are deprecated in the following thrift - * https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/thrift/com/twitter/clientapp/gen/client_app.thrift?L131 - * UUA filters two types of Tweets only: `TWEET` and `QUOTED_TWEET` - */ - def isItemTypeTweet(itemTypeOpt: Option[ItemType]): Boolean = - itemTypeOpt.exists(itemType => TweetItemTypes.contains(itemType)) - - def isItemTypeTopic(itemTypeOpt: Option[ItemType]): Boolean = - itemTypeOpt.exists(itemType => TopicItemTypes.contains(itemType)) - - def isItemTypeProfile(itemTypeOpt: Option[ItemType]): Boolean = - itemTypeOpt.exists(itemType => ProfileItemTypes.contains(itemType)) - - def isItemTypeTypeaheadResult(itemTypeOpt: Option[ItemType]): Boolean = - itemTypeOpt.exists(itemType => TypeaheadResultItemTypes.contains(itemType)) - - def isItemTypeForSearchResultsPageFeedbackSubmit(itemTypeOpt: Option[ItemType]): Boolean = - itemTypeOpt.exists(itemType => SearchResultsPageFeedbackSubmitItemTypes.contains(itemType)) - - /** - * Always return true. Use this when there is no need to filter based on `item_type` and all - * values of `item_type` are acceptable. - */ - def ignoreItemType(itemTypeOpt: Option[ItemType]): Boolean = true -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/NotificationClientEventUtils.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/NotificationClientEventUtils.docx new file mode 100644 index 000000000..89fbb8069 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/NotificationClientEventUtils.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/NotificationClientEventUtils.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/NotificationClientEventUtils.scala deleted file mode 100644 index 4a49a155f..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/NotificationClientEventUtils.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.unified_user_actions.adapter.client_event - -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.clientapp.thriftscala.{Item => LogEventItem} - -object NotificationClientEventUtils { - - // Notification id for notification in the Notification Tab - def getNotificationIdForNotificationTab( - ceItem: LogEventItem - ): Option[String] = { - for { - notificationTabDetails <- ceItem.notificationTabDetails - clientEventMetaData <- notificationTabDetails.clientEventMetadata - notificationId <- clientEventMetaData.upstreamId - } yield { - notificationId - } - } - - // Notification id for Push Notification - def getNotificationIdForPushNotification(logEvent: LogEvent): Option[String] = for { - pushNotificationDetails <- logEvent.notificationDetails - notificationId <- pushNotificationDetails.impressionId - } yield notificationId -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ProductSurfaceUtils.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ProductSurfaceUtils.docx new file mode 100644 index 000000000..631a17352 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ProductSurfaceUtils.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ProductSurfaceUtils.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ProductSurfaceUtils.scala deleted file mode 100644 index d0d0e5825..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/ProductSurfaceUtils.scala +++ /dev/null @@ -1,109 +0,0 @@ -package com.twitter.unified_user_actions.adapter.client_event - -import com.twitter.clientapp.thriftscala.EventNamespace -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.clientapp.thriftscala.{Item => LogEventItem} -import com.twitter.suggests.controller_data.home_tweets.thriftscala.HomeTweetsControllerDataAliases.V1Alias -import com.twitter.unified_user_actions.thriftscala._ - -object ProductSurfaceUtils { - - def getProductSurface(eventNamespace: Option[EventNamespace]): Option[ProductSurface] = { - ( - eventNamespace.flatMap(_.page), - eventNamespace.flatMap(_.section), - eventNamespace.flatMap(_.element)) match { - case (Some("home") | Some("home_latest"), _, _) => Some(ProductSurface.HomeTimeline) - case (Some("ntab"), _, _) => Some(ProductSurface.NotificationTab) - case (Some(page), Some(section), _) if isPushNotification(page, section) => - Some(ProductSurface.PushNotification) - case (Some("search"), _, _) => Some(ProductSurface.SearchResultsPage) - case (_, _, Some("typeahead")) => Some(ProductSurface.SearchTypeahead) - case _ => None - } - } - - private def isPushNotification(page: String, section: String): Boolean = { - Seq[String]("notification", "toasts").contains(page) || - (page == "app" && section == "push") - } - - def getProductSurfaceInfo( - productSurface: Option[ProductSurface], - ceItem: LogEventItem, - logEvent: LogEvent - ): Option[ProductSurfaceInfo] = { - productSurface match { - case Some(ProductSurface.HomeTimeline) => createHomeTimelineInfo(ceItem) - case Some(ProductSurface.NotificationTab) => createNotificationTabInfo(ceItem) - case Some(ProductSurface.PushNotification) => createPushNotificationInfo(logEvent) - case Some(ProductSurface.SearchResultsPage) => createSearchResultPageInfo(ceItem, logEvent) - case Some(ProductSurface.SearchTypeahead) => createSearchTypeaheadInfo(ceItem, logEvent) - case _ => None - } - } - - private def createPushNotificationInfo(logEvent: LogEvent): Option[ProductSurfaceInfo] = - NotificationClientEventUtils.getNotificationIdForPushNotification(logEvent) match { - case Some(notificationId) => - Some( - ProductSurfaceInfo.PushNotificationInfo( - PushNotificationInfo(notificationId = notificationId))) - case _ => None - } - - private def createNotificationTabInfo(ceItem: LogEventItem): Option[ProductSurfaceInfo] = - NotificationClientEventUtils.getNotificationIdForNotificationTab(ceItem) match { - case Some(notificationId) => - Some( - ProductSurfaceInfo.NotificationTabInfo( - NotificationTabInfo(notificationId = notificationId))) - case _ => None - } - - private def createHomeTimelineInfo(ceItem: LogEventItem): Option[ProductSurfaceInfo] = { - def suggestType: Option[String] = HomeInfoUtils.getSuggestType(ceItem) - def controllerData: Option[V1Alias] = HomeInfoUtils.getHomeTweetControllerDataV1(ceItem) - - if (suggestType.isDefined || controllerData.isDefined) { - Some( - ProductSurfaceInfo.HomeTimelineInfo( - HomeTimelineInfo( - suggestionType = suggestType, - injectedPosition = controllerData.flatMap(_.injectedPosition) - ))) - } else None - } - - private def createSearchResultPageInfo( - ceItem: LogEventItem, - logEvent: LogEvent - ): Option[ProductSurfaceInfo] = { - val searchInfoUtil = new SearchInfoUtils(ceItem) - searchInfoUtil.getQueryOptFromItem(logEvent).map { query => - ProductSurfaceInfo.SearchResultsPageInfo( - SearchResultsPageInfo( - query = query, - querySource = searchInfoUtil.getQuerySourceOptFromControllerDataFromItem, - itemPosition = ceItem.position, - tweetResultSources = searchInfoUtil.getTweetResultSources, - userResultSources = searchInfoUtil.getUserResultSources, - queryFilterType = searchInfoUtil.getQueryFilterType(logEvent) - )) - } - } - - private def createSearchTypeaheadInfo( - ceItem: LogEventItem, - logEvent: LogEvent - ): Option[ProductSurfaceInfo] = { - logEvent.searchDetails.flatMap(_.query).map { query => - ProductSurfaceInfo.SearchTypeaheadInfo( - SearchTypeaheadInfo( - query = query, - itemPosition = ceItem.position - ) - ) - } - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/SearchInfoUtils.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/SearchInfoUtils.docx new file mode 100644 index 000000000..05f705f06 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/SearchInfoUtils.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/SearchInfoUtils.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/SearchInfoUtils.scala deleted file mode 100644 index 4ebbbbeee..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/SearchInfoUtils.scala +++ /dev/null @@ -1,129 +0,0 @@ -package com.twitter.unified_user_actions.adapter.client_event - -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.clientapp.thriftscala.{Item => LogEventItem} -import com.twitter.search.common.constants.thriftscala.ThriftQuerySource -import com.twitter.search.common.constants.thriftscala.TweetResultSource -import com.twitter.search.common.constants.thriftscala.UserResultSource -import com.twitter.suggests.controller_data.search_response.item_types.thriftscala.ItemTypesControllerData -import com.twitter.suggests.controller_data.search_response.item_types.thriftscala.ItemTypesControllerData.TweetTypesControllerData -import com.twitter.suggests.controller_data.search_response.item_types.thriftscala.ItemTypesControllerData.UserTypesControllerData -import com.twitter.suggests.controller_data.search_response.request.thriftscala.RequestControllerData -import com.twitter.suggests.controller_data.search_response.thriftscala.SearchResponseControllerData.V1 -import com.twitter.suggests.controller_data.search_response.thriftscala.SearchResponseControllerDataAliases.V1Alias -import com.twitter.suggests.controller_data.thriftscala.ControllerData.V2 -import com.twitter.suggests.controller_data.v2.thriftscala.ControllerData.SearchResponse -import com.twitter.unified_user_actions.thriftscala.SearchQueryFilterType -import com.twitter.unified_user_actions.thriftscala.SearchQueryFilterType._ - -class SearchInfoUtils(item: LogEventItem) { - private val searchControllerDataOpt: Option[V1Alias] = item.suggestionDetails.flatMap { sd => - sd.decodedControllerData.flatMap { decodedControllerData => - decodedControllerData match { - case V2(v2ControllerData) => - v2ControllerData match { - case SearchResponse(searchResponseControllerData) => - searchResponseControllerData match { - case V1(searchResponseControllerDataV1) => - Some(searchResponseControllerDataV1) - case _ => None - } - case _ => - None - } - case _ => None - } - } - } - - private val requestControllerDataOptFromItem: Option[RequestControllerData] = - searchControllerDataOpt.flatMap { searchControllerData => - searchControllerData.requestControllerData - } - private val itemTypesControllerDataOptFromItem: Option[ItemTypesControllerData] = - searchControllerDataOpt.flatMap { searchControllerData => - searchControllerData.itemTypesControllerData - } - - def checkBit(bitmap: Long, idx: Int): Boolean = { - (bitmap / Math.pow(2, idx)).toInt % 2 == 1 - } - - def getQueryOptFromSearchDetails(logEvent: LogEvent): Option[String] = { - logEvent.searchDetails.flatMap { sd => sd.query } - } - - def getQueryOptFromControllerDataFromItem: Option[String] = { - requestControllerDataOptFromItem.flatMap { rd => rd.rawQuery } - } - - def getQueryOptFromItem(logEvent: LogEvent): Option[String] = { - // First we try to get the query from controller data, and if that's not available, we fall - // back to query in search details. If both are None, queryOpt is None. - getQueryOptFromControllerDataFromItem.orElse(getQueryOptFromSearchDetails(logEvent)) - } - - def getTweetTypesOptFromControllerDataFromItem: Option[TweetTypesControllerData] = { - itemTypesControllerDataOptFromItem.flatMap { itemTypes => - itemTypes match { - case TweetTypesControllerData(tweetTypesControllerData) => - Some(TweetTypesControllerData(tweetTypesControllerData)) - case _ => None - } - } - } - - def getUserTypesOptFromControllerDataFromItem: Option[UserTypesControllerData] = { - itemTypesControllerDataOptFromItem.flatMap { itemTypes => - itemTypes match { - case UserTypesControllerData(userTypesControllerData) => - Some(UserTypesControllerData(userTypesControllerData)) - case _ => None - } - } - } - - def getQuerySourceOptFromControllerDataFromItem: Option[ThriftQuerySource] = { - requestControllerDataOptFromItem - .flatMap { rd => rd.querySource } - .flatMap { querySourceVal => ThriftQuerySource.get(querySourceVal) } - } - - def getTweetResultSources: Option[Set[TweetResultSource]] = { - getTweetTypesOptFromControllerDataFromItem - .flatMap { cd => cd.tweetTypesControllerData.tweetTypesBitmap } - .map { tweetTypesBitmap => - TweetResultSource.list.filter { t => checkBit(tweetTypesBitmap, t.value) }.toSet - } - } - - def getUserResultSources: Option[Set[UserResultSource]] = { - getUserTypesOptFromControllerDataFromItem - .flatMap { cd => cd.userTypesControllerData.userTypesBitmap } - .map { userTypesBitmap => - UserResultSource.list.filter { t => checkBit(userTypesBitmap, t.value) }.toSet - } - } - - def getQueryFilterType(logEvent: LogEvent): Option[SearchQueryFilterType] = { - val searchTab = logEvent.eventNamespace.map(_.client).flatMap { - case Some("m5") | Some("android") => logEvent.eventNamespace.flatMap(_.element) - case _ => logEvent.eventNamespace.flatMap(_.section) - } - searchTab.flatMap { - case "search_filter_top" => Some(Top) - case "search_filter_live" => Some(Latest) - // android uses search_filter_tweets instead of search_filter_live - case "search_filter_tweets" => Some(Latest) - case "search_filter_user" => Some(People) - case "search_filter_image" => Some(Photos) - case "search_filter_video" => Some(Videos) - case _ => None - } - } - - def getRequestJoinId: Option[Long] = requestControllerDataOptFromItem.flatMap(_.requestJoinId) - - def getTraceId: Option[Long] = requestControllerDataOptFromItem.flatMap(_.traceId) - -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/TopicIdUtils.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/TopicIdUtils.docx new file mode 100644 index 000000000..90ebed227 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/TopicIdUtils.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/TopicIdUtils.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/TopicIdUtils.scala deleted file mode 100644 index 16f8c9b35..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/TopicIdUtils.scala +++ /dev/null @@ -1,157 +0,0 @@ -package com.twitter.unified_user_actions.adapter.client_event - -import com.twitter.clientapp.thriftscala.EventNamespace -import com.twitter.clientapp.thriftscala.Item -import com.twitter.clientapp.thriftscala.ItemType.Topic -import com.twitter.guide.scribing.thriftscala.TopicModuleMetadata -import com.twitter.guide.scribing.thriftscala.TransparentGuideDetails -import com.twitter.suggests.controller_data.home_hitl_topic_annotation_prompt.thriftscala.HomeHitlTopicAnnotationPromptControllerData -import com.twitter.suggests.controller_data.home_hitl_topic_annotation_prompt.v1.thriftscala.{ - HomeHitlTopicAnnotationPromptControllerData => HomeHitlTopicAnnotationPromptControllerDataV1 -} -import com.twitter.suggests.controller_data.home_topic_annotation_prompt.thriftscala.HomeTopicAnnotationPromptControllerData -import com.twitter.suggests.controller_data.home_topic_annotation_prompt.v1.thriftscala.{ - HomeTopicAnnotationPromptControllerData => HomeTopicAnnotationPromptControllerDataV1 -} -import com.twitter.suggests.controller_data.home_topic_follow_prompt.thriftscala.HomeTopicFollowPromptControllerData -import com.twitter.suggests.controller_data.home_topic_follow_prompt.v1.thriftscala.{ - HomeTopicFollowPromptControllerData => HomeTopicFollowPromptControllerDataV1 -} -import com.twitter.suggests.controller_data.home_tweets.thriftscala.HomeTweetsControllerData -import com.twitter.suggests.controller_data.home_tweets.v1.thriftscala.{ - HomeTweetsControllerData => HomeTweetsControllerDataV1 -} -import com.twitter.suggests.controller_data.search_response.item_types.thriftscala.ItemTypesControllerData -import com.twitter.suggests.controller_data.search_response.thriftscala.SearchResponseControllerData -import com.twitter.suggests.controller_data.search_response.topic_follow_prompt.thriftscala.SearchTopicFollowPromptControllerData -import com.twitter.suggests.controller_data.search_response.tweet_types.thriftscala.TweetTypesControllerData -import com.twitter.suggests.controller_data.search_response.v1.thriftscala.{ - SearchResponseControllerData => SearchResponseControllerDataV1 -} -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.timelines_topic.thriftscala.TimelinesTopicControllerData -import com.twitter.suggests.controller_data.timelines_topic.v1.thriftscala.{ - TimelinesTopicControllerData => TimelinesTopicControllerDataV1 -} -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} -import com.twitter.util.Try - -object TopicIdUtils { - val DomainId: Long = 131 // Topical Domain - - def getTopicId( - item: Item, - namespace: EventNamespace - ): Option[Long] = - getTopicIdFromHomeSearch(item) - .orElse(getTopicFromGuide(item)) - .orElse(getTopicFromOnboarding(item, namespace)) - .orElse(getTopicIdFromItem(item)) - - def getTopicIdFromItem(item: Item): Option[Long] = - if (item.itemType.contains(Topic)) - item.id - else None - - def getTopicIdFromHomeSearch( - item: Item - ): Option[Long] = { - val decodedControllerData = item.suggestionDetails.flatMap(_.decodedControllerData) - decodedControllerData match { - case Some( - ControllerData.V2( - ControllerDataV2.HomeTweets( - HomeTweetsControllerData.V1(homeTweets: HomeTweetsControllerDataV1))) - ) => - homeTweets.topicId - case Some( - ControllerData.V2( - ControllerDataV2.HomeTopicFollowPrompt( - HomeTopicFollowPromptControllerData.V1( - homeTopicFollowPrompt: HomeTopicFollowPromptControllerDataV1))) - ) => - homeTopicFollowPrompt.topicId - case Some( - ControllerData.V2( - ControllerDataV2.TimelinesTopic( - TimelinesTopicControllerData.V1( - timelinesTopic: TimelinesTopicControllerDataV1 - ))) - ) => - Some(timelinesTopic.topicId) - case Some( - ControllerData.V2( - ControllerDataV2.SearchResponse( - SearchResponseControllerData.V1(s: SearchResponseControllerDataV1))) - ) => - s.itemTypesControllerData match { - case Some( - ItemTypesControllerData.TopicFollowControllerData( - topicFollowControllerData: SearchTopicFollowPromptControllerData)) => - topicFollowControllerData.topicId - case Some( - ItemTypesControllerData.TweetTypesControllerData( - tweetTypesControllerData: TweetTypesControllerData)) => - tweetTypesControllerData.topicId - case _ => None - } - case Some( - ControllerData.V2( - ControllerDataV2.HomeTopicAnnotationPrompt( - HomeTopicAnnotationPromptControllerData.V1( - homeTopicAnnotationPrompt: HomeTopicAnnotationPromptControllerDataV1 - ))) - ) => - Some(homeTopicAnnotationPrompt.topicId) - case Some( - ControllerData.V2( - ControllerDataV2.HomeHitlTopicAnnotationPrompt( - HomeHitlTopicAnnotationPromptControllerData.V1( - homeHitlTopicAnnotationPrompt: HomeHitlTopicAnnotationPromptControllerDataV1 - ))) - ) => - Some(homeHitlTopicAnnotationPrompt.topicId) - - case _ => None - } - } - - def getTopicFromOnboarding( - item: Item, - namespace: EventNamespace - ): Option[Long] = - if (namespace.page.contains("onboarding") && - (namespace.section.exists(_.contains("topic")) || - namespace.component.exists(_.contains("topic")) || - namespace.element.exists(_.contains("topic")))) { - item.description.flatMap { description => - // description: "id=123,main=xyz,row=1" - val tokens = description.split(",").headOption.map(_.split("=")) - tokens match { - case Some(Array("id", token, _*)) => Try(token.toLong).toOption - case _ => None - } - } - } else None - - def getTopicFromGuide( - item: Item - ): Option[Long] = - item.guideItemDetails.flatMap { - _.transparentGuideDetails match { - case Some(TransparentGuideDetails.TopicMetadata(topicMetadata)) => - topicMetadata match { - case TopicModuleMetadata.TttInterest(_) => - None - case TopicModuleMetadata.SemanticCoreInterest(semanticCoreInterest) => - if (semanticCoreInterest.domainId == DomainId.toString) - Try(semanticCoreInterest.entityId.toLong).toOption - else None - case TopicModuleMetadata.SimClusterInterest(_) => - None - case TopicModuleMetadata.UnknownUnionField(_) => None - } - case _ => None - } - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/VideoClientEventUtils.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/VideoClientEventUtils.docx new file mode 100644 index 000000000..7f71e9328 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/VideoClientEventUtils.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/VideoClientEventUtils.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/VideoClientEventUtils.scala deleted file mode 100644 index 842c501be..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event/VideoClientEventUtils.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.unified_user_actions.adapter.client_event - -import com.twitter.clientapp.thriftscala.AmplifyDetails -import com.twitter.clientapp.thriftscala.MediaDetails -import com.twitter.unified_user_actions.thriftscala.TweetVideoWatch -import com.twitter.unified_user_actions.thriftscala.TweetActionInfo -import com.twitter.video.analytics.thriftscala.MediaIdentifier - -object VideoClientEventUtils { - - /** - * For Tweets with multiple videos, find the id of the video that generated the client-event - */ - def videoIdFromMediaIdentifier(mediaIdentifier: MediaIdentifier): Option[String] = - mediaIdentifier match { - case MediaIdentifier.MediaPlatformIdentifier(mediaPlatformIdentifier) => - mediaPlatformIdentifier.mediaId.map(_.toString) - case _ => None - } - - /** - * Given: - * 1. the id of the video (`mediaId`) - * 2. details about all the media items in the Tweet (`mediaItems`), - * iterate over the `mediaItems` to lookup the metadata about the video with id `mediaId`. - */ - def getVideoMetadata( - mediaId: String, - mediaItems: Seq[MediaDetails], - amplifyDetails: Option[AmplifyDetails] - ): Option[TweetActionInfo] = { - mediaItems.collectFirst { - case media if media.contentId.contains(mediaId) => - TweetActionInfo.TweetVideoWatch( - TweetVideoWatch( - mediaType = media.mediaType, - isMonetizable = media.dynamicAds, - videoType = amplifyDetails.flatMap(_.videoType) - )) - } - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/AdapterUtils.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/AdapterUtils.docx new file mode 100644 index 000000000..94833f5c0 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/AdapterUtils.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/AdapterUtils.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/AdapterUtils.scala deleted file mode 100644 index 3d5b85002..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/AdapterUtils.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.unified_user_actions.adapter.common - -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.util.Time - -object AdapterUtils { - def currentTimestampMs: Long = Time.now.inMilliseconds - def getTimestampMsFromTweetId(tweetId: Long): Long = SnowflakeId.unixTimeMillisFromId(tweetId) - - // For now just make sure both language code and country code are in upper cases for consistency - // For language code, there are mixed lower and upper cases - // For country code, there are mixed lower and upper cases - def normalizeLanguageCode(inputLanguageCode: String): String = inputLanguageCode.toUpperCase - def normalizeCountryCode(inputCountryCode: String): String = inputCountryCode.toUpperCase -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/BUILD b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/BUILD deleted file mode 100644 index f5d2c526c..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - "snowflake/src/main/scala/com/twitter/snowflake/id", - "util/util-core:util-core-util", - ], -) diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/BUILD.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/BUILD.docx new file mode 100644 index 000000000..03567911b Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/BUILD b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/BUILD deleted file mode 100644 index 612b89436..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "src/thrift/com/twitter/ibis:logging-scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/BUILD.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/BUILD.docx new file mode 100644 index 000000000..86bc18bc7 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventAdapter.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventAdapter.docx new file mode 100644 index 000000000..a9d2cb966 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventAdapter.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventAdapter.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventAdapter.scala deleted file mode 100644 index c994f5c81..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventAdapter.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.unified_user_actions.adapter.email_notification_event - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.ibis.thriftscala.NotificationScribe -import com.twitter.ibis.thriftscala.NotificationScribeType -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.EmailNotificationInfo -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.ProductSurface -import com.twitter.unified_user_actions.thriftscala.ProductSurfaceInfo -import com.twitter.unified_user_actions.thriftscala.TweetInfo -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.unified_user_actions.thriftscala.UserIdentifier - -class EmailNotificationEventAdapter - extends AbstractAdapter[NotificationScribe, UnKeyed, UnifiedUserAction] { - import EmailNotificationEventAdapter._ - override def adaptOneToKeyedMany( - input: NotificationScribe, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[(UnKeyed, UnifiedUserAction)] = - adaptEvent(input).map { e => (UnKeyed, e) } -} - -object EmailNotificationEventAdapter { - - def adaptEvent(scribe: NotificationScribe): Seq[UnifiedUserAction] = { - Option(scribe).flatMap { e => - e.`type` match { - case NotificationScribeType.Click => - val tweetIdOpt = e.logBase.flatMap(EmailNotificationEventUtils.extractTweetId) - (tweetIdOpt, e.impressionId) match { - case (Some(tweetId), Some(impressionId)) => - Some( - UnifiedUserAction( - userIdentifier = UserIdentifier(userId = e.userId), - item = Item.TweetInfo(TweetInfo(actionTweetId = tweetId)), - actionType = ActionType.ClientTweetEmailClick, - eventMetadata = EmailNotificationEventUtils.extractEventMetaData(e), - productSurface = Some(ProductSurface.EmailNotification), - productSurfaceInfo = Some( - ProductSurfaceInfo.EmailNotificationInfo( - EmailNotificationInfo(notificationId = impressionId))) - ) - ) - case _ => None - } - case _ => None - } - }.toSeq - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventUtils.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventUtils.docx new file mode 100644 index 000000000..ab38a4956 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventUtils.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventUtils.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventUtils.scala deleted file mode 100644 index 85bd1999f..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event/EmailNotificationEventUtils.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.unified_user_actions.adapter.email_notification_event - -import com.twitter.ibis.thriftscala.NotificationScribe -import com.twitter.logbase.thriftscala.LogBase -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala.EventMetadata -import com.twitter.unified_user_actions.thriftscala.SourceLineage - -object EmailNotificationEventUtils { - - /* - * Extract TweetId from Logbase.page, here is a sample page below - * https://twitter.com/i/events/1580827044245544962?cn=ZmxleGlibGVfcmVjcw%3D%3D&refsrc=email - * */ - def extractTweetId(path: String): Option[Long] = { - val ptn = raw".*/([0-9]+)\\??.*".r - path match { - case ptn(tweetId) => - Some(tweetId.toLong) - case _ => - None - } - } - - def extractTweetId(logBase: LogBase): Option[Long] = logBase.page match { - case Some(path) => extractTweetId(path) - case None => None - } - - def extractEventMetaData(scribe: NotificationScribe): EventMetadata = - EventMetadata( - sourceTimestampMs = scribe.timestamp, - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.EmailNotificationEvents, - language = scribe.logBase.flatMap(_.language), - countryCode = scribe.logBase.flatMap(_.country), - clientAppId = scribe.logBase.flatMap(_.clientAppId), - ) -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/BUILD b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/BUILD deleted file mode 100644 index 6baf312d6..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "fanoutservice/thrift/src/main/thrift:thrift-scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/BUILD.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/BUILD.docx new file mode 100644 index 000000000..540a13c2a Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/FavoriteArchivalEventsAdapter.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/FavoriteArchivalEventsAdapter.docx new file mode 100644 index 000000000..f7ccb050b Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/FavoriteArchivalEventsAdapter.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/FavoriteArchivalEventsAdapter.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/FavoriteArchivalEventsAdapter.scala deleted file mode 100644 index 1121dcfe5..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events/FavoriteArchivalEventsAdapter.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.unified_user_actions.adapter.favorite_archival_events - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.timelineservice.fanout.thriftscala.FavoriteArchivalEvent -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala._ - -class FavoriteArchivalEventsAdapter - extends AbstractAdapter[FavoriteArchivalEvent, UnKeyed, UnifiedUserAction] { - - import FavoriteArchivalEventsAdapter._ - override def adaptOneToKeyedMany( - input: FavoriteArchivalEvent, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[(UnKeyed, UnifiedUserAction)] = - adaptEvent(input).map { e => (UnKeyed, e) } -} - -object FavoriteArchivalEventsAdapter { - - def adaptEvent(e: FavoriteArchivalEvent): Seq[UnifiedUserAction] = - Option(e).map { e => - UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(e.favoriterId)), - item = getItem(e), - actionType = - if (e.isArchivingAction.getOrElse(true)) ActionType.ServerTweetArchiveFavorite - else ActionType.ServerTweetUnarchiveFavorite, - eventMetadata = getEventMetadata(e) - ) - }.toSeq - - def getItem(e: FavoriteArchivalEvent): Item = - Item.TweetInfo( - TweetInfo( - // Please note that here we always use TweetId (not sourceTweetId)!!! - actionTweetId = e.tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = e.tweetUserId)), - retweetedTweetId = e.sourceTweetId - ) - ) - - def getEventMetadata(e: FavoriteArchivalEvent): EventMetadata = - EventMetadata( - sourceTimestampMs = e.timestampMs, - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.ServerFavoriteArchivalEvents, - ) -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/BUILD b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/BUILD deleted file mode 100644 index 6baf312d6..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "fanoutservice/thrift/src/main/thrift:thrift-scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/BUILD.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/BUILD.docx new file mode 100644 index 000000000..540a13c2a Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/RetweetArchivalEventsAdapter.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/RetweetArchivalEventsAdapter.docx new file mode 100644 index 000000000..018d02671 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/RetweetArchivalEventsAdapter.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/RetweetArchivalEventsAdapter.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/RetweetArchivalEventsAdapter.scala deleted file mode 100644 index 7efdd11d5..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events/RetweetArchivalEventsAdapter.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.unified_user_actions.adapter.retweet_archival_events - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.tweetypie.thriftscala.RetweetArchivalEvent -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala._ - -class RetweetArchivalEventsAdapter - extends AbstractAdapter[RetweetArchivalEvent, UnKeyed, UnifiedUserAction] { - - import RetweetArchivalEventsAdapter._ - override def adaptOneToKeyedMany( - input: RetweetArchivalEvent, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[(UnKeyed, UnifiedUserAction)] = - adaptEvent(input).map { e => (UnKeyed, e) } -} - -object RetweetArchivalEventsAdapter { - - def adaptEvent(e: RetweetArchivalEvent): Seq[UnifiedUserAction] = - Option(e).map { e => - UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(e.retweetUserId)), - item = getItem(e), - actionType = - if (e.isArchivingAction.getOrElse(true)) ActionType.ServerTweetArchiveRetweet - else ActionType.ServerTweetUnarchiveRetweet, - eventMetadata = getEventMetadata(e) - ) - }.toSeq - - def getItem(e: RetweetArchivalEvent): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = e.srcTweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(e.srcTweetUserId))), - retweetingTweetId = Some(e.retweetId) - ) - ) - - def getEventMetadata(e: RetweetArchivalEvent): EventMetadata = - EventMetadata( - sourceTimestampMs = e.timestampMs, - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.ServerRetweetArchivalEvents, - ) -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BUILD b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BUILD deleted file mode 100644 index c23748f7b..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BUILD.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BUILD.docx new file mode 100644 index 000000000..cbd2cfe6e Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseReportSocialGraphWriteEvent.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseReportSocialGraphWriteEvent.docx new file mode 100644 index 000000000..2328e771c Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseReportSocialGraphWriteEvent.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseReportSocialGraphWriteEvent.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseReportSocialGraphWriteEvent.scala deleted file mode 100644 index c9626e7d8..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseReportSocialGraphWriteEvent.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.unified_user_actions.adapter.social_graph_event - -import com.twitter.socialgraph.thriftscala.Action -import com.twitter.socialgraph.thriftscala.SrcTargetRequest -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.ProfileActionInfo -import com.twitter.unified_user_actions.thriftscala.ProfileInfo -import com.twitter.unified_user_actions.thriftscala.ServerProfileReport - -abstract class BaseReportSocialGraphWriteEvent[T] extends BaseSocialGraphWriteEvent[T] { - def socialGraphAction: Action - - override def getSocialGraphItem(socialGraphSrcTargetRequest: SrcTargetRequest): Item = { - Item.ProfileInfo( - ProfileInfo( - actionProfileId = socialGraphSrcTargetRequest.target, - profileActionInfo = Some( - ProfileActionInfo.ServerProfileReport( - ServerProfileReport(reportType = socialGraphAction) - )) - ) - ) - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseSocialGraphWriteEvent.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseSocialGraphWriteEvent.docx new file mode 100644 index 000000000..5eac10f48 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseSocialGraphWriteEvent.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseSocialGraphWriteEvent.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseSocialGraphWriteEvent.scala deleted file mode 100644 index 91ca9581e..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/BaseSocialGraphWriteEvent.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.unified_user_actions.adapter.social_graph_event - -import com.twitter.socialgraph.thriftscala.LogEventContext -import com.twitter.socialgraph.thriftscala.SrcTargetRequest -import com.twitter.socialgraph.thriftscala.WriteEvent -import com.twitter.socialgraph.thriftscala.WriteRequestResult -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.EventMetadata -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.ProfileInfo -import com.twitter.unified_user_actions.thriftscala.SourceLineage -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.unified_user_actions.thriftscala.UserIdentifier - -trait BaseSocialGraphWriteEvent[T] { - def uuaActionType: ActionType - - def getSrcTargetRequest( - e: WriteEvent - ): Seq[SrcTargetRequest] = getSubType(e) match { - case Some(subType: Seq[T]) => - getWriteRequestResultFromSubType(subType).collect { - case r if r.validationError.isEmpty => r.request - } - case _ => Nil - } - - def getSubType(e: WriteEvent): Option[Seq[T]] - def getWriteRequestResultFromSubType(subType: Seq[T]): Seq[WriteRequestResult] - - def toUnifiedUserAction( - writeEvent: WriteEvent, - uuaAction: BaseSocialGraphWriteEvent[_] - ): Seq[UnifiedUserAction] = - uuaAction.getSrcTargetRequest(writeEvent).map { srcTargetRequest => - UnifiedUserAction( - userIdentifier = UserIdentifier(userId = writeEvent.context.loggedInUserId), - item = getSocialGraphItem(srcTargetRequest), - actionType = uuaAction.uuaActionType, - eventMetadata = getEventMetadata(writeEvent.context) - ) - } - - def getSocialGraphItem(socialGraphSrcTargetRequest: SrcTargetRequest): Item = { - Item.ProfileInfo( - ProfileInfo( - actionProfileId = socialGraphSrcTargetRequest.target - ) - ) - } - - def getEventMetadata(context: LogEventContext): EventMetadata = { - EventMetadata( - sourceTimestampMs = context.timestamp, - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.ServerSocialGraphEvents, - ) - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphAdapter.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphAdapter.docx new file mode 100644 index 000000000..d3b2536ab Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphAdapter.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphAdapter.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphAdapter.scala deleted file mode 100644 index a4eee6be3..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphAdapter.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.unified_user_actions.adapter.social_graph_event - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.socialgraph.thriftscala.Action._ -import com.twitter.socialgraph.thriftscala.WriteEvent -import com.twitter.socialgraph.thriftscala.{Action => SocialGraphAction} -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.adapter.social_graph_event.SocialGraphEngagement._ -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction - -class SocialGraphAdapter extends AbstractAdapter[WriteEvent, UnKeyed, UnifiedUserAction] { - - import SocialGraphAdapter._ - - override def adaptOneToKeyedMany( - input: WriteEvent, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[(UnKeyed, UnifiedUserAction)] = - adaptEvent(input).map { e => (UnKeyed, e) } -} - -object SocialGraphAdapter { - - def adaptEvent(writeEvent: WriteEvent): Seq[UnifiedUserAction] = - Option(writeEvent).flatMap { e => - socialGraphWriteEventTypeToUuaEngagementType.get(e.action) - } match { - case Some(uuaAction) => uuaAction.toUnifiedUserAction(writeEvent, uuaAction) - case None => Nil - } - - private val socialGraphWriteEventTypeToUuaEngagementType: Map[ - SocialGraphAction, - BaseSocialGraphWriteEvent[_] - ] = - Map[SocialGraphAction, BaseSocialGraphWriteEvent[_]]( - Follow -> ProfileFollow, - Unfollow -> ProfileUnfollow, - Block -> ProfileBlock, - Unblock -> ProfileUnblock, - Mute -> ProfileMute, - Unmute -> ProfileUnmute, - ReportAsSpam -> ProfileReportAsSpam, - ReportAsAbuse -> ProfileReportAsAbuse - ) -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphEngagement.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphEngagement.docx new file mode 100644 index 000000000..891891570 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphEngagement.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphEngagement.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphEngagement.scala deleted file mode 100644 index 952531c9f..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event/SocialGraphEngagement.scala +++ /dev/null @@ -1,157 +0,0 @@ -package com.twitter.unified_user_actions.adapter.social_graph_event - -import com.twitter.socialgraph.thriftscala.Action -import com.twitter.socialgraph.thriftscala.BlockGraphEvent -import com.twitter.socialgraph.thriftscala.FollowGraphEvent -import com.twitter.socialgraph.thriftscala.MuteGraphEvent -import com.twitter.socialgraph.thriftscala.ReportAsAbuseGraphEvent -import com.twitter.socialgraph.thriftscala.ReportAsSpamGraphEvent -import com.twitter.socialgraph.thriftscala.WriteEvent -import com.twitter.socialgraph.thriftscala.WriteRequestResult -import com.twitter.unified_user_actions.thriftscala.{ActionType => UuaActionType} - -object SocialGraphEngagement { - - /** - * This is "Follow" event to indicate user1 follows user2 captured in ServerProfileFollow - */ - object ProfileFollow extends BaseSocialGraphWriteEvent[FollowGraphEvent] { - override def uuaActionType: UuaActionType = UuaActionType.ServerProfileFollow - - override def getSubType( - e: WriteEvent - ): Option[Seq[FollowGraphEvent]] = - e.follow - - override def getWriteRequestResultFromSubType( - e: Seq[FollowGraphEvent] - ): Seq[WriteRequestResult] = { - // Remove all redundant operations (FollowGraphEvent.redundantOperation == Some(true)) - e.collect { - case fe if !fe.redundantOperation.getOrElse(false) => fe.result - } - } - } - - /** - * This is "Unfollow" event to indicate user1 unfollows user2 captured in ServerProfileUnfollow - * - * Both Unfollow and Follow use the struct FollowGraphEvent, but are treated in its individual case - * class. - */ - object ProfileUnfollow extends BaseSocialGraphWriteEvent[FollowGraphEvent] { - override def uuaActionType: UuaActionType = UuaActionType.ServerProfileUnfollow - - override def getSubType( - e: WriteEvent - ): Option[Seq[FollowGraphEvent]] = - e.follow - - override def getWriteRequestResultFromSubType( - e: Seq[FollowGraphEvent] - ): Seq[WriteRequestResult] = - e.collect { - case fe if !fe.redundantOperation.getOrElse(false) => fe.result - } - } - - /** - * This is "Block" event to indicate user1 blocks user2 captured in ServerProfileBlock - */ - object ProfileBlock extends BaseSocialGraphWriteEvent[BlockGraphEvent] { - override def uuaActionType: UuaActionType = UuaActionType.ServerProfileBlock - - override def getSubType( - e: WriteEvent - ): Option[Seq[BlockGraphEvent]] = - e.block - - override def getWriteRequestResultFromSubType( - e: Seq[BlockGraphEvent] - ): Seq[WriteRequestResult] = - e.map(_.result) - } - - /** - * This is "Unblock" event to indicate user1 unblocks user2 captured in ServerProfileUnblock - * - * Both Unblock and Block use struct BlockGraphEvent, but are treated in its individual case - * class. - */ - object ProfileUnblock extends BaseSocialGraphWriteEvent[BlockGraphEvent] { - override def uuaActionType: UuaActionType = UuaActionType.ServerProfileUnblock - - override def getSubType( - e: WriteEvent - ): Option[Seq[BlockGraphEvent]] = - e.block - - override def getWriteRequestResultFromSubType( - e: Seq[BlockGraphEvent] - ): Seq[WriteRequestResult] = - e.map(_.result) - } - - /** - * This is "Mute" event to indicate user1 mutes user2 captured in ServerProfileMute - */ - object ProfileMute extends BaseSocialGraphWriteEvent[MuteGraphEvent] { - override def uuaActionType: UuaActionType = UuaActionType.ServerProfileMute - - override def getSubType( - e: WriteEvent - ): Option[Seq[MuteGraphEvent]] = - e.mute - - override def getWriteRequestResultFromSubType(e: Seq[MuteGraphEvent]): Seq[WriteRequestResult] = - e.map(_.result) - } - - /** - * This is "Unmute" event to indicate user1 unmutes user2 captured in ServerProfileUnmute - * - * Both Unmute and Mute use the struct MuteGraphEvent, but are treated in its individual case - * class. - */ - object ProfileUnmute extends BaseSocialGraphWriteEvent[MuteGraphEvent] { - override def uuaActionType: UuaActionType = UuaActionType.ServerProfileUnmute - - override def getSubType( - e: WriteEvent - ): Option[Seq[MuteGraphEvent]] = - e.mute - - override def getWriteRequestResultFromSubType(e: Seq[MuteGraphEvent]): Seq[WriteRequestResult] = - e.map(_.result) - } - - object ProfileReportAsSpam extends BaseReportSocialGraphWriteEvent[ReportAsSpamGraphEvent] { - override def uuaActionType: UuaActionType = UuaActionType.ServerProfileReport - override def socialGraphAction: Action = Action.ReportAsSpam - - override def getSubType( - e: WriteEvent - ): Option[Seq[ReportAsSpamGraphEvent]] = - e.reportAsSpam - - override def getWriteRequestResultFromSubType( - e: Seq[ReportAsSpamGraphEvent] - ): Seq[WriteRequestResult] = - e.map(_.result) - } - - object ProfileReportAsAbuse extends BaseReportSocialGraphWriteEvent[ReportAsAbuseGraphEvent] { - override def uuaActionType: UuaActionType = UuaActionType.ServerProfileReport - override def socialGraphAction: Action = Action.ReportAsAbuse - - override def getSubType( - e: WriteEvent - ): Option[Seq[ReportAsAbuseGraphEvent]] = - e.reportAsAbuse - - override def getWriteRequestResultFromSubType( - e: Seq[ReportAsAbuseGraphEvent] - ): Seq[WriteRequestResult] = - e.map(_.result) - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/BUILD b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/BUILD deleted file mode 100644 index 0281de0ef..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/BUILD.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/BUILD.docx new file mode 100644 index 000000000..d965fbbac Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/TlsFavsAdapter.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/TlsFavsAdapter.docx new file mode 100644 index 000000000..3d04e0cff Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/TlsFavsAdapter.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/TlsFavsAdapter.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/TlsFavsAdapter.scala deleted file mode 100644 index d76157949..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event/TlsFavsAdapter.scala +++ /dev/null @@ -1,109 +0,0 @@ -package com.twitter.unified_user_actions.adapter.tls_favs_event - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.timelineservice.thriftscala._ -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala._ - -class TlsFavsAdapter - extends AbstractAdapter[ContextualizedFavoriteEvent, UnKeyed, UnifiedUserAction] { - - import TlsFavsAdapter._ - - override def adaptOneToKeyedMany( - input: ContextualizedFavoriteEvent, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[(UnKeyed, UnifiedUserAction)] = - adaptEvent(input).map { e => (UnKeyed, e) } -} - -object TlsFavsAdapter { - - def adaptEvent(e: ContextualizedFavoriteEvent): Seq[UnifiedUserAction] = - Option(e).flatMap { e => - e.event match { - case FavoriteEventUnion.Favorite(favoriteEvent) => - Some( - UnifiedUserAction( - userIdentifier = getUserIdentifier(Left(favoriteEvent)), - item = getFavItem(favoriteEvent), - actionType = ActionType.ServerTweetFav, - eventMetadata = getEventMetadata(Left(favoriteEvent), e.context), - productSurface = None, - productSurfaceInfo = None - )) - - case FavoriteEventUnion.Unfavorite(unfavoriteEvent) => - Some( - UnifiedUserAction( - userIdentifier = getUserIdentifier(Right(unfavoriteEvent)), - item = getUnfavItem(unfavoriteEvent), - actionType = ActionType.ServerTweetUnfav, - eventMetadata = getEventMetadata(Right(unfavoriteEvent), e.context), - productSurface = None, - productSurfaceInfo = None - )) - - case _ => None - } - }.toSeq - - def getFavItem(favoriteEvent: FavoriteEvent): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = favoriteEvent.tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(favoriteEvent.tweetUserId))), - retweetingTweetId = favoriteEvent.retweetId - ) - ) - - def getUnfavItem(unfavoriteEvent: UnfavoriteEvent): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = unfavoriteEvent.tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(unfavoriteEvent.tweetUserId))), - retweetingTweetId = unfavoriteEvent.retweetId - ) - ) - - def getEventMetadata( - event: Either[FavoriteEvent, UnfavoriteEvent], - context: LogEventContext - ): EventMetadata = { - val sourceTimestampMs = event match { - case Left(favoriteEvent) => favoriteEvent.eventTimeMs - case Right(unfavoriteEvent) => unfavoriteEvent.eventTimeMs - } - // Client UI language, see more at http://go/languagepriority. The format should be ISO 639-1. - val language = event match { - case Left(favoriteEvent) => favoriteEvent.viewerContext.flatMap(_.requestLanguageCode) - case Right(unfavoriteEvent) => unfavoriteEvent.viewerContext.flatMap(_.requestLanguageCode) - } - // From the request (user’s current location), - // see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/thrift/com/twitter/context/viewer.thrift?L54 - // The format should be ISO_3166-1_alpha-2. - val countryCode = event match { - case Left(favoriteEvent) => favoriteEvent.viewerContext.flatMap(_.requestCountryCode) - case Right(unfavoriteEvent) => unfavoriteEvent.viewerContext.flatMap(_.requestCountryCode) - } - EventMetadata( - sourceTimestampMs = sourceTimestampMs, - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.ServerTlsFavs, - language = language.map(AdapterUtils.normalizeLanguageCode), - countryCode = countryCode.map(AdapterUtils.normalizeCountryCode), - traceId = Some(context.traceId), - clientAppId = context.clientApplicationId, - ) - } - - // Get id of the user that took the action - def getUserIdentifier(event: Either[FavoriteEvent, UnfavoriteEvent]): UserIdentifier = - event match { - case Left(favoriteEvent) => UserIdentifier(userId = Some(favoriteEvent.userId)) - case Right(unfavoriteEvent) => UserIdentifier(userId = Some(unfavoriteEvent.userId)) - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BUILD b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BUILD deleted file mode 100644 index 9a526255f..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/tweetypie:events-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BUILD.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BUILD.docx new file mode 100644 index 000000000..c123e1318 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEvent.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEvent.docx new file mode 100644 index 000000000..311c09bc9 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEvent.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEvent.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEvent.scala deleted file mode 100644 index 2e33d2970..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEvent.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.unified_user_actions.adapter.tweetypie_event - -import com.twitter.tweetypie.thriftscala.TweetEventFlags -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.EventMetadata -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.unified_user_actions.thriftscala.UserIdentifier - -/** - * Base class for Tweetypie Tweet Event. - * Extends this class if you need to implement the parser for a new Tweetypie Tweet Event Type. - * @see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/thrift/com/twitter/tweetypie/tweet_events.thrift?L225 - */ -trait BaseTweetypieTweetEvent[T] { - - /** - * Returns an Optional UnifiedUserAction from the event. - */ - def getUnifiedUserAction(event: T, flags: TweetEventFlags): Option[UnifiedUserAction] - - /** - * Returns UnifiedUserAction.ActionType for each type of event. - */ - protected def actionType: ActionType - - /** - * Output type of the predicate. Could be an input of getItem. - */ - type ExtractedEvent - - /** - * Returns Some(ExtractedEvent) if the event is valid and None otherwise. - */ - protected def extract(event: T): Option[ExtractedEvent] - - /** - * Get the UnifiedUserAction.Item from the event. - */ - protected def getItem(extractedEvent: ExtractedEvent, event: T): Item - - /** - * Get the UnifiedUserAction.UserIdentifier from the event. - */ - protected def getUserIdentifier(event: T): UserIdentifier - - /** - * Get UnifiedUserAction.EventMetadata from the event. - */ - protected def getEventMetadata(event: T, flags: TweetEventFlags): EventMetadata -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventCreate.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventCreate.docx new file mode 100644 index 000000000..ba8e68595 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventCreate.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventCreate.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventCreate.scala deleted file mode 100644 index 5ede2f388..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventCreate.scala +++ /dev/null @@ -1,200 +0,0 @@ -package com.twitter.unified_user_actions.adapter.tweetypie_event - -import com.twitter.tweetypie.thriftscala.QuotedTweet -import com.twitter.tweetypie.thriftscala.Share -import com.twitter.tweetypie.thriftscala.TweetCreateEvent -import com.twitter.tweetypie.thriftscala.TweetEventFlags -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.AuthorInfo -import com.twitter.unified_user_actions.thriftscala.EventMetadata -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.SourceLineage -import com.twitter.unified_user_actions.thriftscala.TweetInfo -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.unified_user_actions.thriftscala.UserIdentifier - -/** - * Base class for Tweetypie TweetCreateEvent including Quote, Reply, Retweet, and Create. - */ -trait BaseTweetypieTweetEventCreate extends BaseTweetypieTweetEvent[TweetCreateEvent] { - type ExtractedEvent - protected def actionType: ActionType - - /** - * This is the country code where actionTweetId is sent from. For the definitions, - * check https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/thrift/com/twitter/tweetypie/tweet.thrift?L1001. - * - * UUA sets this to be consistent with IESource to meet existing use requirement. - * - * For ServerTweetReply/Retweet/Quote, the geo-tagging country code is not available in TweetCreatEvent. - * Thus, user signup country is picked to meet a customer use case. - * - * The definition here conflicts with the intention of UUA to log the request country code - * rather than the signup / geo-tagging country. - * - */ - protected def getCountryCode(tce: TweetCreateEvent): Option[String] = { - tce.tweet.place match { - case Some(p) => p.countryCode - case _ => tce.user.safety.flatMap(_.signupCountryCode) - } - } - - protected def getItem( - extractedEvent: ExtractedEvent, - tweetCreateEvent: TweetCreateEvent - ): Item - protected def extract(tweetCreateEvent: TweetCreateEvent): Option[ExtractedEvent] - - def getUnifiedUserAction( - tweetCreateEvent: TweetCreateEvent, - tweetEventFlags: TweetEventFlags - ): Option[UnifiedUserAction] = { - extract(tweetCreateEvent).map { extractedEvent => - UnifiedUserAction( - userIdentifier = getUserIdentifier(tweetCreateEvent), - item = getItem(extractedEvent, tweetCreateEvent), - actionType = actionType, - eventMetadata = getEventMetadata(tweetCreateEvent, tweetEventFlags), - productSurface = None, - productSurfaceInfo = None - ) - } - } - - protected def getUserIdentifier(tweetCreateEvent: TweetCreateEvent): UserIdentifier = - UserIdentifier(userId = Some(tweetCreateEvent.user.id)) - - protected def getEventMetadata( - tweetCreateEvent: TweetCreateEvent, - flags: TweetEventFlags - ): EventMetadata = - EventMetadata( - sourceTimestampMs = flags.timestampMs, - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.ServerTweetypieEvents, - traceId = None, // Currently traceId is not stored in TweetCreateEvent - // UUA sets this to None since there is no request level language info. - language = None, - countryCode = getCountryCode(tweetCreateEvent), - clientAppId = tweetCreateEvent.tweet.deviceSource.flatMap(_.clientAppId), - clientVersion = None // Currently clientVersion is not stored in TweetCreateEvent - ) -} - -/** - * Get UnifiedUserAction from a tweet Create. - * Note the Create is generated when the tweet is not a Quote/Retweet/Reply. - */ -object TweetypieCreateEvent extends BaseTweetypieTweetEventCreate { - type ExtractedEvent = Long - override protected val actionType: ActionType = ActionType.ServerTweetCreate - override protected def extract(tweetCreateEvent: TweetCreateEvent): Option[Long] = - Option(tweetCreateEvent.tweet.id) - - protected def getItem( - tweetId: Long, - tweetCreateEvent: TweetCreateEvent - ): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(tweetCreateEvent.user.id))) - )) -} - -/** - * Get UnifiedUserAction from a Reply. - * Note the Reply is generated when someone is replying to a tweet. - */ -object TweetypieReplyEvent extends BaseTweetypieTweetEventCreate { - case class PredicateOutput(tweetId: Long, userId: Long) - override type ExtractedEvent = PredicateOutput - override protected val actionType: ActionType = ActionType.ServerTweetReply - override protected def extract(tweetCreateEvent: TweetCreateEvent): Option[PredicateOutput] = - tweetCreateEvent.tweet.coreData - .flatMap(_.reply).flatMap(r => - r.inReplyToStatusId.map(tweetId => PredicateOutput(tweetId, r.inReplyToUserId))) - - override protected def getItem( - repliedTweet: PredicateOutput, - tweetCreateEvent: TweetCreateEvent - ): Item = { - Item.TweetInfo( - TweetInfo( - actionTweetId = repliedTweet.tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(repliedTweet.userId))), - replyingTweetId = Some(tweetCreateEvent.tweet.id) - ) - ) - } -} - -/** - * Get UnifiedUserAction from a Quote. - * Note the Quote is generated when someone is quoting (retweeting with comment) a tweet. - */ -object TweetypieQuoteEvent extends BaseTweetypieTweetEventCreate { - override protected val actionType: ActionType = ActionType.ServerTweetQuote - type ExtractedEvent = QuotedTweet - override protected def extract(tweetCreateEvent: TweetCreateEvent): Option[QuotedTweet] = - tweetCreateEvent.tweet.quotedTweet - - override protected def getItem( - quotedTweet: QuotedTweet, - tweetCreateEvent: TweetCreateEvent - ): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = quotedTweet.tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(quotedTweet.userId))), - quotingTweetId = Some(tweetCreateEvent.tweet.id) - ) - ) -} - -/** - * Get UnifiedUserAction from a Retweet. - * Note the Retweet is generated when someone is retweeting (without comment) a tweet. - */ -object TweetypieRetweetEvent extends BaseTweetypieTweetEventCreate { - override type ExtractedEvent = Share - override protected val actionType: ActionType = ActionType.ServerTweetRetweet - override protected def extract(tweetCreateEvent: TweetCreateEvent): Option[Share] = - tweetCreateEvent.tweet.coreData.flatMap(_.share) - - override protected def getItem(share: Share, tweetCreateEvent: TweetCreateEvent): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = share.sourceStatusId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(share.sourceUserId))), - retweetingTweetId = Some(tweetCreateEvent.tweet.id) - ) - ) -} - -/** - * Get UnifiedUserAction from a TweetEdit. - * Note the Edit is generated when someone is editing their quote or default tweet. The edit will - * generate a new Tweet. - */ -object TweetypieEditEvent extends BaseTweetypieTweetEventCreate { - override type ExtractedEvent = Long - override protected def actionType: ActionType = ActionType.ServerTweetEdit - override protected def extract(tweetCreateEvent: TweetCreateEvent): Option[Long] = - TweetypieEventUtils.editedTweetIdFromTweet(tweetCreateEvent.tweet) - - override protected def getItem( - editedTweetId: Long, - tweetCreateEvent: TweetCreateEvent - ): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = tweetCreateEvent.tweet.id, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(tweetCreateEvent.user.id))), - editedTweetId = Some(editedTweetId), - quotedTweetId = tweetCreateEvent.tweet.quotedTweet.map(_.tweetId) - ) - ) -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventDelete.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventDelete.docx new file mode 100644 index 000000000..14ffdb6c4 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventDelete.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventDelete.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventDelete.scala deleted file mode 100644 index 140c851ee..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/BaseTweetypieTweetEventDelete.scala +++ /dev/null @@ -1,146 +0,0 @@ -package com.twitter.unified_user_actions.adapter.tweetypie_event - -import com.twitter.tweetypie.thriftscala.QuotedTweet -import com.twitter.tweetypie.thriftscala.Share -import com.twitter.tweetypie.thriftscala.TweetDeleteEvent -import com.twitter.tweetypie.thriftscala.TweetEventFlags -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.AuthorInfo -import com.twitter.unified_user_actions.thriftscala.EventMetadata -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.SourceLineage -import com.twitter.unified_user_actions.thriftscala.TweetInfo -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.unified_user_actions.thriftscala.UserIdentifier - -trait BaseTweetypieTweetEventDelete extends BaseTweetypieTweetEvent[TweetDeleteEvent] { - type ExtractedEvent - protected def actionType: ActionType - - def getUnifiedUserAction( - tweetDeleteEvent: TweetDeleteEvent, - tweetEventFlags: TweetEventFlags - ): Option[UnifiedUserAction] = - extract(tweetDeleteEvent).map { extractedEvent => - UnifiedUserAction( - userIdentifier = getUserIdentifier(tweetDeleteEvent), - item = getItem(extractedEvent, tweetDeleteEvent), - actionType = actionType, - eventMetadata = getEventMetadata(tweetDeleteEvent, tweetEventFlags) - ) - } - - protected def extract(tweetDeleteEvent: TweetDeleteEvent): Option[ExtractedEvent] - - protected def getItem(extractedEvent: ExtractedEvent, tweetDeleteEvent: TweetDeleteEvent): Item - - protected def getUserIdentifier(tweetDeleteEvent: TweetDeleteEvent): UserIdentifier = - UserIdentifier(userId = tweetDeleteEvent.user.map(_.id)) - - protected def getEventMetadata( - tweetDeleteEvent: TweetDeleteEvent, - flags: TweetEventFlags - ): EventMetadata = - EventMetadata( - sourceTimestampMs = flags.timestampMs, - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.ServerTweetypieEvents, - traceId = None, // Currently traceId is not stored in TweetDeleteEvent. - // UUA sets this to None since there is no request level language info. - language = None, - // UUA sets this to be consistent with IESource. For the definition, - // see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/thrift/com/twitter/tweetypie/tweet.thrift?L1001. - // The definition here conflicts with the intention of UUA to log the request country code - // rather than the signup / geo-tagging country. - countryCode = tweetDeleteEvent.tweet.place.flatMap(_.countryCode), - /* clientApplicationId is user's app id if the delete is initiated by a user, - * or auditor's app id if the delete is initiated by an auditor */ - clientAppId = tweetDeleteEvent.audit.flatMap(_.clientApplicationId), - clientVersion = None // Currently clientVersion is not stored in TweetDeleteEvent. - ) -} - -object TweetypieDeleteEvent extends BaseTweetypieTweetEventDelete { - type ExtractedEvent = Long - override protected val actionType: ActionType = ActionType.ServerTweetDelete - - override protected def extract(tweetDeleteEvent: TweetDeleteEvent): Option[Long] = Some( - tweetDeleteEvent.tweet.id) - - protected def getItem( - tweetId: Long, - tweetDeleteEvent: TweetDeleteEvent - ): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = tweetId, - actionTweetAuthorInfo = - Some(AuthorInfo(authorId = tweetDeleteEvent.tweet.coreData.map(_.userId))) - )) -} - -object TweetypieUnretweetEvent extends BaseTweetypieTweetEventDelete { - override protected val actionType: ActionType = ActionType.ServerTweetUnretweet - - override type ExtractedEvent = Share - - override protected def extract(tweetDeleteEvent: TweetDeleteEvent): Option[Share] = - tweetDeleteEvent.tweet.coreData.flatMap(_.share) - - override protected def getItem(share: Share, tweetDeleteEvent: TweetDeleteEvent): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = share.sourceStatusId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(share.sourceUserId))), - retweetingTweetId = Some(tweetDeleteEvent.tweet.id) - ) - ) -} - -object TweetypieUnreplyEvent extends BaseTweetypieTweetEventDelete { - case class PredicateOutput(tweetId: Long, userId: Long) - - override type ExtractedEvent = PredicateOutput - - override protected val actionType: ActionType = ActionType.ServerTweetUnreply - - override protected def extract(tweetDeleteEvent: TweetDeleteEvent): Option[PredicateOutput] = - tweetDeleteEvent.tweet.coreData - .flatMap(_.reply).flatMap(r => - r.inReplyToStatusId.map(tweetId => PredicateOutput(tweetId, r.inReplyToUserId))) - - override protected def getItem( - repliedTweet: PredicateOutput, - tweetDeleteEvent: TweetDeleteEvent - ): Item = { - Item.TweetInfo( - TweetInfo( - actionTweetId = repliedTweet.tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(repliedTweet.userId))), - replyingTweetId = Some(tweetDeleteEvent.tweet.id) - ) - ) - } -} - -object TweetypieUnquoteEvent extends BaseTweetypieTweetEventDelete { - override protected val actionType: ActionType = ActionType.ServerTweetUnquote - - type ExtractedEvent = QuotedTweet - - override protected def extract(tweetDeleteEvent: TweetDeleteEvent): Option[QuotedTweet] = - tweetDeleteEvent.tweet.quotedTweet - - override protected def getItem( - quotedTweet: QuotedTweet, - tweetDeleteEvent: TweetDeleteEvent - ): Item = - Item.TweetInfo( - TweetInfo( - actionTweetId = quotedTweet.tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(quotedTweet.userId))), - quotingTweetId = Some(tweetDeleteEvent.tweet.id) - ) - ) -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventAdapter.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventAdapter.docx new file mode 100644 index 000000000..51d2fdee2 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventAdapter.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventAdapter.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventAdapter.scala deleted file mode 100644 index 472a87ee2..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventAdapter.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.unified_user_actions.adapter.tweetypie_event - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.tweetypie.thriftscala.TweetEvent -import com.twitter.tweetypie.thriftscala.TweetEventData -import com.twitter.tweetypie.thriftscala.TweetCreateEvent -import com.twitter.tweetypie.thriftscala.TweetDeleteEvent -import com.twitter.tweetypie.thriftscala.TweetEventFlags -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction - -class TweetypieEventAdapter extends AbstractAdapter[TweetEvent, UnKeyed, UnifiedUserAction] { - import TweetypieEventAdapter._ - override def adaptOneToKeyedMany( - tweetEvent: TweetEvent, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[(UnKeyed, UnifiedUserAction)] = - adaptEvent(tweetEvent).map(e => (UnKeyed, e)) -} - -object TweetypieEventAdapter { - def adaptEvent(tweetEvent: TweetEvent): Seq[UnifiedUserAction] = { - Option(tweetEvent).flatMap { e => - e.data match { - case TweetEventData.TweetCreateEvent(tweetCreateEvent: TweetCreateEvent) => - getUUAFromTweetCreateEvent(tweetCreateEvent, e.flags) - case TweetEventData.TweetDeleteEvent(tweetDeleteEvent: TweetDeleteEvent) => - getUUAFromTweetDeleteEvent(tweetDeleteEvent, e.flags) - case _ => None - } - }.toSeq - } - - def getUUAFromTweetCreateEvent( - tweetCreateEvent: TweetCreateEvent, - tweetEventFlags: TweetEventFlags - ): Option[UnifiedUserAction] = { - val tweetTypeOpt = TweetypieEventUtils.tweetTypeFromTweet(tweetCreateEvent.tweet) - - tweetTypeOpt.flatMap { tweetType => - tweetType match { - case TweetTypeReply => - TweetypieReplyEvent.getUnifiedUserAction(tweetCreateEvent, tweetEventFlags) - case TweetTypeRetweet => - TweetypieRetweetEvent.getUnifiedUserAction(tweetCreateEvent, tweetEventFlags) - case TweetTypeQuote => - TweetypieQuoteEvent.getUnifiedUserAction(tweetCreateEvent, tweetEventFlags) - case TweetTypeDefault => - TweetypieCreateEvent.getUnifiedUserAction(tweetCreateEvent, tweetEventFlags) - case TweetTypeEdit => - TweetypieEditEvent.getUnifiedUserAction(tweetCreateEvent, tweetEventFlags) - } - } - } - - def getUUAFromTweetDeleteEvent( - tweetDeleteEvent: TweetDeleteEvent, - tweetEventFlags: TweetEventFlags - ): Option[UnifiedUserAction] = { - val tweetTypeOpt = TweetypieEventUtils.tweetTypeFromTweet(tweetDeleteEvent.tweet) - - tweetTypeOpt.flatMap { tweetType => - tweetType match { - case TweetTypeRetweet => - TweetypieUnretweetEvent.getUnifiedUserAction(tweetDeleteEvent, tweetEventFlags) - case TweetTypeReply => - TweetypieUnreplyEvent.getUnifiedUserAction(tweetDeleteEvent, tweetEventFlags) - case TweetTypeQuote => - TweetypieUnquoteEvent.getUnifiedUserAction(tweetDeleteEvent, tweetEventFlags) - case TweetTypeDefault | TweetTypeEdit => - TweetypieDeleteEvent.getUnifiedUserAction(tweetDeleteEvent, tweetEventFlags) - } - } - } - -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventUtils.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventUtils.docx new file mode 100644 index 000000000..9d486954e Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventUtils.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventUtils.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventUtils.scala deleted file mode 100644 index e3798f383..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event/TweetypieEventUtils.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.unified_user_actions.adapter.tweetypie_event - -import com.twitter.tweetypie.thriftscala.EditControl -import com.twitter.tweetypie.thriftscala.EditControlEdit -import com.twitter.tweetypie.thriftscala.Tweet - -sealed trait TweetypieTweetType -object TweetTypeDefault extends TweetypieTweetType -object TweetTypeReply extends TweetypieTweetType -object TweetTypeRetweet extends TweetypieTweetType -object TweetTypeQuote extends TweetypieTweetType -object TweetTypeEdit extends TweetypieTweetType - -object TweetypieEventUtils { - def editedTweetIdFromTweet(tweet: Tweet): Option[Long] = tweet.editControl.flatMap { - case EditControl.Edit(EditControlEdit(initialTweetId, _)) => Some(initialTweetId) - case _ => None - } - - def tweetTypeFromTweet(tweet: Tweet): Option[TweetypieTweetType] = { - val data = tweet.coreData - val inReplyingToStatusIdOpt = data.flatMap(_.reply).flatMap(_.inReplyToStatusId) - val shareOpt = data.flatMap(_.share) - val quotedTweetOpt = tweet.quotedTweet - val editedTweetIdOpt = editedTweetIdFromTweet(tweet) - - (inReplyingToStatusIdOpt, shareOpt, quotedTweetOpt, editedTweetIdOpt) match { - // Reply - case (Some(_), None, _, None) => - Some(TweetTypeReply) - // For any kind of retweet (be it retweet of quote tweet or retweet of a regular tweet) - // we only need to look at the `share` field - // https://confluence.twitter.biz/pages/viewpage.action?spaceKey=CSVC&title=TweetyPie+FAQ#TweetypieFAQ-HowdoItellifaTweetisaRetweet - case (None, Some(_), _, None) => - Some(TweetTypeRetweet) - // quote - case (None, None, Some(_), None) => - Some(TweetTypeQuote) - // create - case (None, None, None, None) => - Some(TweetTypeDefault) - // edit - case (None, None, _, Some(_)) => - Some(TweetTypeEdit) - // reply and retweet shouldn't be present at the same time - case (Some(_), Some(_), _, _) => - None - // reply and edit / retweet and edit shouldn't be present at the same time - case (Some(_), None, _, Some(_)) | (None, Some(_), _, Some(_)) => - None - } - } - -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/BUILD.bazel b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/BUILD.bazel deleted file mode 100644 index 24a0aab09..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/BUILD.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/BUILD.docx new file mode 100644 index 000000000..f323a9265 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModificationAdapter.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModificationAdapter.docx new file mode 100644 index 000000000..f358262fe Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModificationAdapter.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModificationAdapter.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModificationAdapter.scala deleted file mode 100644 index 24e111b96..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModificationAdapter.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.unified_user_actions.adapter.user_modification - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.gizmoduck.thriftscala.UserModification -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.adapter.user_modification_event.UserCreate -import com.twitter.unified_user_actions.adapter.user_modification_event.UserUpdate -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction - -class UserModificationAdapter - extends AbstractAdapter[UserModification, UnKeyed, UnifiedUserAction] { - - import UserModificationAdapter._ - - override def adaptOneToKeyedMany( - input: UserModification, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[(UnKeyed, UnifiedUserAction)] = - adaptEvent(input).map { e => (UnKeyed, e) } -} - -object UserModificationAdapter { - - def adaptEvent(input: UserModification): Seq[UnifiedUserAction] = - Option(input).toSeq.flatMap { e => - if (e.create.isDefined) { // User create - Some(UserCreate.getUUA(input)) - } else if (e.update.isDefined) { // User updates - Some(UserUpdate.getUUA(input)) - } else if (e.destroy.isDefined) { - None - } else if (e.erase.isDefined) { - None - } else { - throw new IllegalArgumentException( - "None of the possible events is defined, there must be something with the source") - } - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModifications.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModifications.docx new file mode 100644 index 000000000..ab6a85701 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModifications.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModifications.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModifications.scala deleted file mode 100644 index 50b8a822d..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event/UserModifications.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.unified_user_actions.adapter.user_modification_event - -import com.twitter.gizmoduck.thriftscala.UserModification -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.EventMetadata -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.ProfileActionInfo -import com.twitter.unified_user_actions.thriftscala.ServerUserUpdate -import com.twitter.unified_user_actions.thriftscala.ProfileInfo -import com.twitter.unified_user_actions.thriftscala.SourceLineage -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.unified_user_actions.thriftscala.UserIdentifier - -abstract class BaseUserModificationEvent(actionType: ActionType) { - - def getUUA(input: UserModification): UnifiedUserAction = { - val userIdentifier: UserIdentifier = UserIdentifier(userId = input.userId) - - UnifiedUserAction( - userIdentifier = userIdentifier, - item = getItem(input), - actionType = actionType, - eventMetadata = getEventMetadata(input), - ) - } - - protected def getItem(input: UserModification): Item = - Item.ProfileInfo( - ProfileInfo( - actionProfileId = input.userId - .getOrElse(throw new IllegalArgumentException("target user_id is missing")) - ) - ) - - protected def getEventMetadata(input: UserModification): EventMetadata = - EventMetadata( - sourceTimestampMs = input.updatedAtMsec - .getOrElse(throw new IllegalArgumentException("timestamp is required")), - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.ServerGizmoduckUserModificationEvents, - ) -} - -/** - * When there is a new user creation event in Gizmoduck - */ -object UserCreate extends BaseUserModificationEvent(ActionType.ServerUserCreate) { - override protected def getItem(input: UserModification): Item = - Item.ProfileInfo( - ProfileInfo( - actionProfileId = input.create - .map { user => - user.id - }.getOrElse(throw new IllegalArgumentException("target user_id is missing")), - name = input.create.flatMap { user => - user.profile.map(_.name) - }, - handle = input.create.flatMap { user => - user.profile.map(_.screenName) - }, - description = input.create.flatMap { user => - user.profile.map(_.description) - } - ) - ) - - override protected def getEventMetadata(input: UserModification): EventMetadata = - EventMetadata( - sourceTimestampMs = input.create - .map { user => - user.updatedAtMsec - }.getOrElse(throw new IllegalArgumentException("timestamp is required")), - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.ServerGizmoduckUserModificationEvents, - ) -} - -object UserUpdate extends BaseUserModificationEvent(ActionType.ServerUserUpdate) { - override protected def getItem(input: UserModification): Item = - Item.ProfileInfo( - ProfileInfo( - actionProfileId = - input.userId.getOrElse(throw new IllegalArgumentException("userId is required")), - profileActionInfo = Some( - ProfileActionInfo.ServerUserUpdate( - ServerUserUpdate(updates = input.update.getOrElse(Nil), success = input.success))) - ) - ) - - override protected def getEventMetadata(input: UserModification): EventMetadata = - EventMetadata( - sourceTimestampMs = input.updatedAtMsec.getOrElse(AdapterUtils.currentTimestampMs), - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = SourceLineage.ServerGizmoduckUserModificationEvents, - ) -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/BUILD b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/BUILD deleted file mode 100644 index fac4cd426..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "iesource/thrift/src/main/thrift:thrift-scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/BUILD.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/BUILD.docx new file mode 100644 index 000000000..1882e509e Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/README b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/README deleted file mode 100644 index b90dfd50d..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/README +++ /dev/null @@ -1,11 +0,0 @@ -Currently this dir contains multiple adapters. -The goal is similar: to generate Rekeyed (key by TweetId) `KeyedUuaTweet` events that can be -used for View Counts (aggregation). - -The 2 adapters: -1. Reads from UUA-all topic -2. Reads from InteractionEvents -We have 2 adapters mainly because currently InteractionEvents have 10% more TweetRenderImpressions -than what UUA has. Details can be found at https://docs.google.com/document/d/1UcEzAZ7rFrsU_6kl20R3YZ6u_Jt8PH_4-mVHWe216eM/edit# - -It is still unclear which source should be used, but at a time there should be only one service running. diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/README.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/README.docx new file mode 100644 index 000000000..b7f924476 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/README.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaAdapter.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaAdapter.docx new file mode 100644 index 000000000..f9905a9f5 Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaAdapter.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaAdapter.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaAdapter.scala deleted file mode 100644 index 08cc46a21..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaAdapter.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.unified_user_actions.adapter.uua_aggregates - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.thriftscala._ - -/** - * The main purpose of the rekey adapter and the rekey service is to not break the existing - * customers with the existing Unkeyed and also making the value as a super light-weight schema. - * After we rekey from Unkeyed to Long (tweetId), downstream KafkaStreams can directly consume - * without repartitioning. - */ -class RekeyUuaAdapter extends AbstractAdapter[UnifiedUserAction, Long, KeyedUuaTweet] { - - import RekeyUuaAdapter._ - override def adaptOneToKeyedMany( - input: UnifiedUserAction, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[(Long, KeyedUuaTweet)] = - adaptEvent(input).map { e => (e.tweetId, e) } -} - -object RekeyUuaAdapter { - def adaptEvent(e: UnifiedUserAction): Seq[KeyedUuaTweet] = - Option(e).flatMap { e => - e.actionType match { - case ActionType.ClientTweetRenderImpression => - ClientTweetRenderImpressionUua.getRekeyedUUA(e) - case _ => None - } - }.toSeq -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaFromInteractionEventsAdapter.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaFromInteractionEventsAdapter.docx new file mode 100644 index 000000000..cb1cb162f Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaFromInteractionEventsAdapter.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaFromInteractionEventsAdapter.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaFromInteractionEventsAdapter.scala deleted file mode 100644 index a513d7298..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/RekeyUuaFromInteractionEventsAdapter.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.unified_user_actions.adapter.uua_aggregates - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.iesource.thriftscala.ClientEventContext -import com.twitter.iesource.thriftscala.EngagingContext -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.iesource.thriftscala.InteractionType -import com.twitter.iesource.thriftscala.InteractionEvent -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.EventMetadata -import com.twitter.unified_user_actions.thriftscala.KeyedUuaTweet -import com.twitter.unified_user_actions.thriftscala.SourceLineage -import com.twitter.unified_user_actions.thriftscala.UserIdentifier - -/** - * This is to read directly from InteractionEvents - */ -class RekeyUuaFromInteractionEventsAdapter - extends AbstractAdapter[InteractionEvent, Long, KeyedUuaTweet] { - - import RekeyUuaFromInteractionEventsAdapter._ - override def adaptOneToKeyedMany( - input: InteractionEvent, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[(Long, KeyedUuaTweet)] = - adaptEvent(input, statsReceiver).map { e => (e.tweetId, e) } -} - -object RekeyUuaFromInteractionEventsAdapter { - - def adaptEvent( - e: InteractionEvent, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Seq[KeyedUuaTweet] = - Option(e).flatMap { e => - e.interactionType.flatMap { - case InteractionType.TweetRenderImpression if !isDetailImpression(e.engagingContext) => - getRekeyedUUA( - input = e, - actionType = ActionType.ClientTweetRenderImpression, - sourceLineage = SourceLineage.ClientEvents, - statsReceiver = statsReceiver) - case _ => None - } - }.toSeq - - def getRekeyedUUA( - input: InteractionEvent, - actionType: ActionType, - sourceLineage: SourceLineage, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Option[KeyedUuaTweet] = - input.engagingUserId match { - // please see https://docs.google.com/document/d/1-fy2S-8-YMRQgEN0Sco0OLTmeOIUdqgiZ5G1KwTHt2g/edit# - // in order to withstand of potential attacks, we filter out the logged-out users. - // Checking user id is 0 is the reverse engineering of - // https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/iesource/thrift/src/main/thrift/com/twitter/iesource/interaction_event.thrift?L220 - // https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/iesource/common/src/main/scala/com/twitter/iesource/common/converters/client/LogEventConverter.scala?L198 - case 0L => - statsReceiver.counter("loggedOutEvents").incr() - None - case _ => - Some( - KeyedUuaTweet( - tweetId = input.targetId, - actionType = actionType, - userIdentifier = UserIdentifier(userId = Some(input.engagingUserId)), - eventMetadata = EventMetadata( - sourceTimestampMs = input.triggeredTimestampMillis.getOrElse(input.timestampMillis), - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = sourceLineage - ) - )) - } - - def isDetailImpression(engagingContext: EngagingContext): Boolean = - engagingContext match { - case EngagingContext.ClientEventContext( - ClientEventContext(_, _, _, _, _, _, _, Some(isDetailsImpression), _) - ) if isDetailsImpression => - true - case _ => false - } -} diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/UuaActions.docx b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/UuaActions.docx new file mode 100644 index 000000000..2bd151b9e Binary files /dev/null and b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/UuaActions.docx differ diff --git a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/UuaActions.scala b/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/UuaActions.scala deleted file mode 100644 index eaf307ec8..000000000 --- a/unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates/UuaActions.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.unified_user_actions.adapter.uua_aggregates - -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.EventMetadata -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.KeyedUuaTweet -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction - -abstract class BaseUuaAction(actionType: ActionType) { - def getRekeyedUUA(input: UnifiedUserAction): Option[KeyedUuaTweet] = - getTweetIdFromItem(input.item).map { tweetId => - KeyedUuaTweet( - tweetId = tweetId, - actionType = input.actionType, - userIdentifier = input.userIdentifier, - eventMetadata = EventMetadata( - sourceTimestampMs = input.eventMetadata.sourceTimestampMs, - receivedTimestampMs = AdapterUtils.currentTimestampMs, - sourceLineage = input.eventMetadata.sourceLineage - ) - ) - } - - protected def getTweetIdFromItem(item: Item): Option[Long] = { - item match { - case Item.TweetInfo(tweetInfo) => Some(tweetInfo.actionTweetId) - case _ => None - } - } -} - -/** - * When there is a new user creation event in Gizmoduck - */ -object ClientTweetRenderImpressionUua extends BaseUuaAction(ActionType.ClientTweetRenderImpression) diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdapterUtilsSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdapterUtilsSpec.docx new file mode 100644 index 000000000..29c3a1b63 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdapterUtilsSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdapterUtilsSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdapterUtilsSpec.scala deleted file mode 100644 index c28ab3653..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdapterUtilsSpec.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.inject.Test -import com.twitter.unified_user_actions.adapter.common.AdapterUtils -import com.twitter.util.Time - -class AdapterUtilsSpec extends Test { - trait Fixture { - - val frozenTime: Time = Time.fromMilliseconds(1658949273000L) - val languageCode = "en" - val countryCode = "us" - } - - test("tests") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val actual = Time.fromMilliseconds(AdapterUtils.currentTimestampMs) - assert(frozenTime === actual) - } - - val actionedTweetId = 1554576940756246272L - assert(AdapterUtils.getTimestampMsFromTweetId(actionedTweetId) === 1659474999976L) - - assert(languageCode.toUpperCase === AdapterUtils.normalizeLanguageCode(languageCode)) - assert(countryCode.toUpperCase === AdapterUtils.normalizeCountryCode(countryCode)) - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdsCallbackEngagementsAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdsCallbackEngagementsAdapterSpec.docx new file mode 100644 index 000000000..01bb392b3 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdsCallbackEngagementsAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdsCallbackEngagementsAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdsCallbackEngagementsAdapterSpec.scala deleted file mode 100644 index 48309085c..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/AdsCallbackEngagementsAdapterSpec.scala +++ /dev/null @@ -1,282 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.ads.spendserver.thriftscala.SpendServerEvent -import com.twitter.adserver.thriftscala.EngagementType -import com.twitter.clientapp.thriftscala.AmplifyDetails -import com.twitter.inject.Test -import com.twitter.unified_user_actions.adapter.TestFixtures.AdsCallbackEngagementsFixture -import com.twitter.unified_user_actions.adapter.ads_callback_engagements.AdsCallbackEngagementsAdapter -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.TweetActionInfo -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.util.Time -import org.scalatest.prop.TableDrivenPropertyChecks - -class AdsCallbackEngagementsAdapterSpec extends Test with TableDrivenPropertyChecks { - - test("Test basic conversion for ads callback engagement type fav") { - - new AdsCallbackEngagementsFixture { - Time.withTimeAt(frozenTime) { _ => - val events = Table( - ("inputEvent", "expectedUuaOutput"), - ( // Test with authorId - createSpendServerEvent(EngagementType.Fav), - Seq( - createExpectedUua( - ActionType.ServerPromotedTweetFav, - createTweetInfoItem(authorInfo = Some(authorInfo))))) - ) - forEvery(events) { (event: SpendServerEvent, expected: Seq[UnifiedUserAction]) => - val actual = AdsCallbackEngagementsAdapter.adaptEvent(event) - assert(expected === actual) - } - } - } - } - - test("Test basic conversion for different engagement types") { - new AdsCallbackEngagementsFixture { - Time.withTimeAt(frozenTime) { _ => - val mappings = Table( - ("engagementType", "actionType"), - (EngagementType.Unfav, ActionType.ServerPromotedTweetUnfav), - (EngagementType.Reply, ActionType.ServerPromotedTweetReply), - (EngagementType.Retweet, ActionType.ServerPromotedTweetRetweet), - (EngagementType.Block, ActionType.ServerPromotedTweetBlockAuthor), - (EngagementType.Unblock, ActionType.ServerPromotedTweetUnblockAuthor), - (EngagementType.Send, ActionType.ServerPromotedTweetComposeTweet), - (EngagementType.Detail, ActionType.ServerPromotedTweetClick), - (EngagementType.Report, ActionType.ServerPromotedTweetReport), - (EngagementType.Mute, ActionType.ServerPromotedTweetMuteAuthor), - (EngagementType.ProfilePic, ActionType.ServerPromotedTweetClickProfile), - (EngagementType.ScreenName, ActionType.ServerPromotedTweetClickProfile), - (EngagementType.UserName, ActionType.ServerPromotedTweetClickProfile), - (EngagementType.Hashtag, ActionType.ServerPromotedTweetClickHashtag), - (EngagementType.CarouselSwipeNext, ActionType.ServerPromotedTweetCarouselSwipeNext), - ( - EngagementType.CarouselSwipePrevious, - ActionType.ServerPromotedTweetCarouselSwipePrevious), - (EngagementType.DwellShort, ActionType.ServerPromotedTweetLingerImpressionShort), - (EngagementType.DwellMedium, ActionType.ServerPromotedTweetLingerImpressionMedium), - (EngagementType.DwellLong, ActionType.ServerPromotedTweetLingerImpressionLong), - (EngagementType.DismissSpam, ActionType.ServerPromotedTweetDismissSpam), - (EngagementType.DismissWithoutReason, ActionType.ServerPromotedTweetDismissWithoutReason), - (EngagementType.DismissUninteresting, ActionType.ServerPromotedTweetDismissUninteresting), - (EngagementType.DismissRepetitive, ActionType.ServerPromotedTweetDismissRepetitive), - ) - - forEvery(mappings) { (engagementType: EngagementType, actionType: ActionType) => - val event = createSpendServerEvent(engagementType) - val actual = AdsCallbackEngagementsAdapter.adaptEvent(event) - val expected = - Seq(createExpectedUua(actionType, createTweetInfoItem(authorInfo = Some(authorInfo)))) - assert(expected === actual) - } - } - } - } - - test("Test conversion for ads callback engagement type spotlight view and click") { - new AdsCallbackEngagementsFixture { - Time.withTimeAt(frozenTime) { _ => - val input = Table( - ("adsEngagement", "uuaAction"), - (EngagementType.SpotlightClick, ActionType.ServerPromotedTweetClickSpotlight), - (EngagementType.SpotlightView, ActionType.ServerPromotedTweetViewSpotlight), - (EngagementType.TrendView, ActionType.ServerPromotedTrendView), - (EngagementType.TrendClick, ActionType.ServerPromotedTrendClick), - ) - forEvery(input) { (engagementType: EngagementType, actionType: ActionType) => - val adsEvent = createSpendServerEvent(engagementType) - val expected = Seq(createExpectedUua(actionType, trendInfoItem)) - val actual = AdsCallbackEngagementsAdapter.adaptEvent(adsEvent) - assert(expected === actual) - } - } - } - } - - test("Test basic conversion for ads callback engagement open link with or without url") { - new AdsCallbackEngagementsFixture { - Time.withTimeAt(frozenTime) { _ => - val input = Table( - ("url", "tweetActionInfo"), - (Some("go/url"), openLinkWithUrl), - (None, openLinkWithoutUrl) - ) - - forEvery(input) { (url: Option[String], tweetActionInfo: TweetActionInfo) => - val event = createSpendServerEvent(engagementType = EngagementType.Url, url = url) - val actual = AdsCallbackEngagementsAdapter.adaptEvent(event) - val expected = Seq(createExpectedUua( - ActionType.ServerPromotedTweetOpenLink, - createTweetInfoItem(authorInfo = Some(authorInfo), actionInfo = Some(tweetActionInfo)))) - assert(expected === actual) - } - } - } - } - - test("Test basic conversion for different engagement types with profile info") { - new AdsCallbackEngagementsFixture { - Time.withTimeAt(frozenTime) { _ => - val mappings = Table( - ("engagementType", "actionType"), - (EngagementType.Follow, ActionType.ServerPromotedProfileFollow), - (EngagementType.Unfollow, ActionType.ServerPromotedProfileUnfollow) - ) - forEvery(mappings) { (engagementType: EngagementType, actionType: ActionType) => - val event = createSpendServerEvent(engagementType) - val actual = AdsCallbackEngagementsAdapter.adaptEvent(event) - val expected = Seq(createExpectedUuaWithProfileInfo(actionType)) - assert(expected === actual) - } - } - } - } - - test("Test basic conversion for ads callback engagement type video_content_*") { - new AdsCallbackEngagementsFixture { - Time.withTimeAt(frozenTime) { _ => - val events = Table( - ("engagementType", "amplifyDetails", "actionType", "tweetActionInfo"), - //For video_content_* events on promoted tweets when there is no preroll ad played - ( - EngagementType.VideoContentPlayback25, - amplifyDetailsPromotedTweetWithoutAd, - ActionType.ServerPromotedTweetVideoPlayback25, - tweetActionInfoPromotedTweetWithoutAd), - ( - EngagementType.VideoContentPlayback50, - amplifyDetailsPromotedTweetWithoutAd, - ActionType.ServerPromotedTweetVideoPlayback50, - tweetActionInfoPromotedTweetWithoutAd), - ( - EngagementType.VideoContentPlayback75, - amplifyDetailsPromotedTweetWithoutAd, - ActionType.ServerPromotedTweetVideoPlayback75, - tweetActionInfoPromotedTweetWithoutAd), - //For video_content_* events on promoted tweets when there is a preroll ad - ( - EngagementType.VideoContentPlayback25, - amplifyDetailsPromotedTweetWithAd, - ActionType.ServerPromotedTweetVideoPlayback25, - tweetActionInfoPromotedTweetWithAd), - ( - EngagementType.VideoContentPlayback50, - amplifyDetailsPromotedTweetWithAd, - ActionType.ServerPromotedTweetVideoPlayback50, - tweetActionInfoPromotedTweetWithAd), - ( - EngagementType.VideoContentPlayback75, - amplifyDetailsPromotedTweetWithAd, - ActionType.ServerPromotedTweetVideoPlayback75, - tweetActionInfoPromotedTweetWithAd), - ) - forEvery(events) { - ( - engagementType: EngagementType, - amplifyDetails: Option[AmplifyDetails], - actionType: ActionType, - actionInfo: Option[TweetActionInfo] - ) => - val spendEvent = - createVideoSpendServerEvent(engagementType, amplifyDetails, promotedTweetId, None) - val expected = Seq(createExpectedVideoUua(actionType, actionInfo, promotedTweetId)) - - val actual = AdsCallbackEngagementsAdapter.adaptEvent(spendEvent) - assert(expected === actual) - } - } - } - } - - test("Test basic conversion for ads callback engagement type video_ad_*") { - - new AdsCallbackEngagementsFixture { - Time.withTimeAt(frozenTime) { _ => - val events = Table( - ( - "engagementType", - "amplifyDetails", - "actionType", - "tweetActionInfo", - "promotedTweetId", - "organicTweetId"), - //For video_ad_* events when the preroll ad is on a promoted tweet. - ( - EngagementType.VideoAdPlayback25, - amplifyDetailsPrerollAd, - ActionType.ServerPromotedTweetVideoAdPlayback25, - tweetActionInfoPrerollAd, - promotedTweetId, - None - ), - ( - EngagementType.VideoAdPlayback50, - amplifyDetailsPrerollAd, - ActionType.ServerPromotedTweetVideoAdPlayback50, - tweetActionInfoPrerollAd, - promotedTweetId, - None - ), - ( - EngagementType.VideoAdPlayback75, - amplifyDetailsPrerollAd, - ActionType.ServerPromotedTweetVideoAdPlayback75, - tweetActionInfoPrerollAd, - promotedTweetId, - None - ), - // For video_ad_* events when the preroll ad is on an organic tweet. - ( - EngagementType.VideoAdPlayback25, - amplifyDetailsPrerollAd, - ActionType.ServerTweetVideoAdPlayback25, - tweetActionInfoPrerollAd, - None, - organicTweetId - ), - ( - EngagementType.VideoAdPlayback50, - amplifyDetailsPrerollAd, - ActionType.ServerTweetVideoAdPlayback50, - tweetActionInfoPrerollAd, - None, - organicTweetId - ), - ( - EngagementType.VideoAdPlayback75, - amplifyDetailsPrerollAd, - ActionType.ServerTweetVideoAdPlayback75, - tweetActionInfoPrerollAd, - None, - organicTweetId - ), - ) - forEvery(events) { - ( - engagementType: EngagementType, - amplifyDetails: Option[AmplifyDetails], - actionType: ActionType, - actionInfo: Option[TweetActionInfo], - promotedTweetId: Option[Long], - organicTweetId: Option[Long], - ) => - val spendEvent = - createVideoSpendServerEvent( - engagementType, - amplifyDetails, - promotedTweetId, - organicTweetId) - val actionTweetId = if (organicTweetId.isDefined) organicTweetId else promotedTweetId - val expected = Seq(createExpectedVideoUua(actionType, actionInfo, actionTweetId)) - - val actual = AdsCallbackEngagementsAdapter.adaptEvent(spendEvent) - assert(expected === actual) - } - } - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/BUILD.bazel b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/BUILD.bazel deleted file mode 100644 index 4c6d8e27a..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -junit_tests( - sources = ["**/*.scala"], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "finatra/inject/inject-core/src/test/scala:test-deps", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/ads_callback_engagements", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/common", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates", - "util/util-mock/src/main/scala/com/twitter/util/mock", - ], -) diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/BUILD.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/BUILD.docx new file mode 100644 index 000000000..dc7b49009 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/BUILD.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/ClientEventAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/ClientEventAdapterSpec.docx new file mode 100644 index 000000000..bcd6a6cc2 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/ClientEventAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/ClientEventAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/ClientEventAdapterSpec.scala deleted file mode 100644 index dde8a2f02..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/ClientEventAdapterSpec.scala +++ /dev/null @@ -1,2157 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.clientapp.thriftscala.EventNamespace -import com.twitter.clientapp.thriftscala.{Item => LogEventItem} -import com.twitter.clientapp.thriftscala.ItemType -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.clientapp.thriftscala.NotificationTabDetails -import com.twitter.clientapp.thriftscala.ReportDetails -import com.twitter.clientapp.thriftscala.SearchDetails -import com.twitter.clientapp.thriftscala.SuggestionDetails -import com.twitter.inject.Test -import com.twitter.logbase.thriftscala.ClientEventReceiver -import com.twitter.reportflow.thriftscala.ReportType -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.unified_user_actions.adapter.client_event.ClientEventAdapter -import com.twitter.unified_user_actions.thriftscala._ -import com.twitter.util.Time -import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatest.prop.TableFor1 -import org.scalatest.prop.TableFor2 -import scala.language.implicitConversions - -class ClientEventAdapterSpec extends Test with TableDrivenPropertyChecks { - // Tests for invalid client-events - test("should ignore events") { - new TestFixtures.ClientEventFixture { - val eventsToBeIgnored: TableFor2[String, LogEvent] = Table( - ("namespace", "event"), - ("ddg", ddgEvent), - ("qig_ranker", qigRankerEvent), - ("timelnemixer", timelineMixerEvent), - ("timelineservice", timelineServiceEvent), - ("tweetconvosvc", tweetConcServiceEvent), - ("item-type is non-tweet", renderNonTweetItemTypeEvent) - ) - - forEvery(eventsToBeIgnored) { (_: String, event: LogEvent) => - val actual = ClientEventAdapter.adaptEvent(event) - assert(actual.isEmpty) - } - } - } - - test("Tests for ItemType filter") { - /// Tweet events - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val events = Table( - ("itemType", "expectedUUA"), - (Some(ItemType.Tweet), Seq(expectedTweetRenderDefaultTweetUUA)), - (Some(ItemType.QuotedTweet), Seq(expectedTweetRenderDefaultTweetUUA)), - (Some(ItemType.Topic), Nil), - (None, Nil) - ) - - forEvery(events) { (itemTypeOpt: Option[ItemType], expected: Seq[UnifiedUserAction]) => - val actual = ClientEventAdapter.adaptEvent( - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceRenderEventNamespace), - itemTypeOpt = itemTypeOpt - )) - assert(expected === actual) - } - } - } - - /// Topic events - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val expected: UnifiedUserAction = mkExpectedUUAForActionTowardTopicEvent( - topicId = topicId, - clientEventNamespace = Some(uuaTopicFollowClientEventNamespace1), - actionType = ActionType.ClientTopicFollow - ) - val events = Table( - ("itemType", "expectedUUA"), - (Some(ItemType.Tweet), Seq(expected)), - (Some(ItemType.QuotedTweet), Seq(expected)), - (Some(ItemType.Topic), Seq(expected)), - (None, Nil) - ) - - forEvery(events) { (itemTypeOpt: Option[ItemType], expected: Seq[UnifiedUserAction]) => - val actual = ClientEventAdapter.adaptEvent( - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceTopicFollow1), - itemId = None, - suggestionDetails = - Some(SuggestionDetails(decodedControllerData = Some(homeTweetControllerData()))), - itemTypeOpt = itemTypeOpt - )) - assert(expected === actual) - } - } - } - } - - // Tests for ClientTweetRenderImpression - test("ClientTweetRenderImpression") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("actionTweetType", "clientEvent", "expectedUUAEvent"), - ( - "Default", - actionTowardDefaultTweetEvent(eventNamespace = Some(ceRenderEventNamespace)), - Seq(expectedTweetRenderDefaultTweetUUA)), - ( - "Reply", - actionTowardReplyEvent(eventNamespace = Some(ceRenderEventNamespace)), - Seq(expectedTweetRenderReplyUUA)), - ( - "Retweet", - actionTowardRetweetEvent(eventNamespace = Some(ceRenderEventNamespace)), - Seq(expectedTweetRenderRetweetUUA)), - ( - "Quote", - actionTowardQuoteEvent( - eventNamespace = Some(ceRenderEventNamespace), - quotedAuthorId = Some(456L)), - Seq(expectedTweetRenderQuoteUUA1, expectedTweetRenderQuoteUUA2)), - ( - "Retweet of a reply that quoted another Tweet", - actionTowardRetweetEventWithReplyAndQuote(eventNamespace = - Some(ceRenderEventNamespace)), - Seq( - expectedTweetRenderRetweetWithReplyAndQuoteUUA1, - expectedTweetRenderRetweetWithReplyAndQuoteUUA2)) - ) - forEvery(clientEvents) { - (_: String, event: LogEvent, expectedUUA: Seq[UnifiedUserAction]) => - val actual = ClientEventAdapter.adaptEvent(event) - actual should contain theSameElementsAs expectedUUA - } - } - } - } - - test("ClientTweetGallery/DetailImpression") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("actionTweetType", "clientEvent", "expectedUUAEvent"), - ( - "DetailImpression: tweet::tweet::impression", - actionTowardDefaultTweetEvent(eventNamespace = Some(ceTweetDetailsEventNamespace1)), - expectedTweetDetailImpressionUUA1), - ( - "GalleryImpression: gallery:photo:impression", - actionTowardDefaultTweetEvent(eventNamespace = Some(ceGalleryEventNamespace)), - expectedTweetGalleryImpressionUUA), - ) - forEvery(clientEvents) { (_: String, event: LogEvent, expectedUUA: UnifiedUserAction) => - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetLingerImpression - test("ClientTweetLingerImpression") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("actionTweetType", "clientEvent", "expectedUUAEvent"), - ("Default", lingerDefaultTweetEvent, expectedTweetLingerDefaultTweetUUA), - ("Reply", lingerReplyEvent, expectedTweetLingerReplyUUA), - ("Retweet", lingerRetweetEvent, expectedTweetLingerRetweetUUA), - ("Quote", lingerQuoteEvent, expectedTweetLingerQuoteUUA), - ( - "Retweet of a reply that quoted another Tweet", - lingerRetweetWithReplyAndQuoteEvent, - expectedTweetLingerRetweetWithReplyAndQuoteUUA), - ) - forEvery(clientEvents) { (_: String, event: LogEvent, expectedUUA: UnifiedUserAction) => - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetClickQuote - test( - "ClickQuote, which is the click on the quote button, results in setting retweeting, inReplyTo, quoted tweet ids") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actual = ClientEventAdapter.adaptEvent( - // there shouldn't be any quotingTweetId in CE when it is "quote" - actionTowardRetweetEventWithReplyAndQuote(eventNamespace = Some( - EventNamespace( - action = Some("quote") - )))) - assert(Seq(expectedTweetClickQuoteUUA) === actual) - } - } - } - - // Tests for ClientTweetQuote - test( - "Quote, which is sending the quote, results in setting retweeting, inReplyTo, quoted tweet ids") { - new TestFixtures.ClientEventFixture { - val actions: TableFor1[String] = Table( - "action", - "send_quote_tweet", - "retweet_with_comment" - ) - - Time.withTimeAt(frozenTime) { _ => - forEvery(actions) { action => - val actual = ClientEventAdapter.adaptEvent( - // there shouldn't be any quotingTweetId in CE when it is "quote" - actionTowardRetweetEventWithReplyAndQuote(eventNamespace = Some( - EventNamespace( - action = Some(action) - )))) - assert(Seq(expectedTweetQuoteUUA(action)) === actual) - } - } - } - } - - // Tests for ClientTweetFav and ClientTweetUnfav - test("ClientTweetFav and ClientTweetUnfav") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("actionTweetType", "clientEvent", "expectedUUAEvent"), - ( - "Default Tweet favorite", - actionTowardDefaultTweetEvent(eventNamespace = Some(ceFavoriteEventNamespace)), - expectedTweetFavoriteDefaultTweetUUA), - ( - "Reply Tweet favorite", - actionTowardReplyEvent(eventNamespace = Some(ceFavoriteEventNamespace)), - expectedTweetFavoriteReplyUUA), - ( - "Retweet Tweet favorite", - actionTowardRetweetEvent(eventNamespace = Some(ceFavoriteEventNamespace)), - expectedTweetFavoriteRetweetUUA), - ( - "Quote Tweet favorite", - actionTowardQuoteEvent(eventNamespace = Some(ceFavoriteEventNamespace)), - expectedTweetFavoriteQuoteUUA), - ( - "Retweet of a reply that quoted another Tweet favorite", - actionTowardRetweetEventWithReplyAndQuote(eventNamespace = - Some(ceFavoriteEventNamespace)), - expectedTweetFavoriteRetweetWithReplyAndQuoteUUA), - ( - "Default Tweet unfavorite", - actionTowardDefaultTweetEvent( - eventNamespace = Some(EventNamespace(action = Some("unfavorite"))), - ), - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(ClientEventNamespace(action = Some("unfavorite"))), - actionType = ActionType.ClientTweetUnfav - )) - ) - forEvery(clientEvents) { (_: String, event: LogEvent, expectedUUA: UnifiedUserAction) => - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetClickReply - test("ClientTweetClickReply") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("actionTweetType", "clientEvent", "expectedUUAEvent"), - ( - "Default", - actionTowardDefaultTweetEvent(eventNamespace = Some(ceClickReplyEventNamespace)), - expectedTweetClickReplyDefaultTweetUUA), - ( - "Reply", - actionTowardReplyEvent(eventNamespace = Some(ceClickReplyEventNamespace)), - expectedTweetClickReplyReplyUUA), - ( - "Retweet", - actionTowardRetweetEvent(eventNamespace = Some(ceClickReplyEventNamespace)), - expectedTweetClickReplyRetweetUUA), - ( - "Quote", - actionTowardQuoteEvent(eventNamespace = Some(ceClickReplyEventNamespace)), - expectedTweetClickReplyQuoteUUA), - ( - "Retweet of a reply that quoted another Tweet", - actionTowardRetweetEventWithReplyAndQuote(eventNamespace = - Some(ceClickReplyEventNamespace)), - expectedTweetClickReplyRetweetWithReplyAndQuoteUUA) - ) - forEvery(clientEvents) { (_: String, event: LogEvent, expectedUUA: UnifiedUserAction) => - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetReply - test("ClientTweetReply") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("actionTweetType", "clientEvent", "expectedUUAEvent"), - ("DefaultOrReply", replyToDefaultTweetOrReplyEvent, expectedTweetReplyDefaultTweetUUA), - ("Retweet", replyToRetweetEvent, expectedTweetReplyRetweetUUA), - ("Quote", replyToQuoteEvent, expectedTweetReplyQuoteUUA), - ( - "Retweet of a reply that quoted another Tweet", - replyToRetweetWithReplyAndQuoteEvent, - expectedTweetReplyRetweetWithReplyAndQuoteUUA) - ) - forEvery(clientEvents) { (_: String, event: LogEvent, expectedUUA: UnifiedUserAction) => - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetRetweet and ClientTweetUnretweet - test("ClientTweetRetweet and ClientTweetUnretweet") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("actionTweetType", "clientEvent", "expectedUUAEvent"), - ( - "Default Tweet retweet", - actionTowardDefaultTweetEvent(eventNamespace = Some(ceRetweetEventNamespace)), - expectedTweetRetweetDefaultTweetUUA), - ( - "Reply Tweet retweet", - actionTowardReplyEvent(eventNamespace = Some(ceRetweetEventNamespace)), - expectedTweetRetweetReplyUUA), - ( - "Retweet Tweet retweet", - actionTowardRetweetEvent(eventNamespace = Some(ceRetweetEventNamespace)), - expectedTweetRetweetRetweetUUA), - ( - "Quote Tweet retweet", - actionTowardQuoteEvent(eventNamespace = Some(ceRetweetEventNamespace)), - expectedTweetRetweetQuoteUUA), - ( - "Retweet of a reply that quoted another Tweet retweet", - actionTowardRetweetEventWithReplyAndQuote(eventNamespace = - Some(ceRetweetEventNamespace)), - expectedTweetRetweetRetweetWithReplyAndQuoteUUA), - ( - "Default Tweet unretweet", - actionTowardDefaultTweetEvent( - eventNamespace = Some(EventNamespace(action = Some("unretweet"))), - ), - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(ClientEventNamespace(action = Some("unretweet"))), - actionType = ActionType.ClientTweetUnretweet - )) - ) - forEvery(clientEvents) { (_: String, event: LogEvent, expectedUUA: UnifiedUserAction) => - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - test("include Topic Id") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actual = ClientEventAdapter.adaptEvent(renderDefaultTweetWithTopicIdEvent) - assert(Seq(expectedTweetRenderDefaultTweetWithTopicIdUUA) === actual) - } - } - } - - // Tests for ClientTweetVideoPlayback0, 25, 50, 75, 95, 100 PlayFromTap, QualityView, - // VideoView, MrcView, ViewThreshold - test("ClientTweetVideoPlayback*") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("ceNamespace", "uuaNamespace", "uuaActionType"), - ( - ceVideoPlayback25, - uuaVideoPlayback25ClientEventNamespace, - ActionType.ClientTweetVideoPlayback25), - ( - ceVideoPlayback50, - uuaVideoPlayback50ClientEventNamespace, - ActionType.ClientTweetVideoPlayback50), - ( - ceVideoPlayback75, - uuaVideoPlayback75ClientEventNamespace, - ActionType.ClientTweetVideoPlayback75), - ( - ceVideoPlayback95, - uuaVideoPlayback95ClientEventNamespace, - ActionType.ClientTweetVideoPlayback95), - ( - ceVideoPlayFromTap, - uuaVideoPlayFromTapClientEventNamespace, - ActionType.ClientTweetVideoPlayFromTap), - ( - ceVideoQualityView, - uuaVideoQualityViewClientEventNamespace, - ActionType.ClientTweetVideoQualityView), - (ceVideoView, uuaVideoViewClientEventNamespace, ActionType.ClientTweetVideoView), - (ceVideoMrcView, uuaVideoMrcViewClientEventNamespace, ActionType.ClientTweetVideoMrcView), - ( - ceVideoViewThreshold, - uuaVideoViewThresholdClientEventNamespace, - ActionType.ClientTweetVideoViewThreshold), - ( - ceVideoCtaUrlClick, - uuaVideoCtaUrlClickClientEventNamespace, - ActionType.ClientTweetVideoCtaUrlClick), - ( - ceVideoCtaWatchClick, - uuaVideoCtaWatchClickClientEventNamespace, - ActionType.ClientTweetVideoCtaWatchClick), - ) - - for (element <- videoEventElementValues) { - forEvery(clientEvents) { - ( - ceNamespace: EventNamespace, - uuaNamespace: ClientEventNamespace, - uuaActionType: ActionType - ) => - val event = actionTowardDefaultTweetEvent( - eventNamespace = Some(ceNamespace.copy(element = Some(element))), - mediaDetailsV2 = Some(mediaDetailsV2), - clientMediaEvent = Some(clientMediaEvent), - cardDetails = Some(cardDetails) - ) - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaNamespace.copy(element = Some(element))), - actionType = uuaActionType, - tweetActionInfo = Some(videoMetadata) - ) - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - } - - // Tests for ClientTweetPhotoExpand - test("Client Tweet Photo Expand") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvent = actionTowardDefaultTweetEvent(eventNamespace = Some(cePhotoExpand)) - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaPhotoExpandClientEventNamespace), - actionType = ActionType.ClientTweetPhotoExpand - ) - assert(Seq(expectedUUA) === ClientEventAdapter.adaptEvent(clientEvent)) - } - } - } - - // Tests for ClientCardClick - test("Client Card Related") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("ceNamespace", "ceItemType", "uuaNamespace", "uuaActionType"), - ( - ceCardClick, - ItemType.Tweet, - uuaCardClickClientEventNamespace, - ActionType.ClientCardClick), - ( - ceCardClick, - ItemType.User, - uuaCardClickClientEventNamespace, - ActionType.ClientCardClick), - ( - ceCardOpenApp, - ItemType.Tweet, - uuaCardOpenAppClientEventNamespace, - ActionType.ClientCardOpenApp), - ( - ceCardAppInstallAttempt, - ItemType.Tweet, - uuaCardAppInstallAttemptClientEventNamespace, - ActionType.ClientCardAppInstallAttempt), - ( - cePollCardVote1, - ItemType.Tweet, - uuaPollCardVote1ClientEventNamespace, - ActionType.ClientPollCardVote), - ( - cePollCardVote2, - ItemType.Tweet, - uuaPollCardVote2ClientEventNamespace, - ActionType.ClientPollCardVote), - ) - forEvery(clientEvents) { - ( - ceNamespace: EventNamespace, - ceItemType: ItemType, - uuaNamespace: ClientEventNamespace, - uuaActionType: ActionType - ) => - val event = actionTowardDefaultTweetEvent( - eventNamespace = Some(ceNamespace), - itemTypeOpt = Some(ceItemType), - authorId = Some(authorId) - ) - val expectedUUA = mkExpectedUUAForCardEvent( - id = Some(itemTweetId), - clientEventNamespace = Some(uuaNamespace), - actionType = uuaActionType, - itemType = Some(ceItemType), - authorId = Some(authorId) - ) - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetClickMentionScreenName - test("ClientTweetClickMentionScreenName") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val userHandle = "someHandle" - val clientEvent = actionTowardDefaultTweetEvent( - eventNamespace = Some(ceMentionClick), - targets = Some( - Seq( - LogEventItem( - itemType = Some(ItemType.User), - id = Some(userId), - name = Some(userHandle))))) - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaMentionClickClientEventNamespace), - actionType = ActionType.ClientTweetClickMentionScreenName, - tweetActionInfo = Some( - TweetActionInfo.ClientTweetClickMentionScreenName( - ClientTweetClickMentionScreenName(actionProfileId = userId, handle = userHandle))) - ) - assert(Seq(expectedUUA) === ClientEventAdapter.adaptEvent(clientEvent)) - } - } - } - - // Tests for Topic Follow/Unfollow actions - test("Topic Follow/Unfollow Actions") { - // The Topic Id is mostly from TimelineTopic controller data or HomeTweets controller data! - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("clientEventNamesapce", "expectedUUANamespace", "controllerData", "actionType"), - ( - ceTopicFollow1, - uuaTopicFollowClientEventNamespace1, - timelineTopicControllerData(), - ActionType.ClientTopicFollow - ), - ( - ceTopicFollow1, - uuaTopicFollowClientEventNamespace1, - homeTweetControllerData(), - ActionType.ClientTopicFollow), - ( - ceTopicFollow2, - uuaTopicFollowClientEventNamespace2, - timelineTopicControllerData(), - ActionType.ClientTopicFollow - ), - ( - ceTopicFollow2, - uuaTopicFollowClientEventNamespace2, - homeTweetControllerData(), - ActionType.ClientTopicFollow), - ( - ceTopicFollow3, - uuaTopicFollowClientEventNamespace3, - timelineTopicControllerData(), - ActionType.ClientTopicFollow - ), - ( - ceTopicFollow3, - uuaTopicFollowClientEventNamespace3, - homeTweetControllerData(), - ActionType.ClientTopicFollow), - ( - ceTopicUnfollow1, - uuaTopicUnfollowClientEventNamespace1, - timelineTopicControllerData(), - ActionType.ClientTopicUnfollow - ), - ( - ceTopicUnfollow1, - uuaTopicUnfollowClientEventNamespace1, - homeTweetControllerData(), - ActionType.ClientTopicUnfollow), - ( - ceTopicUnfollow2, - uuaTopicUnfollowClientEventNamespace2, - timelineTopicControllerData(), - ActionType.ClientTopicUnfollow - ), - ( - ceTopicFollow2, - uuaTopicFollowClientEventNamespace2, - homeTweetControllerData(), - ActionType.ClientTopicFollow), - ( - ceTopicUnfollow3, - uuaTopicUnfollowClientEventNamespace3, - timelineTopicControllerData(), - ActionType.ClientTopicUnfollow - ), - ( - ceTopicUnfollow3, - uuaTopicUnfollowClientEventNamespace3, - homeTweetControllerData(), - ActionType.ClientTopicUnfollow), - ) - - forEvery(clientEvents) { - ( - eventNamespace: EventNamespace, - uuaNs: ClientEventNamespace, - controllerData: ControllerData, - actionType: ActionType - ) => - val event = actionTowardDefaultTweetEvent( - eventNamespace = Some(eventNamespace), - itemId = None, - suggestionDetails = - Some(SuggestionDetails(decodedControllerData = Some(controllerData))) - ) - val expectedUUA = mkExpectedUUAForActionTowardTopicEvent( - topicId = topicId, - traceId = None, - clientEventNamespace = Some(uuaNs), - actionType = actionType - ) - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for Topic NotInterestedIn & its Undo actions - test("Topic NotInterestedIn & its Undo actions") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("clientEventNamesapce", "expectedUUANamespace", "controllerData", "actionType"), - ( - ceTopicNotInterestedIn1, - uuaTopicNotInterestedInClientEventNamespace1, - timelineTopicControllerData(), - ActionType.ClientTopicNotInterestedIn - ), - ( - ceTopicNotInterestedIn1, - uuaTopicNotInterestedInClientEventNamespace1, - homeTweetControllerData(), - ActionType.ClientTopicNotInterestedIn), - ( - ceTopicNotInterestedIn2, - uuaTopicNotInterestedInClientEventNamespace2, - timelineTopicControllerData(), - ActionType.ClientTopicNotInterestedIn - ), - ( - ceTopicNotInterestedIn2, - uuaTopicNotInterestedInClientEventNamespace2, - homeTweetControllerData(), - ActionType.ClientTopicNotInterestedIn), - ( - ceTopicUndoNotInterestedIn1, - uuaTopicUndoNotInterestedInClientEventNamespace1, - timelineTopicControllerData(), - ActionType.ClientTopicUndoNotInterestedIn - ), - ( - ceTopicUndoNotInterestedIn1, - uuaTopicUndoNotInterestedInClientEventNamespace1, - homeTweetControllerData(), - ActionType.ClientTopicUndoNotInterestedIn), - ( - ceTopicUndoNotInterestedIn2, - uuaTopicUndoNotInterestedInClientEventNamespace2, - timelineTopicControllerData(), - ActionType.ClientTopicUndoNotInterestedIn - ), - ( - ceTopicUndoNotInterestedIn2, - uuaTopicUndoNotInterestedInClientEventNamespace2, - homeTweetControllerData(), - ActionType.ClientTopicUndoNotInterestedIn), - ) - - forEvery(clientEvents) { - ( - eventNamespace: EventNamespace, - uuaNs: ClientEventNamespace, - controllerData: ControllerData, - actionType: ActionType - ) => - val event = actionTowardDefaultTweetEvent( - eventNamespace = Some(eventNamespace), - itemId = None, - suggestionDetails = - Some(SuggestionDetails(decodedControllerData = Some(controllerData))) - ) - val expectedUUA = mkExpectedUUAForActionTowardTopicEvent( - topicId = topicId, - traceId = None, - clientEventNamespace = Some(uuaNs), - actionType = actionType - ) - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for authorInfo - test("authorInfo") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("authorIdOpt", "isFollowedByActingUser", "isFollowingActingUser"), - (Some(authorId), true, false), - (Some(authorId), true, true), - (Some(authorId), false, true), - (Some(authorId), false, false), - (None, true, true), - ) - forEvery(clientEvents) { - ( - authorIdOpt: Option[Long], - isFollowedByActingUser: Boolean, - isFollowingActingUser: Boolean - ) => - val actual = ClientEventAdapter.adaptEvent( - renderDefaultTweetUserFollowStatusEvent( - authorId = authorIdOpt, - isFollowedByActingUser = isFollowedByActingUser, - isFollowingActingUser = isFollowingActingUser - )) - val expected = - expectedTweetRenderDefaultTweetWithAuthorInfoUUA(authorInfo = authorIdOpt.map { id => - AuthorInfo( - authorId = Some(id), - isFollowedByActingUser = Some(isFollowedByActingUser), - isFollowingActingUser = Some(isFollowingActingUser) - ) - }) - assert(Seq(expected) === actual) - } - } - } - } - - // Tests for ClientTweetReport - test("ClientTweetReport") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val ceNTabTweetReport: EventNamespace = - ceTweetReport.copy(page = Some("ntab"), section = Some("all"), component = Some("urt")) - - val uuaNTabTweetReport: ClientEventNamespace = - uuaTweetReport.copy(page = Some("ntab"), section = Some("all"), component = Some("urt")) - - val params = Table( - ( - "eventType", - "ceNamespace", - "ceNotificationTabDetails", - "ceReportDetails", - "uuaNamespace", - "uuaTweetActionInfo", - "uuaProductSurface", - "uuaProductSurfaceInfo"), - ( - "ntabReportTweetClick", - ceNTabTweetReport.copy(action = Some("click")), - Some(notificationTabTweetEventDetails), - None, - uuaNTabTweetReport.copy(action = Some("click")), - reportTweetClick, - Some(ProductSurface.NotificationTab), - Some(notificationTabProductSurfaceInfo) - ), - ( - "ntabReportTweetDone", - ceNTabTweetReport.copy(action = Some("done")), - Some(notificationTabTweetEventDetails), - None, - uuaNTabTweetReport.copy(action = Some("done")), - reportTweetDone, - Some(ProductSurface.NotificationTab), - Some(notificationTabProductSurfaceInfo) - ), - ( - "defaultReportTweetDone", - ceTweetReport.copy(page = Some("tweet"), action = Some("done")), - None, - None, - uuaTweetReport.copy(page = Some("tweet"), action = Some("done")), - reportTweetDone, - None, - None - ), - ( - "defaultReportTweetWithReportFlowId", - ceTweetReport.copy(page = Some("tweet"), action = Some("done")), - None, - Some(ReportDetails(reportFlowId = Some(reportFlowId))), - uuaTweetReport.copy(page = Some("tweet"), action = Some("done")), - reportTweetWithReportFlowId, - None, - None - ), - ( - "defaultReportTweetWithoutReportFlowId", - ceTweetReport.copy(page = Some("tweet"), action = Some("done")), - None, - None, - uuaTweetReport.copy(page = Some("tweet"), action = Some("done")), - reportTweetWithoutReportFlowId, - None, - None - ), - ) - - forEvery(params) { - ( - _: String, - ceNamespace: EventNamespace, - ceNotificationTabDetails: Option[NotificationTabDetails], - ceReportDetails: Option[ReportDetails], - uuaNamespace: ClientEventNamespace, - uuaTweetActionInfo: TweetActionInfo, - productSurface: Option[ProductSurface], - productSurfaceInfo: Option[ProductSurfaceInfo] - ) => - val actual = ClientEventAdapter.adaptEvent( - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceNamespace), - notificationTabDetails = ceNotificationTabDetails, - reportDetails = ceReportDetails)) - - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaNamespace), - actionType = ActionType.ClientTweetReport, - tweetActionInfo = Some(uuaTweetActionInfo), - productSurface = productSurface, - productSurfaceInfo = productSurfaceInfo - ) - - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetNotHelpful and ClientTweetUndoNotHelpful - test("ClientTweetNotHelpful & UndoNotHelpful") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actions = Table(("action"), "click", "undo") - val element = "feedback_givefeedback" - forEvery(actions) { action => - val clientEvent = - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceEventNamespace(element, action)), - ) - - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaClientEventNamespace(element, action)), - actionType = action match { - case "click" => ActionType.ClientTweetNotHelpful - case "undo" => ActionType.ClientTweetUndoNotHelpful - } - ) - - val actual = ClientEventAdapter.adaptEvent(clientEvent) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetNotInterestedIn and ClientTweetUndoNotInterestedIn - test("ClientTweetNotInterestedIn & UndoNotInterestedIn") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actions = Table(("action"), "click", "undo") - val element = "feedback_dontlike" - forEvery(actions) { action => - val clientEvent = - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceEventNamespace(element, action)), - ) - - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaClientEventNamespace(element, action)), - actionType = action match { - case "click" => ActionType.ClientTweetNotInterestedIn - case "undo" => ActionType.ClientTweetUndoNotInterestedIn - } - ) - - val actual = ClientEventAdapter.adaptEvent(clientEvent) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetNotAboutTopic & ClientTweetUndoNotAboutTopic - test("ClientTweetNotAboutTopic & ClientTweetUndoNotAboutTopic") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actions = Table(("action"), "click", "undo") - val element = "feedback_notabouttopic" - forEvery(actions) { action => - val clientEvent = - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceEventNamespace(element, action)), - ) - - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaClientEventNamespace(element, action)), - actionType = action match { - case "click" => ActionType.ClientTweetNotAboutTopic - case "undo" => ActionType.ClientTweetUndoNotAboutTopic - } - ) - - val actual = ClientEventAdapter.adaptEvent(clientEvent) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetNotRecent and ClientTweetUndoNotRecent - test("ClientTweetNotRecent & UndoNotRecent") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actions = Table(("action"), "click", "undo") - val element = "feedback_notrecent" - forEvery(actions) { action => - val clientEvent = - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceEventNamespace(element, action)), - ) - - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaClientEventNamespace(element, action)), - actionType = action match { - case "click" => ActionType.ClientTweetNotRecent - case "undo" => ActionType.ClientTweetUndoNotRecent - } - ) - - val actual = ClientEventAdapter.adaptEvent(clientEvent) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetSeeFewer and ClientTweetUndoSeeFewer - test("ClientTweetSeeFewer & ClientTweetUndoSeeFewer") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actions = Table(("action"), "click", "undo") - val element = "feedback_seefewer" - forEvery(actions) { action => - val clientEvent = - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceEventNamespace(element, action)), - ) - - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaClientEventNamespace(element, action)), - actionType = action match { - case "click" => ActionType.ClientTweetSeeFewer - case "undo" => ActionType.ClientTweetUndoSeeFewer - } - ) - - val actual = ClientEventAdapter.adaptEvent(clientEvent) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for getEventMetadata - test("getEventMetadata") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("clientEventNamesapce", "expectedUUANamespace", "controllerData"), - ( - ceRenderEventNamespace, - uuaRenderClientEventNamespace, - homeTweetControllerData() - ), - ) - - forEvery(clientEvents) { - ( - eventNamespace: EventNamespace, - uuaNs: ClientEventNamespace, - controllerData: ControllerData - ) => - val event = actionTowardDefaultTweetEvent( - eventNamespace = Some(eventNamespace), - suggestionDetails = - Some(SuggestionDetails(decodedControllerData = Some(controllerData))) - ) - val expectedEventMetaData = mkUUAEventMetadata( - clientEventNamespace = Some(uuaNs) - ) - val actual = ClientEventAdapter.adaptEvent(event).head.eventMetadata - assert(expectedEventMetaData === actual) - } - } - } - } - - // Tests for getSourceTimestamp - test("getSourceTimestamp") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val params = Table( - ("testCase", "clientEvent", "expectedUUAEventTimestamp"), - ( - "CES event with DriftAdjustedEventCreatedAtMs", - actionTowardDefaultTweetEvent(eventNamespace = Some(ceRenderEventNamespace)), - logBase.driftAdjustedEventCreatedAtMs), - ( - "CES event without DriftAdjustedEventCreatedAtMs: ignore", - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceRenderEventNamespace), - logBase = logBase.unsetDriftAdjustedEventCreatedAtMs), - None), - ( - "Non-CES event without DriftAdjustedEventCreatedAtMs: use logBase.timestamp", - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceRenderEventNamespace), - logBase = logBase - .copy( - clientEventReceiver = - Some(ClientEventReceiver.Unknown)).unsetDriftAdjustedEventCreatedAtMs - ), - Some(logBase.timestamp)) - ) - forEvery(params) { (_: String, event: LogEvent, expectedUUAEventTimestamp: Option[Long]) => - val actual = - ClientEventAdapter.adaptEvent(event).map(_.eventMetadata.sourceTimestampMs).headOption - assert(expectedUUAEventTimestamp === actual) - } - } - } - } - - // Tests for ServerTweetReport - test("ServerTweetReport") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val params = Table( - ("eventType", "ceNamespace", "ceReportDetails", "uuaNamespace", "uuaTweetActionInfo"), - ( - "ReportImpressionIsNotAdapted", - ceTweetReportFlow(page = "report_abuse", action = "impression"), - Some(ReportDetails(reportFlowId = Some(reportFlowId))), - None, - None - ), - ( - "ReportSubmitIsAdapted", - ceTweetReportFlow(page = "report_abuse", action = "submit"), - Some( - ReportDetails( - reportFlowId = Some(reportFlowId), - reportType = Some(ReportType.Abuse))), - Some(uuaTweetReportFlow(page = "report_abuse", action = "submit")), - Some(reportTweetSubmit) - ), - ) - - forEvery(params) { - ( - _: String, - ceNamespace: EventNamespace, - ceReportDetails: Option[ReportDetails], - uuaNamespace: Option[ClientEventNamespace], - uuaTweetActionInfo: Option[TweetActionInfo] - ) => - val actual = ClientEventAdapter.adaptEvent( - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceNamespace), - reportDetails = ceReportDetails)) - - val expectedUUA = - if (ceNamespace.action.contains("submit")) - Seq( - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = uuaNamespace, - actionType = ActionType.ServerTweetReport, - tweetActionInfo = uuaTweetActionInfo - )) - else Nil - - assert(expectedUUA === actual) - } - } - } - } - - // Tests for ClientNotificationOpen - test("ClientNotificationOpen") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvent = - pushNotificationEvent( - eventNamespace = Some(ceNotificationOpen), - notificationDetails = Some(notificationDetails)) - - val expectedUUA = mkExpectedUUAForNotificationEvent( - clientEventNamespace = Some(uuaNotificationOpen), - actionType = ActionType.ClientNotificationOpen, - notificationContent = tweetNotificationContent, - productSurface = Some(ProductSurface.PushNotification), - productSurfaceInfo = Some( - ProductSurfaceInfo.PushNotificationInfo( - PushNotificationInfo(notificationId = notificationId))) - ) - - val actual = ClientEventAdapter.adaptEvent(clientEvent) - assert(Seq(expectedUUA) === actual) - } - } - } - - // Tests for ClientNotificationClick - test("ClientNotificationClick") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val params = Table( - ("notificationType", "ceNotificationTabDetails", "uuaNotificationContent"), - ("tweetNotification", notificationTabTweetEventDetails, tweetNotificationContent), - ( - "multiTweetNotification", - notificationTabMultiTweetEventDetails, - multiTweetNotificationContent), - ( - "unknownNotification", - notificationTabUnknownEventDetails, - unknownNotificationContent - ), - ) - - forEvery(params) { - ( - _: String, - ceNotificationTabDetails: NotificationTabDetails, - uuaNotificationContent: NotificationContent - ) => - val actual = ClientEventAdapter.adaptEvent( - actionTowardNotificationEvent( - eventNamespace = Some(ceNotificationClick), - notificationTabDetails = Some(ceNotificationTabDetails))) - - val expectedUUA = mkExpectedUUAForNotificationEvent( - clientEventNamespace = Some(uuaNotificationClick), - actionType = ActionType.ClientNotificationClick, - notificationContent = uuaNotificationContent, - productSurface = Some(ProductSurface.NotificationTab), - productSurfaceInfo = Some(notificationTabProductSurfaceInfo) - ) - - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientNotificationSeeLessOften - test("ClientNotificationSeeLessOften") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val params = Table( - ("notificationType", "ceNotificationTabDetails", "uuaNotificationContent"), - ("tweetNotification", notificationTabTweetEventDetails, tweetNotificationContent), - ( - "multiTweetNotification", - notificationTabMultiTweetEventDetails, - multiTweetNotificationContent), - ("unknownNotification", notificationTabUnknownEventDetails, unknownNotificationContent), - ) - - forEvery(params) { - ( - _: String, - ceNotificationTabDetails: NotificationTabDetails, - uuaNotificationContent: NotificationContent - ) => - val actual = ClientEventAdapter.adaptEvent( - actionTowardNotificationEvent( - eventNamespace = Some(ceNotificationSeeLessOften), - notificationTabDetails = Some(ceNotificationTabDetails))) - - val expectedUUA = mkExpectedUUAForNotificationEvent( - clientEventNamespace = Some(uuaNotificationSeeLessOften), - actionType = ActionType.ClientNotificationSeeLessOften, - notificationContent = uuaNotificationContent, - productSurface = Some(ProductSurface.NotificationTab), - productSurfaceInfo = Some(notificationTabProductSurfaceInfo) - ) - - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetClick - test("ClientTweetClick") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val params = Table( - ("eventName", "page", "nTabDetails", "uuaProductSurface", "uuaProductSurfaceInfo"), - ("tweetClick", "messages", None, None, None), - ( - "tweetClickInNTab", - "ntab", - Some(notificationTabTweetEventDetails), - Some(ProductSurface.NotificationTab), - Some(notificationTabProductSurfaceInfo)) - ) - - forEvery(params) { - ( - _: String, - page: String, - notificationTabDetails: Option[NotificationTabDetails], - uuaProductSurface: Option[ProductSurface], - uuaProductSurfaceInfo: Option[ProductSurfaceInfo] - ) => - val actual = ClientEventAdapter.adaptEvent( - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceTweetClick.copy(page = Some(page))), - notificationTabDetails = notificationTabDetails)) - - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaTweetClick.copy(page = Some(page))), - actionType = ActionType.ClientTweetClick, - productSurface = uuaProductSurface, - productSurfaceInfo = uuaProductSurfaceInfo - ) - - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetClickProfile - test("ClientTweetClickProfile") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actual = - ClientEventAdapter.adaptEvent( - profileClickEvent(eventNamespace = Some(ceTweetClickProfile))) - - val expectedUUA = mkExpectedUUAForProfileClick( - clientEventNamespace = Some(uuaTweetClickProfile), - actionType = ActionType.ClientTweetClickProfile, - authorInfo = Some( - AuthorInfo( - authorId = Some(authorId) - ))) - assert(Seq(expectedUUA) === actual) - } - } - } - - // Tests for ClientTweetClickShare - test("ClientTweetClickShare") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actual = - ClientEventAdapter.adaptEvent( - actionTowardDefaultTweetEvent( - eventNamespace = Some(EventNamespace(action = Some("share_menu_click"))), - authorId = Some(authorId), - tweetPosition = Some(1), - promotedId = Some("promted_123") - )) - - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(ClientEventNamespace(action = Some("share_menu_click"))), - actionType = ActionType.ClientTweetClickShare, - authorInfo = Some( - AuthorInfo( - authorId = Some(authorId) - )), - tweetPosition = Some(1), - promotedId = Some("promted_123") - ) - assert(Seq(expectedUUA) === actual) - } - } - } - - // Tests for ClientTweetShareVia* and ClientTweetUnbookmark - test("ClientTweetShareVia and Unbookmark") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val input = Table( - ("eventNamespaceAction", "uuaActionTypes"), - ("bookmark", Seq(ActionType.ClientTweetShareViaBookmark, ActionType.ClientTweetBookmark)), - ("copy_link", Seq(ActionType.ClientTweetShareViaCopyLink)), - ("share_via_dm", Seq(ActionType.ClientTweetClickSendViaDirectMessage)), - ("unbookmark", Seq(ActionType.ClientTweetUnbookmark)) - ) - - forEvery(input) { (eventNamespaceAction: String, uuaActionTypes: Seq[ActionType]) => - val actual: Seq[UnifiedUserAction] = - ClientEventAdapter.adaptEvent( - actionTowardDefaultTweetEvent( - eventNamespace = Some(EventNamespace(action = Some(eventNamespaceAction))), - authorId = Some(authorId))) - - implicit def any2iterable[A](a: A): Iterable[A] = Some(a) - val expectedUUA: Seq[UnifiedUserAction] = uuaActionTypes.flatMap { uuaActionType => - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = - Some(ClientEventNamespace(action = Some(eventNamespaceAction))), - actionType = uuaActionType, - authorInfo = Some( - AuthorInfo( - authorId = Some(authorId) - )) - ) - } - assert(expectedUUA === actual) - } - } - } - } - - // Test for ClientTweetClickHashtag - test("ClientTweetClickHashtag") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val events = Table( - ("targets", "tweetActionInfo"), - ( - Some(Seq(LogEventItem(name = Some("test_hashtag")))), - Some( - TweetActionInfo.ClientTweetClickHashtag( - ClientTweetClickHashtag(hashtag = Some("test_hashtag"))))), - ( - Some(Seq.empty[LogEventItem]), - Some(TweetActionInfo.ClientTweetClickHashtag(ClientTweetClickHashtag(hashtag = None)))), - ( - Some(Nil), - Some(TweetActionInfo.ClientTweetClickHashtag(ClientTweetClickHashtag(hashtag = None)))), - ( - None, - Some(TweetActionInfo.ClientTweetClickHashtag(ClientTweetClickHashtag(hashtag = None)))) - ) - forEvery(events) { - (targets: Option[Seq[LogEventItem]], tweetActionInfo: Option[TweetActionInfo]) => - val clientEvent = - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceClickHashtag), - targets = targets) - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaClickHashtagClientEventNamespace), - actionType = ActionType.ClientTweetClickHashtag, - tweetActionInfo = tweetActionInfo - ) - assert(Seq(expectedUUA) === ClientEventAdapter.adaptEvent(clientEvent)) - } - - } - } - } - - // Tests for ClientTweetVideoPlaybackStart and ClientTweetVideoPlaybackComplete - test("Client Tweet Video Playback Start and Complete") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val input = Table( - ("ceNamespace", "uuaNamespace", "uuaActionType"), - ( - ceVideoPlaybackStart, - uuaVideoPlaybackStartClientEventNamespace, - ActionType.ClientTweetVideoPlaybackStart), - ( - ceVideoPlaybackComplete, - uuaVideoPlaybackCompleteClientEventNamespace, - ActionType.ClientTweetVideoPlaybackComplete), - ) - for (element <- videoEventElementValues) { - forEvery(input) { - ( - ceNamespace: EventNamespace, - uuaNamespace: ClientEventNamespace, - uuaActionType: ActionType - ) => - val clientEvent = actionTowardDefaultTweetEvent( - eventNamespace = Some(ceNamespace.copy(element = Some(element))), - mediaDetailsV2 = Some(mediaDetailsV2), - clientMediaEvent = Some(clientMediaEvent), - cardDetails = Some(cardDetails) - ) - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaNamespace.copy(element = Some(element))), - actionType = uuaActionType, - tweetActionInfo = Some(videoMetadata) - ) - assert(ClientEventAdapter.adaptEvent(clientEvent).contains(expectedUUA)) - } - } - - for (element <- invalidVideoEventElementValues) { - forEvery(input) { - ( - ceNamespace: EventNamespace, - uuaNamespace: ClientEventNamespace, - uuaActionType: ActionType - ) => - val clientEvent = actionTowardDefaultTweetEvent( - eventNamespace = Some(ceNamespace.copy(element = Some(element))), - mediaDetailsV2 = Some(mediaDetailsV2), - clientMediaEvent = Some(clientMediaEvent) - ) - val unexpectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaNamespace.copy(element = Some(element))), - actionType = uuaActionType, - tweetActionInfo = Some(videoMetadata) - ) - assert(!ClientEventAdapter.adaptEvent(clientEvent).contains(unexpectedUUA)) - } - } - } - } - } - - // Tests for ClientTweetNotRelevant and ClientTweetUndoNotRelevant - test("ClientTweetNotRelevant & UndoNotRelevant") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actions = Table(("action"), "click", "undo") - val element = "feedback_notrelevant" - forEvery(actions) { action => - val clientEvent = - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceEventNamespace(element, action)), - ) - - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaClientEventNamespace(element, action)), - actionType = action match { - case "click" => ActionType.ClientTweetNotRelevant - case "undo" => ActionType.ClientTweetUndoNotRelevant - } - ) - - val actual = ClientEventAdapter.adaptEvent(clientEvent) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientNotificationDismiss - test("ClientNotificationDismiss") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvent = - pushNotificationEvent( - eventNamespace = Some(ceNotificationDismiss), - notificationDetails = Some(notificationDetails)) - - val expectedUUA = mkExpectedUUAForNotificationEvent( - clientEventNamespace = Some(uuaNotificationDismiss), - actionType = ActionType.ClientNotificationDismiss, - notificationContent = tweetNotificationContent, - productSurface = Some(ProductSurface.PushNotification), - productSurfaceInfo = Some( - ProductSurfaceInfo.PushNotificationInfo( - PushNotificationInfo(notificationId = notificationId))) - ) - - val actual = ClientEventAdapter.adaptEvent(clientEvent) - assert(Seq(expectedUUA) === actual) - } - } - } - - // Tests for ClientTypeaheadClick - test("ClientTypeaheadClick") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val searchQuery = "searchQuery" - - val input = Table( - ("clientEventTargets", "typeaheadActionInfo"), - ( - Some(Seq(LogEventItem(id = Some(userId), itemType = Some(ItemType.User)))), - TypeaheadActionInfo.UserResult(UserResult(profileId = userId))), - ( - Some(Seq(LogEventItem(name = Some(s"$searchQuery"), itemType = Some(ItemType.Search)))), - TypeaheadActionInfo.TopicQueryResult( - TopicQueryResult(suggestedTopicQuery = s"$searchQuery"))) - ) - forEvery(input) { - ( - clientEventTargets: Option[Seq[LogEventItem]], - typeaheadActionInfo: TypeaheadActionInfo, - ) => - val clientEvent = - actionTowardsTypeaheadEvent( - eventNamespace = Some(ceTypeaheadClick), - targets = clientEventTargets, - searchQuery = searchQuery) - val expectedUUA = mkExpectedUUAForTypeaheadAction( - clientEventNamespace = Some(uuaTypeaheadClick), - actionType = ActionType.ClientTypeaheadClick, - typeaheadActionInfo = typeaheadActionInfo, - searchQuery = searchQuery - ) - val actual = ClientEventAdapter.adaptEvent(clientEvent) - assert(Seq(expectedUUA) === actual) - } - // Testing invalid target item type case - assert( - Seq() === ClientEventAdapter.adaptEvent( - actionTowardsTypeaheadEvent( - eventNamespace = Some(ceTypeaheadClick), - targets = - Some(Seq(LogEventItem(id = Some(itemTweetId), itemType = Some(ItemType.Tweet)))), - searchQuery = searchQuery))) - } - } - } - - // Tests for ClientFeedbackPromptSubmit - test("ClientFeedbackPromptSubmit") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val searchQuery: String = "searchQuery" - val searchDetails = Some(SearchDetails(query = Some(searchQuery))) - val input = Table( - ("logEvent", "uuaNamespace", "uuaActionType", "FeedbackPromptInfo"), - ( - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceTweetRelevantToSearch), - searchDetails = searchDetails - ), - uuaTweetRelevantToSearch, - ActionType.ClientFeedbackPromptSubmit, - FeedbackPromptInfo(feedbackPromptActionInfo = - FeedbackPromptActionInfo.TweetRelevantToSearch( - TweetRelevantToSearch( - searchQuery = searchQuery, - tweetId = itemTweetId, - isRelevant = Some(true))))), - ( - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceTweetNotRelevantToSearch), - searchDetails = searchDetails - ), - uuaTweetNotRelevantToSearch, - ActionType.ClientFeedbackPromptSubmit, - FeedbackPromptInfo(feedbackPromptActionInfo = - FeedbackPromptActionInfo.TweetRelevantToSearch( - TweetRelevantToSearch( - searchQuery = searchQuery, - tweetId = itemTweetId, - isRelevant = Some(false))))), - ( - actionTowardSearchResultPageEvent( - eventNamespace = Some(ceSearchResultsRelevant), - searchDetails = searchDetails, - items = Some(Seq(LogEventItem(itemType = Some(ItemType.RelevancePrompt)))) - ), - uuaSearchResultsRelevant, - ActionType.ClientFeedbackPromptSubmit, - FeedbackPromptInfo(feedbackPromptActionInfo = - FeedbackPromptActionInfo.DidYouFindItSearch( - DidYouFindItSearch(searchQuery = searchQuery, isRelevant = Some(true))))), - ( - actionTowardSearchResultPageEvent( - eventNamespace = Some(ceSearchResultsNotRelevant), - searchDetails = searchDetails, - items = Some(Seq(LogEventItem(itemType = Some(ItemType.RelevancePrompt)))) - ), - uuaSearchResultsNotRelevant, - ActionType.ClientFeedbackPromptSubmit, - FeedbackPromptInfo(feedbackPromptActionInfo = - FeedbackPromptActionInfo.DidYouFindItSearch( - DidYouFindItSearch(searchQuery = searchQuery, isRelevant = Some(false))))) - ) - - forEvery(input) { - ( - logEvent: LogEvent, - uuaNamespace: ClientEventNamespace, - uuaActionType: ActionType, - feedbackPromptInfo: FeedbackPromptInfo - ) => - val actual = - ClientEventAdapter.adaptEvent(logEvent) - val expectedUUA = mkExpectedUUAForFeedbackSubmitAction( - clientEventNamespace = Some(uuaNamespace), - actionType = uuaActionType, - feedbackPromptInfo = feedbackPromptInfo, - searchQuery = searchQuery) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientProfile* - test("ClientProfile*") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val input = Table( - ("eventName", "ceNamespace", "uuaNamespace", "uuaActionType"), - ("profile_block", ceProfileBlock, uuaProfileBlock, ActionType.ClientProfileBlock), - ("profile_unblock", ceProfileUnblock, uuaProfileUnblock, ActionType.ClientProfileUnblock), - ("profile_mute", ceProfileMute, uuaProfileMute, ActionType.ClientProfileMute), - ("profile_report", ceProfileReport, uuaProfileReport, ActionType.ClientProfileReport), - ("profile_follow", ceProfileFollow, uuaProfileFollow, ActionType.ClientProfileFollow), - ("profile_click", ceProfileClick, uuaProfileClick, ActionType.ClientProfileClick), - ( - "profile_follow_attempt", - ceProfileFollowAttempt, - uuaProfileFollowAttempt, - ActionType.ClientProfileFollowAttempt), - ("profile_show", ceProfileShow, uuaProfileShow, ActionType.ClientProfileShow), - ) - forEvery(input) { - ( - eventName: String, - ceNamespace: EventNamespace, - uuaNamespace: ClientEventNamespace, - uuaActionType: ActionType - ) => - val actual = - ClientEventAdapter.adaptEvent( - actionTowardProfileEvent( - eventName = eventName, - eventNamespace = Some(ceNamespace) - )) - val expectedUUA = mkExpectedUUAForProfileAction( - clientEventNamespace = Some(uuaNamespace), - actionType = uuaActionType, - actionProfileId = itemProfileId) - assert(Seq(expectedUUA) === actual) - } - } - } - } - // Tests for ClientTweetEngagementAttempt - test("ClientTweetEngagementAttempt") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("eventName", "ceNamespace", "uuaNamespace", "uuaActionType"), - ( - "tweet_favourite_attempt", - ceTweetFavoriteAttempt, - uuaTweetFavoriteAttempt, - ActionType.ClientTweetFavoriteAttempt), - ( - "tweet_retweet_attempt", - ceTweetRetweetAttempt, - uuaTweetRetweetAttempt, - ActionType.ClientTweetRetweetAttempt), - ( - "tweet_reply_attempt", - ceTweetReplyAttempt, - uuaTweetReplyAttempt, - ActionType.ClientTweetReplyAttempt), - ) - forEvery(clientEvents) { - ( - eventName: String, - ceNamespace: EventNamespace, - uuaNamespace: ClientEventNamespace, - uuaActionType: ActionType - ) => - val actual = - ClientEventAdapter.adaptEvent(actionTowardDefaultTweetEvent(Some(ceNamespace))) - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaNamespace), - actionType = uuaActionType) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for LoggedOut for ClientLogin* - test("ClientLogin*") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("eventName", "ceNamespace", "uuaNamespace", "uuaActionType"), - ( - "client_click_login", - ceClientCTALoginClick, - uuaClientCTALoginClick, - ActionType.ClientCTALoginClick), - ( - "client_click_show", - ceClientCTALoginStart, - uuaClientCTALoginStart, - ActionType.ClientCTALoginStart), - ( - "client_login_success", - ceClientCTALoginSuccess, - uuaClientCTALoginSuccess, - ActionType.ClientCTALoginSuccess), - ) - - forEvery(clientEvents) { - ( - eventName: String, - ceNamespace: EventNamespace, - uuaNamespace: ClientEventNamespace, - uuaActionType: ActionType - ) => - val actual = - ClientEventAdapter.adaptEvent( - mkLogEvent( - eventName, - Some(ceNamespace), - logBase = Some(logBase1), - eventDetails = None, - pushNotificationDetails = None, - reportDetails = None, - searchDetails = None)) - val expectedUUA = mkExpectedUUAForActionTowardCTAEvent( - clientEventNamespace = Some(uuaNamespace), - actionType = uuaActionType, - guestIdMarketingOpt = logBase1.guestIdMarketing - ) - - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for LoggedOut for ClientSignup* - test("ClientSignup*") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("eventName", "ceNamespace", "uuaNamespace", "uuaActionType"), - ( - "client_click_signup", - ceClientCTASignupClick, - uuaClientCTASignupClick, - ActionType.ClientCTASignupClick), - ( - "client_signup_success", - ceClientCTASignupSuccess, - uuaClientCTASignupSuccess, - ActionType.ClientCTASignupSuccess), - ) - - forEvery(clientEvents) { - ( - eventName: String, - ceNamespace: EventNamespace, - uuaNamespace: ClientEventNamespace, - uuaActionType: ActionType - ) => - val actual = - ClientEventAdapter.adaptEvent( - mkLogEvent( - eventName, - Some(ceNamespace), - logBase = Some(logBase1), - eventDetails = None, - pushNotificationDetails = None, - reportDetails = None, - searchDetails = None)) - val expectedUUA = mkExpectedUUAForActionTowardCTAEvent( - clientEventNamespace = Some(uuaNamespace), - actionType = uuaActionType, - guestIdMarketingOpt = logBase1.guestIdMarketing - ) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetFollowAuthor - test("ClientTweetFollowAuthor") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val testEventsList = Seq( - (ceTweetFollowAuthor1, uuaTweetFollowAuthor1, TweetAuthorFollowClickSource.CaretMenu), - (ceTweetFollowAuthor2, uuaTweetFollowAuthor2, TweetAuthorFollowClickSource.ProfileImage) - ) - testEventsList.foreach { - case (eventNamespace, clientEventNamespace, followClickSource) => - val actual = - ClientEventAdapter.adaptEvent( - tweetActionTowardAuthorEvent( - eventName = "tweet_follow_author", - eventNamespace = Some(eventNamespace) - )) - val expectedUUA = mkExpectedUUAForTweetActionTowardAuthor( - clientEventNamespace = Some(clientEventNamespace), - actionType = ActionType.ClientTweetFollowAuthor, - authorInfo = Some( - AuthorInfo( - authorId = Some(authorId) - )), - tweetActionInfo = Some( - TweetActionInfo.ClientTweetFollowAuthor( - ClientTweetFollowAuthor(followClickSource) - )) - ) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetUnfollowAuthor - test("ClientTweetUnfollowAuthor") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val testEventsList = Seq( - ( - ceTweetUnfollowAuthor1, - uuaTweetUnfollowAuthor1, - TweetAuthorUnfollowClickSource.CaretMenu), - ( - ceTweetUnfollowAuthor2, - uuaTweetUnfollowAuthor2, - TweetAuthorUnfollowClickSource.ProfileImage) - ) - testEventsList.foreach { - case (eventNamespace, clientEventNamespace, unfollowClickSource) => - val actual = - ClientEventAdapter.adaptEvent( - tweetActionTowardAuthorEvent( - eventName = "tweet_unfollow_author", - eventNamespace = Some(eventNamespace) - )) - val expectedUUA = mkExpectedUUAForTweetActionTowardAuthor( - clientEventNamespace = Some(clientEventNamespace), - actionType = ActionType.ClientTweetUnfollowAuthor, - authorInfo = Some( - AuthorInfo( - authorId = Some(authorId) - )), - tweetActionInfo = Some( - TweetActionInfo.ClientTweetUnfollowAuthor( - ClientTweetUnfollowAuthor(unfollowClickSource) - )) - ) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - // Tests for ClientTweetMuteAuthor - test("ClientTweetMuteAuthor") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actual = - ClientEventAdapter.adaptEvent( - tweetActionTowardAuthorEvent( - eventName = "tweet_mute_author", - eventNamespace = Some(ceTweetMuteAuthor) - )) - - val expectedUUA = mkExpectedUUAForTweetActionTowardAuthor( - clientEventNamespace = Some(uuaTweetMuteAuthor), - actionType = ActionType.ClientTweetMuteAuthor, - authorInfo = Some( - AuthorInfo( - authorId = Some(authorId) - ))) - assert(Seq(expectedUUA) === actual) - } - } - } - - // Tests for ClientTweetBlockAuthor - test("ClientTweetBlockAuthor") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actual = - ClientEventAdapter.adaptEvent( - tweetActionTowardAuthorEvent( - eventName = "tweet_block_author", - eventNamespace = Some(ceTweetBlockAuthor) - )) - - val expectedUUA = mkExpectedUUAForTweetActionTowardAuthor( - clientEventNamespace = Some(uuaTweetBlockAuthor), - actionType = ActionType.ClientTweetBlockAuthor, - authorInfo = Some( - AuthorInfo( - authorId = Some(authorId) - ))) - assert(Seq(expectedUUA) === actual) - } - } - } - - // Tests for ClientTweetUnblockAuthor - test("ClientTweetUnblockAuthor") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actual = - ClientEventAdapter.adaptEvent( - tweetActionTowardAuthorEvent( - eventName = "tweet_unblock_author", - eventNamespace = Some(ceTweetUnblockAuthor) - )) - - val expectedUUA = mkExpectedUUAForTweetActionTowardAuthor( - clientEventNamespace = Some(uuaTweetUnblockAuthor), - actionType = ActionType.ClientTweetUnblockAuthor, - authorInfo = Some( - AuthorInfo( - authorId = Some(authorId) - ))) - assert(Seq(expectedUUA) === actual) - } - } - } - - // Test for ClientTweetOpenLink - test("ClientTweetOpenLink") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val input = Table( - ("url", "tweetActionInfo"), - (Some("go/url"), clientOpenLinkWithUrl), - (None, clientOpenLinkWithoutUrl) - ) - - forEvery(input) { (url: Option[String], tweetActionInfo: TweetActionInfo) => - val clientEvent = - actionTowardDefaultTweetEvent(eventNamespace = Some(ceOpenLink), url = url) - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaOpenLinkClientEventNamespace), - actionType = ActionType.ClientTweetOpenLink, - tweetActionInfo = Some(tweetActionInfo) - ) - assert(Seq(expectedUUA) === ClientEventAdapter.adaptEvent(clientEvent)) - } - } - } - } - - // Test for ClientTweetTakeScreenshot - test("Client take screenshot") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvent = - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceTakeScreenshot), - percentVisibleHeight100k = Some(100)) - val expectedUUA = mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaTakeScreenshotClientEventNamespace), - actionType = ActionType.ClientTweetTakeScreenshot, - tweetActionInfo = Some(clientTakeScreenshot) - ) - assert(Seq(expectedUUA) === ClientEventAdapter.adaptEvent(clientEvent)) - } - } - } - - test("Home / Search product surface meta data") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val clientEvents = Table( - ("actionTweetType", "clientEvent", "expectedUUAEvent"), - ( - "homeTweetEventWithControllerData", - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceHomeFavoriteEventNamespace), - suggestionDetails = Some( - SuggestionDetails(decodedControllerData = Some( - homeTweetControllerDataV2( - injectedPosition = Some(1), - traceId = Some(traceId), - requestJoinId = Some(requestJoinId) - )))) - ), - expectedHomeTweetEventWithControllerData), - ( - "homeTweetEventWithSuggestionType", - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceHomeFavoriteEventNamespace), - suggestionDetails = Some( - SuggestionDetails( - suggestionType = Some("Test_type") - ))), - expectedHomeTweetEventWithSuggestType), - ( - "homeTweetEventWithControllerDataSuggestionType", - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceHomeFavoriteEventNamespace), - suggestionDetails = Some( - SuggestionDetails( - suggestionType = Some("Test_type"), - decodedControllerData = Some( - homeTweetControllerDataV2( - injectedPosition = Some(1), - traceId = Some(traceId), - requestJoinId = Some(requestJoinId))) - )) - ), - expectedHomeTweetEventWithControllerDataSuggestType), - ( - "homeLatestTweetEventWithControllerData", - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceHomeLatestFavoriteEventNamespace), - suggestionDetails = Some( - SuggestionDetails(decodedControllerData = Some( - homeTweetControllerDataV2( - injectedPosition = Some(1), - traceId = Some(traceId), - requestJoinId = Some(requestJoinId) - )))) - ), - expectedHomeLatestTweetEventWithControllerData), - ( - "homeLatestTweetEventWithSuggestionType", - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceHomeLatestFavoriteEventNamespace), - suggestionDetails = Some( - SuggestionDetails( - suggestionType = Some("Test_type") - ))), - expectedHomeLatestTweetEventWithSuggestType), - ( - "homeLatestTweetEventWithControllerDataSuggestionType", - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceHomeLatestFavoriteEventNamespace), - suggestionDetails = Some( - SuggestionDetails( - suggestionType = Some("Test_type"), - decodedControllerData = Some( - homeTweetControllerDataV2( - injectedPosition = Some(1), - traceId = Some(traceId), - requestJoinId = Some(requestJoinId))) - )) - ), - expectedHomeLatestTweetEventWithControllerDataSuggestType), - ( - "searchTweetEventWithControllerData", - actionTowardDefaultTweetEvent( - eventNamespace = Some(ceSearchFavoriteEventNamespace), - suggestionDetails = Some( - SuggestionDetails(decodedControllerData = Some( - mkSearchResultControllerData( - queryOpt = Some("twitter"), - traceId = Some(traceId), - requestJoinId = Some(requestJoinId) - )))) - ), - expectedSearchTweetEventWithControllerData), - ) - forEvery(clientEvents) { (_: String, event: LogEvent, expectedUUA: UnifiedUserAction) => - val actual = ClientEventAdapter.adaptEvent(event) - assert(Seq(expectedUUA) === actual) - } - } - } - } - - test("ClientAppExit") { - new TestFixtures.ClientEventFixture { - Time.withTimeAt(frozenTime) { _ => - val duration: Option[Long] = Some(10000L) - val inputTable = Table( - ("eventType", "clientAppId", "section", "duration", "isValidEvent"), - ("uas-iPhone", Some(129032L), Some("enter_background"), duration, true), - ("uas-iPad", Some(191841L), Some("enter_background"), duration, true), - ("uas-android", Some(258901L), None, duration, true), - ("none-clientId", None, None, duration, false), - ("invalid-clientId", Some(1L), None, duration, false), - ("none-duration", Some(258901L), None, None, false), - ("non-uas-iPhone", Some(129032L), None, duration, false) - ) - - forEvery(inputTable) { - ( - _: String, - clientAppId: Option[Long], - section: Option[String], - duration: Option[Long], - isValidEvent: Boolean - ) => - val actual = ClientEventAdapter.adaptEvent( - actionTowardsUasEvent( - eventNamespace = Some(ceAppExit.copy(section = section)), - clientAppId = clientAppId, - duration = duration - )) - - if (isValidEvent) { - // create UUA UAS event - val expectedUUA = mkExpectedUUAForUasEvent( - clientEventNamespace = Some(uuaAppExit.copy(section = section)), - actionType = ActionType.ClientAppExit, - clientAppId = clientAppId, - duration = duration - ) - assert(Seq(expectedUUA) === actual) - } else { - // ignore the event and do not create UUA UAS event - assert(actual.isEmpty) - } - } - } - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventAdapterSpec.docx new file mode 100644 index 000000000..df50ea918 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventAdapterSpec.scala deleted file mode 100644 index 5d00f0d8b..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventAdapterSpec.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.inject.Test -import com.twitter.unified_user_actions.adapter.TestFixtures.EmailNotificationEventFixture -import com.twitter.unified_user_actions.adapter.email_notification_event.EmailNotificationEventAdapter -import com.twitter.util.Time - -class EmailNotificationEventAdapterSpec extends Test { - - test("Notifications with click event") { - new EmailNotificationEventFixture { - Time.withTimeAt(frozenTime) { _ => - val actual = EmailNotificationEventAdapter.adaptEvent(notificationEvent) - assert(expectedUua == actual.head) - assert(EmailNotificationEventAdapter.adaptEvent(notificationEventWOTweetId).isEmpty) - assert(EmailNotificationEventAdapter.adaptEvent(notificationEventWOImpressionId).isEmpty) - } - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventUtilsSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventUtilsSpec.docx new file mode 100644 index 000000000..793b69f93 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventUtilsSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventUtilsSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventUtilsSpec.scala deleted file mode 100644 index b99dc892c..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/EmailNotificationEventUtilsSpec.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.inject.Test -import com.twitter.unified_user_actions.adapter.TestFixtures.EmailNotificationEventFixture -import com.twitter.unified_user_actions.adapter.email_notification_event.EmailNotificationEventUtils - -class EmailNotificationEventUtilsSpec extends Test { - - test("Extract TweetId from pageUrl") { - new EmailNotificationEventFixture { - - val invalidUrls: Seq[String] = - List("", "abc.com/what/not?x=y", "?abc=def", "12345/", "12345/?") - val invalidDomain = "https://twitter.app.link/addressbook/" - val numericHandle = - "https://twitter.com/1234/status/12345?cxt=HBwWgsDTgY3tp&cn=ZmxleGl&refsrc=email)" - - assert(EmailNotificationEventUtils.extractTweetId(pageUrlStatus).contains(tweetIdStatus)) - assert(EmailNotificationEventUtils.extractTweetId(pageUrlEvent).contains(tweetIdEvent)) - assert(EmailNotificationEventUtils.extractTweetId(pageUrlNoArgs).contains(tweetIdNoArgs)) - assert(EmailNotificationEventUtils.extractTweetId(invalidDomain).isEmpty) - assert(EmailNotificationEventUtils.extractTweetId(numericHandle).contains(12345L)) - invalidUrls.foreach(url => assert(EmailNotificationEventUtils.extractTweetId(url).isEmpty)) - } - } - - test("Extract TweetId from LogBase") { - new EmailNotificationEventFixture { - assert(EmailNotificationEventUtils.extractTweetId(logBase1).contains(tweetIdStatus)) - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/FavoriteArchivalEventsAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/FavoriteArchivalEventsAdapterSpec.docx new file mode 100644 index 000000000..ce1641d47 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/FavoriteArchivalEventsAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/FavoriteArchivalEventsAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/FavoriteArchivalEventsAdapterSpec.scala deleted file mode 100644 index 69e670172..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/FavoriteArchivalEventsAdapterSpec.scala +++ /dev/null @@ -1,132 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.inject.Test -import com.twitter.timelineservice.fanout.thriftscala.FavoriteArchivalEvent -import com.twitter.unified_user_actions.adapter.favorite_archival_events.FavoriteArchivalEventsAdapter -import com.twitter.unified_user_actions.thriftscala._ -import com.twitter.util.Time -import org.scalatest.prop.TableDrivenPropertyChecks - -class FavoriteArchivalEventsAdapterSpec extends Test with TableDrivenPropertyChecks { - trait Fixture { - - val frozenTime = Time.fromMilliseconds(1658949273000L) - - val userId = 1L - val authorId = 2L - val tweetId = 101L - val retweetId = 102L - - val favArchivalEventNoRetweet = FavoriteArchivalEvent( - favoriterId = userId, - tweetId = tweetId, - timestampMs = 0L, - isArchivingAction = Some(true), - tweetUserId = Some(authorId) - ) - val favArchivalEventRetweet = FavoriteArchivalEvent( - favoriterId = userId, - tweetId = retweetId, - timestampMs = 0L, - isArchivingAction = Some(true), - tweetUserId = Some(authorId), - sourceTweetId = Some(tweetId) - ) - val favUnarchivalEventNoRetweet = FavoriteArchivalEvent( - favoriterId = userId, - tweetId = tweetId, - timestampMs = 0L, - isArchivingAction = Some(false), - tweetUserId = Some(authorId) - ) - val favUnarchivalEventRetweet = FavoriteArchivalEvent( - favoriterId = userId, - tweetId = retweetId, - timestampMs = 0L, - isArchivingAction = Some(false), - tweetUserId = Some(authorId), - sourceTweetId = Some(tweetId) - ) - - val expectedUua1 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(authorId))), - ) - ), - actionType = ActionType.ServerTweetArchiveFavorite, - eventMetadata = EventMetadata( - sourceTimestampMs = 0L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerFavoriteArchivalEvents, - ) - ) - val expectedUua2 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = retweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(authorId))), - retweetedTweetId = Some(tweetId) - ) - ), - actionType = ActionType.ServerTweetArchiveFavorite, - eventMetadata = EventMetadata( - sourceTimestampMs = 0L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerFavoriteArchivalEvents, - ) - ) - val expectedUua3 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(authorId))), - ) - ), - actionType = ActionType.ServerTweetUnarchiveFavorite, - eventMetadata = EventMetadata( - sourceTimestampMs = 0L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerFavoriteArchivalEvents, - ) - ) - val expectedUua4 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = retweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(authorId))), - retweetedTweetId = Some(tweetId) - ) - ), - actionType = ActionType.ServerTweetUnarchiveFavorite, - eventMetadata = EventMetadata( - sourceTimestampMs = 0L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerFavoriteArchivalEvents, - ) - ) - } - - test("all tests") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val table = Table( - ("event", "expected"), - (favArchivalEventNoRetweet, expectedUua1), - (favArchivalEventRetweet, expectedUua2), - (favUnarchivalEventNoRetweet, expectedUua3), - (favUnarchivalEventRetweet, expectedUua4) - ) - forEvery(table) { (event: FavoriteArchivalEvent, expected: UnifiedUserAction) => - val actual = FavoriteArchivalEventsAdapter.adaptEvent(event) - assert(Seq(expected) === actual) - } - } - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RekeyUuaFromInteractionEventsAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RekeyUuaFromInteractionEventsAdapterSpec.docx new file mode 100644 index 000000000..ae2570ca4 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RekeyUuaFromInteractionEventsAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RekeyUuaFromInteractionEventsAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RekeyUuaFromInteractionEventsAdapterSpec.scala deleted file mode 100644 index 93b741b79..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RekeyUuaFromInteractionEventsAdapterSpec.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.inject.Test -import com.twitter.unified_user_actions.adapter.TestFixtures.InteractionEventsFixtures -import com.twitter.unified_user_actions.adapter.uua_aggregates.RekeyUuaFromInteractionEventsAdapter -import com.twitter.util.Time -import org.scalatest.prop.TableDrivenPropertyChecks - -class RekeyUuaFromInteractionEventsAdapterSpec extends Test with TableDrivenPropertyChecks { - test("ClientTweetRenderImpressions") { - new InteractionEventsFixtures { - Time.withTimeAt(frozenTime) { _ => - assert( - RekeyUuaFromInteractionEventsAdapter.adaptEvent(baseInteractionEvent) === Seq( - expectedBaseKeyedUuaTweet)) - } - } - } - - test("Filter out logged out users") { - new InteractionEventsFixtures { - Time.withTimeAt(frozenTime) { _ => - assert(RekeyUuaFromInteractionEventsAdapter.adaptEvent(loggedOutInteractionEvent) === Nil) - } - } - } - - test("Filter out detail impressions") { - new InteractionEventsFixtures { - Time.withTimeAt(frozenTime) { _ => - assert( - RekeyUuaFromInteractionEventsAdapter.adaptEvent(detailImpressionInteractionEvent) === Nil) - } - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RetweetArchivalEventsAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RetweetArchivalEventsAdapterSpec.docx new file mode 100644 index 000000000..208c1a689 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RetweetArchivalEventsAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RetweetArchivalEventsAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RetweetArchivalEventsAdapterSpec.scala deleted file mode 100644 index 00a78b535..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/RetweetArchivalEventsAdapterSpec.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.inject.Test -import com.twitter.tweetypie.thriftscala.RetweetArchivalEvent -import com.twitter.unified_user_actions.adapter.retweet_archival_events.RetweetArchivalEventsAdapter -import com.twitter.unified_user_actions.thriftscala._ -import com.twitter.util.Time -import org.scalatest.prop.TableDrivenPropertyChecks - -class RetweetArchivalEventsAdapterSpec extends Test with TableDrivenPropertyChecks { - trait Fixture { - - val frozenTime = Time.fromMilliseconds(1658949273000L) - - val authorId = 1L - val tweetId = 101L - val retweetId = 102L - val retweetAuthorId = 2L - - val retweetArchivalEvent = RetweetArchivalEvent( - retweetId = retweetId, - srcTweetId = tweetId, - retweetUserId = retweetAuthorId, - srcTweetUserId = authorId, - timestampMs = 0L, - isArchivingAction = Some(true), - ) - val retweetUnarchivalEvent = RetweetArchivalEvent( - retweetId = retweetId, - srcTweetId = tweetId, - retweetUserId = retweetAuthorId, - srcTweetUserId = authorId, - timestampMs = 0L, - isArchivingAction = Some(false), - ) - - val expectedUua1 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(retweetAuthorId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(authorId))), - retweetingTweetId = Some(retweetId) - ) - ), - actionType = ActionType.ServerTweetArchiveRetweet, - eventMetadata = EventMetadata( - sourceTimestampMs = 0L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerRetweetArchivalEvents, - ) - ) - val expectedUua2 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(retweetAuthorId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = tweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(authorId))), - retweetingTweetId = Some(retweetId) - ) - ), - actionType = ActionType.ServerTweetUnarchiveRetweet, - eventMetadata = EventMetadata( - sourceTimestampMs = 0L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerRetweetArchivalEvents, - ) - ) - } - - test("all tests") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val table = Table( - ("event", "expected"), - (retweetArchivalEvent, expectedUua1), - (retweetUnarchivalEvent, expectedUua2), - ) - forEvery(table) { (event: RetweetArchivalEvent, expected: UnifiedUserAction) => - val actual = RetweetArchivalEventsAdapter.adaptEvent(event) - assert(Seq(expected) === actual) - } - } - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SearchInfoUtilsSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SearchInfoUtilsSpec.docx new file mode 100644 index 000000000..7ce28d4e1 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SearchInfoUtilsSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SearchInfoUtilsSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SearchInfoUtilsSpec.scala deleted file mode 100644 index 6746b3099..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SearchInfoUtilsSpec.scala +++ /dev/null @@ -1,355 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.clientapp.thriftscala.SuggestionDetails -import com.twitter.clientapp.thriftscala._ -import com.twitter.search.common.constants.thriftscala.ThriftQuerySource -import com.twitter.search.common.constants.thriftscala.TweetResultSource -import com.twitter.search.common.constants.thriftscala.UserResultSource -import com.twitter.suggests.controller_data.search_response.item_types.thriftscala.ItemTypesControllerData -import com.twitter.suggests.controller_data.search_response.request.thriftscala.RequestControllerData -import com.twitter.suggests.controller_data.search_response.thriftscala.SearchResponseControllerData -import com.twitter.suggests.controller_data.search_response.tweet_types.thriftscala.TweetTypesControllerData -import com.twitter.suggests.controller_data.search_response.user_types.thriftscala.UserTypesControllerData -import com.twitter.suggests.controller_data.search_response.v1.thriftscala.{ - SearchResponseControllerData => SearchResponseControllerDataV1 -} -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} -import com.twitter.util.mock.Mockito -import org.junit.runner.RunWith -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers -import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatestplus.junit.JUnitRunner -import com.twitter.unified_user_actions.adapter.client_event.SearchInfoUtils -import com.twitter.unified_user_actions.thriftscala.SearchQueryFilterType -import com.twitter.unified_user_actions.thriftscala.SearchQueryFilterType._ -import org.scalatest.prop.TableFor2 - -@RunWith(classOf[JUnitRunner]) -class SearchInfoUtilsSpec - extends AnyFunSuite - with Matchers - with Mockito - with TableDrivenPropertyChecks { - - trait Fixture { - def mkControllerData( - queryOpt: Option[String], - querySourceOpt: Option[Int] = None, - traceId: Option[Long] = None, - requestJoinId: Option[Long] = None - ): ControllerData = { - ControllerData.V2( - ControllerDataV2.SearchResponse( - SearchResponseControllerData.V1( - SearchResponseControllerDataV1(requestControllerData = Some( - RequestControllerData( - rawQuery = queryOpt, - querySource = querySourceOpt, - traceId = traceId, - requestJoinId = requestJoinId - ))) - ))) - } - - def mkTweetTypeControllerData(bitmap: Long, topicId: Option[Long] = None): ControllerData.V2 = { - ControllerData.V2( - ControllerDataV2.SearchResponse( - SearchResponseControllerData.V1( - SearchResponseControllerDataV1(itemTypesControllerData = Some( - ItemTypesControllerData.TweetTypesControllerData( - TweetTypesControllerData( - tweetTypesBitmap = Some(bitmap), - topicId = topicId - )) - )) - ))) - } - - def mkUserTypeControllerData(bitmap: Long): ControllerData.V2 = { - ControllerData.V2( - ControllerDataV2.SearchResponse( - SearchResponseControllerData.V1( - SearchResponseControllerDataV1(itemTypesControllerData = Some( - ItemTypesControllerData.UserTypesControllerData(UserTypesControllerData( - userTypesBitmap = Some(bitmap) - )) - )) - ))) - } - } - - test("getQueryOptFromControllerDataFromItem should return query if present in controller data") { - new Fixture { - - val controllerData: ControllerData = mkControllerData(Some("twitter")) - val suggestionDetails: SuggestionDetails = - SuggestionDetails(decodedControllerData = Some(controllerData)) - val item: Item = Item(suggestionDetails = Some(suggestionDetails)) - val result: Option[String] = new SearchInfoUtils(item).getQueryOptFromControllerDataFromItem - result shouldEqual Option("twitter") - } - } - - test("getRequestJoinId should return requestJoinId if present in controller data") { - new Fixture { - - val controllerData: ControllerData = mkControllerData( - Some("twitter"), - traceId = Some(11L), - requestJoinId = Some(12L) - ) - val suggestionDetails: SuggestionDetails = - SuggestionDetails(decodedControllerData = Some(controllerData)) - val item: Item = Item(suggestionDetails = Some(suggestionDetails)) - val infoUtils = new SearchInfoUtils(item) - infoUtils.getTraceId shouldEqual Some(11L) - infoUtils.getRequestJoinId shouldEqual Some(12L) - } - } - - test("getQueryOptFromControllerDataFromItem should return None if no suggestion details") { - new Fixture { - - val suggestionDetails: SuggestionDetails = SuggestionDetails() - val item: Item = Item(suggestionDetails = Some(suggestionDetails)) - val result: Option[String] = new SearchInfoUtils(item).getQueryOptFromControllerDataFromItem - result shouldEqual None - } - } - - test("getQueryOptFromSearchDetails should return query if present") { - new Fixture { - - val searchDetails: SearchDetails = SearchDetails(query = Some("twitter")) - val result: Option[String] = new SearchInfoUtils(Item()).getQueryOptFromSearchDetails( - LogEvent(eventName = "", searchDetails = Some(searchDetails)) - ) - result shouldEqual Option("twitter") - } - } - - test("getQueryOptFromSearchDetails should return None if not present") { - new Fixture { - - val searchDetails: SearchDetails = SearchDetails() - val result: Option[String] = new SearchInfoUtils(Item()).getQueryOptFromSearchDetails( - LogEvent(eventName = "", searchDetails = Some(searchDetails)) - ) - result shouldEqual None - } - } - - test("getQuerySourceOptFromControllerDataFromItem should return QuerySource if present") { - new Fixture { - - // 1 is Typed Query - val controllerData: ControllerData = mkControllerData(Some("twitter"), Some(1)) - - val item: Item = Item( - suggestionDetails = Some( - SuggestionDetails( - decodedControllerData = Some(controllerData) - )) - ) - new SearchInfoUtils(item).getQuerySourceOptFromControllerDataFromItem shouldEqual Some( - ThriftQuerySource.TypedQuery) - } - } - - test("getQuerySourceOptFromControllerDataFromItem should return None if not present") { - new Fixture { - - val controllerData: ControllerData = mkControllerData(Some("twitter"), None) - - val item: Item = Item( - suggestionDetails = Some( - SuggestionDetails( - decodedControllerData = Some(controllerData) - )) - ) - new SearchInfoUtils(item).getQuerySourceOptFromControllerDataFromItem shouldEqual None - } - } - - test("Decoding Tweet Result Sources bitmap") { - new Fixture { - - TweetResultSource.list - .foreach { tweetResultSource => - val bitmap = (1 << tweetResultSource.getValue()).toLong - val controllerData = mkTweetTypeControllerData(bitmap) - - val item = Item( - suggestionDetails = Some( - SuggestionDetails( - decodedControllerData = Some(controllerData) - )) - ) - - val result = new SearchInfoUtils(item).getTweetResultSources - result shouldEqual Some(Set(tweetResultSource)) - } - } - } - - test("Decoding multiple Tweet Result Sources") { - new Fixture { - - val tweetResultSources: Set[TweetResultSource] = - Set(TweetResultSource.QueryInteractionGraph, TweetResultSource.QueryExpansion) - val bitmap: Long = tweetResultSources.foldLeft(0L) { - case (acc, source) => acc + (1 << source.getValue()) - } - - val controllerData: ControllerData.V2 = mkTweetTypeControllerData(bitmap) - - val item: Item = Item( - suggestionDetails = Some( - SuggestionDetails( - decodedControllerData = Some(controllerData) - )) - ) - - val result: Option[Set[TweetResultSource]] = new SearchInfoUtils(item).getTweetResultSources - result shouldEqual Some(tweetResultSources) - } - } - - test("Decoding User Result Sources bitmap") { - new Fixture { - - UserResultSource.list - .foreach { userResultSource => - val bitmap = (1 << userResultSource.getValue()).toLong - val controllerData = mkUserTypeControllerData(bitmap) - - val item = Item( - suggestionDetails = Some( - SuggestionDetails( - decodedControllerData = Some(controllerData) - )) - ) - - val result = new SearchInfoUtils(item).getUserResultSources - result shouldEqual Some(Set(userResultSource)) - } - } - } - - test("Decoding multiple User Result Sources") { - new Fixture { - - val userResultSources: Set[UserResultSource] = - Set(UserResultSource.QueryInteractionGraph, UserResultSource.ExpertSearch) - val bitmap: Long = userResultSources.foldLeft(0L) { - case (acc, source) => acc + (1 << source.getValue()) - } - - val controllerData: ControllerData.V2 = mkUserTypeControllerData(bitmap) - - val item: Item = Item( - suggestionDetails = Some( - SuggestionDetails( - decodedControllerData = Some(controllerData) - )) - ) - - val result: Option[Set[UserResultSource]] = new SearchInfoUtils(item).getUserResultSources - result shouldEqual Some(userResultSources) - } - } - - test("getQueryFilterTabType should return correct query filter type") { - new Fixture { - val infoUtils = new SearchInfoUtils(Item()) - val eventsToBeChecked: TableFor2[Option[EventNamespace], Option[SearchQueryFilterType]] = - Table( - ("eventNamespace", "queryFilterType"), - ( - Some(EventNamespace(client = Some("m5"), element = Some("search_filter_top"))), - Some(Top)), - ( - Some(EventNamespace(client = Some("m5"), element = Some("search_filter_live"))), - Some(Latest)), - ( - Some(EventNamespace(client = Some("m5"), element = Some("search_filter_user"))), - Some(People)), - ( - Some(EventNamespace(client = Some("m5"), element = Some("search_filter_image"))), - Some(Photos)), - ( - Some(EventNamespace(client = Some("m5"), element = Some("search_filter_video"))), - Some(Videos)), - ( - Some(EventNamespace(client = Some("m5"), section = Some("search_filter_top"))), - None - ), // if client is web, element determines the query filter hence None if element is None - ( - Some(EventNamespace(client = Some("android"), element = Some("search_filter_top"))), - Some(Top)), - ( - Some(EventNamespace(client = Some("android"), element = Some("search_filter_tweets"))), - Some(Latest)), - ( - Some(EventNamespace(client = Some("android"), element = Some("search_filter_user"))), - Some(People)), - ( - Some(EventNamespace(client = Some("android"), element = Some("search_filter_image"))), - Some(Photos)), - ( - Some(EventNamespace(client = Some("android"), element = Some("search_filter_video"))), - Some(Videos)), - ( - Some(EventNamespace(client = Some("m5"), section = Some("search_filter_top"))), - None - ), // if client is android, element determines the query filter hence None if element is None - ( - Some(EventNamespace(client = Some("iphone"), section = Some("search_filter_top"))), - Some(Top)), - ( - Some(EventNamespace(client = Some("iphone"), section = Some("search_filter_live"))), - Some(Latest)), - ( - Some(EventNamespace(client = Some("iphone"), section = Some("search_filter_user"))), - Some(People)), - ( - Some(EventNamespace(client = Some("iphone"), section = Some("search_filter_image"))), - Some(Photos)), - ( - Some(EventNamespace(client = Some("iphone"), section = Some("search_filter_video"))), - Some(Videos)), - ( - Some(EventNamespace(client = Some("iphone"), element = Some("search_filter_top"))), - None - ), // if client is iphone, section determines the query filter hence None if section is None - ( - Some(EventNamespace(client = None, section = Some("search_filter_top"))), - Some(Top) - ), // if client is missing, use section by default - ( - Some(EventNamespace(client = None, element = Some("search_filter_top"))), - None - ), // if client is missing, section is used by default hence None since section is missing - ( - Some(EventNamespace(client = Some("iphone"))), - None - ), // if both element and section missing, expect None - (None, None), // if namespace is missing from LogEvent, expect None - ) - - forEvery(eventsToBeChecked) { - ( - eventNamespace: Option[EventNamespace], - searchQueryFilterType: Option[SearchQueryFilterType] - ) => - infoUtils.getQueryFilterType( - LogEvent( - eventName = "srp_event", - eventNamespace = eventNamespace)) shouldEqual searchQueryFilterType - } - - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SocialGraphAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SocialGraphAdapterSpec.docx new file mode 100644 index 000000000..363c6773c Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SocialGraphAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SocialGraphAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SocialGraphAdapterSpec.scala deleted file mode 100644 index 168700045..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/SocialGraphAdapterSpec.scala +++ /dev/null @@ -1,359 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.inject.Test -import com.twitter.socialgraph.thriftscala.Action -import com.twitter.socialgraph.thriftscala.BlockGraphEvent -import com.twitter.socialgraph.thriftscala.FollowGraphEvent -import com.twitter.socialgraph.thriftscala.FollowRequestGraphEvent -import com.twitter.socialgraph.thriftscala.FollowRetweetsGraphEvent -import com.twitter.socialgraph.thriftscala.LogEventContext -import com.twitter.socialgraph.thriftscala.MuteGraphEvent -import com.twitter.socialgraph.thriftscala.ReportAsAbuseGraphEvent -import com.twitter.socialgraph.thriftscala.ReportAsSpamGraphEvent -import com.twitter.socialgraph.thriftscala.SrcTargetRequest -import com.twitter.socialgraph.thriftscala.WriteEvent -import com.twitter.socialgraph.thriftscala.WriteRequestResult -import com.twitter.unified_user_actions.adapter.social_graph_event.SocialGraphAdapter -import com.twitter.unified_user_actions.thriftscala._ -import com.twitter.util.Time -import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatest.prop.TableFor1 -import org.scalatest.prop.TableFor3 - -class SocialGraphAdapterSpec extends Test with TableDrivenPropertyChecks { - trait Fixture { - - val frozenTime: Time = Time.fromMilliseconds(1658949273000L) - - val testLogEventContext: LogEventContext = LogEventContext( - timestamp = 1001L, - hostname = "", - transactionId = "", - socialGraphClientId = "", - loggedInUserId = Some(1111L), - ) - - val testWriteRequestResult: WriteRequestResult = WriteRequestResult( - request = SrcTargetRequest( - source = 1111L, - target = 2222L - ) - ) - - val testWriteRequestResultWithValidationError: WriteRequestResult = WriteRequestResult( - request = SrcTargetRequest( - source = 1111L, - target = 2222L - ), - validationError = Some("action unsuccessful") - ) - - val baseEvent: WriteEvent = WriteEvent( - context = testLogEventContext, - action = Action.AcceptFollowRequest - ) - - val sgFollowEvent: WriteEvent = baseEvent.copy( - action = Action.Follow, - follow = Some(List(FollowGraphEvent(testWriteRequestResult)))) - - val sgUnfollowEvent: WriteEvent = baseEvent.copy( - action = Action.Unfollow, - follow = Some(List(FollowGraphEvent(testWriteRequestResult)))) - - val sgFollowRedundantEvent: WriteEvent = baseEvent.copy( - action = Action.Follow, - follow = Some( - List( - FollowGraphEvent( - result = testWriteRequestResult, - redundantOperation = Some(true) - )))) - - val sgFollowRedundantIsFalseEvent: WriteEvent = baseEvent.copy( - action = Action.Follow, - follow = Some( - List( - FollowGraphEvent( - result = testWriteRequestResult, - redundantOperation = Some(false) - )))) - - val sgUnfollowRedundantEvent: WriteEvent = baseEvent.copy( - action = Action.Unfollow, - follow = Some( - List( - FollowGraphEvent( - result = testWriteRequestResult, - redundantOperation = Some(true) - )))) - - val sgUnfollowRedundantIsFalseEvent: WriteEvent = baseEvent.copy( - action = Action.Unfollow, - follow = Some( - List( - FollowGraphEvent( - result = testWriteRequestResult, - redundantOperation = Some(false) - )))) - - val sgUnsuccessfulFollowEvent: WriteEvent = baseEvent.copy( - action = Action.Follow, - follow = Some(List(FollowGraphEvent(testWriteRequestResultWithValidationError)))) - - val sgUnsuccessfulUnfollowEvent: WriteEvent = baseEvent.copy( - action = Action.Unfollow, - follow = Some(List(FollowGraphEvent(testWriteRequestResultWithValidationError)))) - - val sgBlockEvent: WriteEvent = baseEvent.copy( - action = Action.Block, - block = Some(List(BlockGraphEvent(testWriteRequestResult)))) - - val sgUnsuccessfulBlockEvent: WriteEvent = baseEvent.copy( - action = Action.Block, - block = Some(List(BlockGraphEvent(testWriteRequestResultWithValidationError)))) - - val sgUnblockEvent: WriteEvent = baseEvent.copy( - action = Action.Unblock, - block = Some(List(BlockGraphEvent(testWriteRequestResult)))) - - val sgUnsuccessfulUnblockEvent: WriteEvent = baseEvent.copy( - action = Action.Unblock, - block = Some(List(BlockGraphEvent(testWriteRequestResultWithValidationError)))) - - val sgMuteEvent: WriteEvent = baseEvent.copy( - action = Action.Mute, - mute = Some(List(MuteGraphEvent(testWriteRequestResult)))) - - val sgUnsuccessfulMuteEvent: WriteEvent = baseEvent.copy( - action = Action.Mute, - mute = Some(List(MuteGraphEvent(testWriteRequestResultWithValidationError)))) - - val sgUnmuteEvent: WriteEvent = baseEvent.copy( - action = Action.Unmute, - mute = Some(List(MuteGraphEvent(testWriteRequestResult)))) - - val sgUnsuccessfulUnmuteEvent: WriteEvent = baseEvent.copy( - action = Action.Unmute, - mute = Some(List(MuteGraphEvent(testWriteRequestResultWithValidationError)))) - - val sgCreateFollowRequestEvent: WriteEvent = baseEvent.copy( - action = Action.CreateFollowRequest, - followRequest = Some(List(FollowRequestGraphEvent(testWriteRequestResult))) - ) - - val sgCancelFollowRequestEvent: WriteEvent = baseEvent.copy( - action = Action.CancelFollowRequest, - followRequest = Some(List(FollowRequestGraphEvent(testWriteRequestResult))) - ) - - val sgAcceptFollowRequestEvent: WriteEvent = baseEvent.copy( - action = Action.AcceptFollowRequest, - followRequest = Some(List(FollowRequestGraphEvent(testWriteRequestResult))) - ) - - val sgAcceptFollowRetweetEvent: WriteEvent = baseEvent.copy( - action = Action.FollowRetweets, - followRetweets = Some(List(FollowRetweetsGraphEvent(testWriteRequestResult))) - ) - - val sgAcceptUnfollowRetweetEvent: WriteEvent = baseEvent.copy( - action = Action.UnfollowRetweets, - followRetweets = Some(List(FollowRetweetsGraphEvent(testWriteRequestResult))) - ) - - val sgReportAsSpamEvent: WriteEvent = baseEvent.copy( - action = Action.ReportAsSpam, - reportAsSpam = Some( - List( - ReportAsSpamGraphEvent( - result = testWriteRequestResult - )))) - - val sgReportAsAbuseEvent: WriteEvent = baseEvent.copy( - action = Action.ReportAsAbuse, - reportAsAbuse = Some( - List( - ReportAsAbuseGraphEvent( - result = testWriteRequestResult - )))) - - def getExpectedUUA( - userId: Long, - actionProfileId: Long, - sourceTimestampMs: Long, - actionType: ActionType, - socialGraphAction: Option[Action] = None - ): UnifiedUserAction = { - val actionItem = socialGraphAction match { - case Some(sgAction) => - Item.ProfileInfo( - ProfileInfo( - actionProfileId = actionProfileId, - profileActionInfo = Some( - ProfileActionInfo.ServerProfileReport( - ServerProfileReport(reportType = sgAction) - )) - ) - ) - case _ => - Item.ProfileInfo( - ProfileInfo( - actionProfileId = actionProfileId - ) - ) - } - - UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = actionItem, - actionType = actionType, - eventMetadata = EventMetadata( - sourceTimestampMs = sourceTimestampMs, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerSocialGraphEvents - ) - ) - } - - val expectedUuaFollow: UnifiedUserAction = getExpectedUUA( - userId = 1111L, - actionProfileId = 2222L, - sourceTimestampMs = 1001L, - actionType = ActionType.ServerProfileFollow - ) - - val expectedUuaUnfollow: UnifiedUserAction = getExpectedUUA( - userId = 1111L, - actionProfileId = 2222L, - sourceTimestampMs = 1001L, - actionType = ActionType.ServerProfileUnfollow - ) - - val expectedUuaMute: UnifiedUserAction = getExpectedUUA( - userId = 1111L, - actionProfileId = 2222L, - sourceTimestampMs = 1001L, - actionType = ActionType.ServerProfileMute - ) - - val expectedUuaUnmute: UnifiedUserAction = getExpectedUUA( - userId = 1111L, - actionProfileId = 2222L, - sourceTimestampMs = 1001L, - actionType = ActionType.ServerProfileUnmute - ) - - val expectedUuaBlock: UnifiedUserAction = getExpectedUUA( - userId = 1111L, - actionProfileId = 2222L, - sourceTimestampMs = 1001L, - actionType = ActionType.ServerProfileBlock - ) - - val expectedUuaUnblock: UnifiedUserAction = getExpectedUUA( - userId = 1111L, - actionProfileId = 2222L, - sourceTimestampMs = 1001L, - actionType = ActionType.ServerProfileUnblock - ) - - val expectedUuaReportAsSpam: UnifiedUserAction = getExpectedUUA( - userId = 1111L, - actionProfileId = 2222L, - sourceTimestampMs = 1001L, - actionType = ActionType.ServerProfileReport, - socialGraphAction = Some(Action.ReportAsSpam) - ) - - val expectedUuaReportAsAbuse: UnifiedUserAction = getExpectedUUA( - userId = 1111L, - actionProfileId = 2222L, - sourceTimestampMs = 1001L, - actionType = ActionType.ServerProfileReport, - socialGraphAction = Some(Action.ReportAsAbuse) - ) - } - - test("SocialGraphAdapter ignore events not in the list") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val ignoredSocialGraphEvents: TableFor1[WriteEvent] = Table( - "ignoredSocialGraphEvents", - sgAcceptUnfollowRetweetEvent, - sgAcceptFollowRequestEvent, - sgAcceptFollowRetweetEvent, - sgCreateFollowRequestEvent, - sgCancelFollowRequestEvent, - ) - forEvery(ignoredSocialGraphEvents) { writeEvent: WriteEvent => - val actual = SocialGraphAdapter.adaptEvent(writeEvent) - assert(actual.isEmpty) - } - } - } - } - - test("Test SocialGraphAdapter consuming Write events") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val socialProfileActions: TableFor3[String, WriteEvent, UnifiedUserAction] = Table( - ("actionType", "event", "expectedUnifiedUserAction"), - ("ProfileFollow", sgFollowEvent, expectedUuaFollow), - ("ProfileUnfollow", sgUnfollowEvent, expectedUuaUnfollow), - ("ProfileBlock", sgBlockEvent, expectedUuaBlock), - ("ProfileUnBlock", sgUnblockEvent, expectedUuaUnblock), - ("ProfileMute", sgMuteEvent, expectedUuaMute), - ("ProfileUnmute", sgUnmuteEvent, expectedUuaUnmute), - ("ProfileReportAsSpam", sgReportAsSpamEvent, expectedUuaReportAsSpam), - ("ProfileReportAsAbuse", sgReportAsAbuseEvent, expectedUuaReportAsAbuse), - ) - forEvery(socialProfileActions) { - (_: String, event: WriteEvent, expected: UnifiedUserAction) => - val actual = SocialGraphAdapter.adaptEvent(event) - assert(Seq(expected) === actual) - } - } - } - } - - test("SocialGraphAdapter ignore redundant follow/unfollow events") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val socialGraphActions: TableFor3[String, WriteEvent, Seq[UnifiedUserAction]] = Table( - ("actionType", "ignoredRedundantFollowUnfollowEvents", "expectedUnifiedUserAction"), - ("ProfileFollow", sgFollowRedundantEvent, Nil), - ("ProfileFollow", sgFollowRedundantIsFalseEvent, Seq(expectedUuaFollow)), - ("ProfileUnfollow", sgUnfollowRedundantEvent, Nil), - ("ProfileUnfollow", sgUnfollowRedundantIsFalseEvent, Seq(expectedUuaUnfollow)) - ) - forEvery(socialGraphActions) { - (_: String, event: WriteEvent, expected: Seq[UnifiedUserAction]) => - val actual = SocialGraphAdapter.adaptEvent(event) - assert(expected === actual) - } - } - } - } - - test("SocialGraphAdapter ignore Unsuccessful SocialGraph events") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val unsuccessfulSocialGraphEvents: TableFor1[WriteEvent] = Table( - "ignoredSocialGraphEvents", - sgUnsuccessfulFollowEvent, - sgUnsuccessfulUnfollowEvent, - sgUnsuccessfulBlockEvent, - sgUnsuccessfulUnblockEvent, - sgUnsuccessfulMuteEvent, - sgUnsuccessfulUnmuteEvent - ) - - forEvery(unsuccessfulSocialGraphEvents) { writeEvent: WriteEvent => - val actual = SocialGraphAdapter.adaptEvent(writeEvent) - assert(actual.isEmpty) - } - } - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TestFixtures.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TestFixtures.docx new file mode 100644 index 000000000..fe67b67ab Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TestFixtures.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TestFixtures.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TestFixtures.scala deleted file mode 100644 index b1e3c9795..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TestFixtures.scala +++ /dev/null @@ -1,2294 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.ads.cards.thriftscala.CardEvent -import com.twitter.ads.eventstream.thriftscala.EngagementEvent -import com.twitter.ads.spendserver.thriftscala.SpendServerEvent -import com.twitter.adserver.thriftscala.ImpressionDataNeededAtEngagementTime -import com.twitter.adserver.thriftscala.ClientInfo -import com.twitter.adserver.thriftscala.EngagementType -import com.twitter.adserver.thriftscala.DisplayLocation -import com.twitter.clientapp.thriftscala.AmplifyDetails -import com.twitter.clientapp.thriftscala.CardDetails -import com.twitter.clientapp.thriftscala.EventDetails -import com.twitter.clientapp.thriftscala.EventNamespace -import com.twitter.clientapp.thriftscala.ImpressionDetails -import com.twitter.clientapp.thriftscala.ItemType -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.clientapp.thriftscala.MediaDetails -import com.twitter.clientapp.thriftscala.MediaDetailsV2 -import com.twitter.clientapp.thriftscala.MediaType -import com.twitter.clientapp.thriftscala.NotificationDetails -import com.twitter.clientapp.thriftscala.NotificationTabDetails -import com.twitter.clientapp.thriftscala.PerformanceDetails -import com.twitter.clientapp.thriftscala.ReportDetails -import com.twitter.clientapp.thriftscala.SearchDetails -import com.twitter.clientapp.thriftscala.SuggestionDetails -import com.twitter.clientapp.thriftscala.{Item => LogEventItem} -import com.twitter.clientapp.thriftscala.{TweetDetails => LogEventTweetDetails} -import com.twitter.gizmoduck.thriftscala.UserModification -import com.twitter.gizmoduck.thriftscala.Profile -import com.twitter.gizmoduck.thriftscala.Auth -import com.twitter.gizmoduck.thriftscala.UpdateDiffItem -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.ibis.thriftscala.NotificationScribe -import com.twitter.ibis.thriftscala.NotificationScribeType -import com.twitter.iesource.thriftscala.ClientEventContext -import com.twitter.iesource.thriftscala.TweetImpression -import com.twitter.iesource.thriftscala.ClientType -import com.twitter.iesource.thriftscala.ContextualEventNamespace -import com.twitter.iesource.thriftscala.EngagingContext -import com.twitter.iesource.thriftscala.EventSource -import com.twitter.iesource.thriftscala.InteractionDetails -import com.twitter.iesource.thriftscala.InteractionEvent -import com.twitter.iesource.thriftscala.InteractionType -import com.twitter.iesource.thriftscala.InteractionTargetType -import com.twitter.iesource.thriftscala.{UserIdentifier => UserIdentifierIE} -import com.twitter.logbase.thriftscala.ClientEventReceiver -import com.twitter.logbase.thriftscala.LogBase -import com.twitter.mediaservices.commons.thriftscala.MediaCategory -import com.twitter.notificationservice.api.thriftscala.NotificationClientEventMetadata -import com.twitter.reportflow.thriftscala.ReportType -import com.twitter.suggests.controller_data.home_tweets.thriftscala.HomeTweetsControllerData -import com.twitter.suggests.controller_data.home_tweets.v1.thriftscala.{ - HomeTweetsControllerData => HomeTweetsControllerDataV1 -} -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.timelines_topic.thriftscala.TimelinesTopicControllerData -import com.twitter.suggests.controller_data.timelines_topic.v1.thriftscala.{ - TimelinesTopicControllerData => TimelinesTopicControllerDataV1 -} -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} -import com.twitter.unified_user_actions.thriftscala._ -import com.twitter.util.Time -import com.twitter.video.analytics.thriftscala.ClientMediaEvent -import com.twitter.video.analytics.thriftscala.SessionState -import com.twitter.video.analytics.thriftscala._ -import com.twitter.suggests.controller_data.search_response.v1.thriftscala.{ - SearchResponseControllerData => SearchResponseControllerDataV1 -} -import com.twitter.suggests.controller_data.search_response.thriftscala.SearchResponseControllerData -import com.twitter.suggests.controller_data.search_response.request.thriftscala.RequestControllerData -import com.twitter.unified_user_actions.thriftscala.FeedbackPromptInfo - -object TestFixtures { - trait CommonFixture { - val frozenTime: Time = Time.fromMilliseconds(1658949273000L) - - val userId: Long = 123L - val authorId: Long = 112233L - val itemTweetId: Long = 111L - val itemProfileId: Long = 123456L - val retweetingTweetId: Long = 222L - val quotedTweetId: Long = 333L - val quotedAuthorId: Long = 456L - val inReplyToTweetId: Long = 444L - val quotingTweetId: Long = 555L - val topicId: Long = 1234L - val traceId: Long = 5678L - val requestJoinId: Long = 91011L - val notificationId: String = "12345" - val tweetIds: Seq[Long] = Seq[Long](111, 222, 333) - val reportFlowId: String = "report-flow-id" - } - - trait ClientEventFixture extends CommonFixture { - - val timestamp = 1001L - - val logBase: LogBase = LogBase( - ipAddress = "", - transactionId = "", - timestamp = 1002L, - driftAdjustedEventCreatedAtMs = Some(1001L), - userId = Some(userId), - clientEventReceiver = Some(ClientEventReceiver.CesHttp) - ) - - val logBase1: LogBase = LogBase( - ipAddress = "", - transactionId = "", - userId = Some(userId), - guestId = Some(2L), - guestIdMarketing = Some(2L), - timestamp = timestamp - ) - - def mkSearchResultControllerData( - queryOpt: Option[String], - querySourceOpt: Option[Int] = None, - traceId: Option[Long] = None, - requestJoinId: Option[Long] = None - ): ControllerData = { - ControllerData.V2( - ControllerDataV2.SearchResponse( - SearchResponseControllerData.V1( - SearchResponseControllerDataV1(requestControllerData = Some( - RequestControllerData( - rawQuery = queryOpt, - querySource = querySourceOpt, - traceId = traceId, - requestJoinId = requestJoinId - ))) - ))) - } - - val videoEventElementValues: Seq[String] = - Seq[String]( - "gif_player", - "periscope_player", - "platform_amplify_card", - "video_player", - "vine_player") - - val invalidVideoEventElementValues: Seq[String] = - Seq[String]( - "dynamic_video_ads", - "live_video_player", - "platform_forward_card", - "video_app_card_canvas", - "youtube_player" - ) - - val clientMediaEvent: ClientMediaEvent = ClientMediaEvent( - sessionState = SessionState( - contentVideoIdentifier = MediaIdentifier.MediaPlatformIdentifier( - MediaPlatformIdentifier(mediaId = 123L, mediaCategory = MediaCategory.TweetVideo)), - sessionId = "", - ), - mediaClientEventType = MediaEventType.IntentToPlay(IntentToPlay()), - playingMediaState = PlayingMediaState( - videoType = VideoType.Content, - mediaAssetUrl = "", - mediaMetadata = MediaMetadata(publisherIdentifier = PublisherIdentifier - .TwitterPublisherIdentifier(TwitterPublisherIdentifier(123456L))) - ), - playerState = PlayerState(isMuted = false) - ) - - val mediaDetailsV2: MediaDetailsV2 = MediaDetailsV2( - mediaItems = Some( - Seq[MediaDetails]( - MediaDetails( - contentId = Some("456"), - mediaType = Some(MediaType.ConsumerVideo), - dynamicAds = Some(false)), - MediaDetails( - contentId = Some("123"), - mediaType = Some(MediaType.ConsumerVideo), - dynamicAds = Some(false)), - MediaDetails( - contentId = Some("789"), - mediaType = Some(MediaType.ConsumerVideo), - dynamicAds = Some(false)) - )) - ) - - val cardDetails = - CardDetails(amplifyDetails = Some(AmplifyDetails(videoType = Some("content")))) - - val videoMetadata: TweetActionInfo = TweetActionInfo.TweetVideoWatch( - TweetVideoWatch( - mediaType = Some(MediaType.ConsumerVideo), - isMonetizable = Some(false), - videoType = Some("content"))) - - val notificationDetails: NotificationDetails = - NotificationDetails(impressionId = Some(notificationId)) - - val notificationTabTweetEventDetails: NotificationTabDetails = - NotificationTabDetails( - clientEventMetadata = Some( - NotificationClientEventMetadata( - tweetIds = Some(Seq[Long](itemTweetId)), - upstreamId = Some(notificationId), - requestId = "", - notificationId = "", - notificationCount = 0)) - ) - - val notificationTabMultiTweetEventDetails: NotificationTabDetails = - NotificationTabDetails( - clientEventMetadata = Some( - NotificationClientEventMetadata( - tweetIds = Some(tweetIds), - upstreamId = Some(notificationId), - requestId = "", - notificationId = "", - notificationCount = 0)) - ) - - val notificationTabUnknownEventDetails: NotificationTabDetails = - NotificationTabDetails( - clientEventMetadata = Some( - NotificationClientEventMetadata( - upstreamId = Some(notificationId), - requestId = "", - notificationId = "", - notificationCount = 0)) - ) - - val tweetNotificationContent: NotificationContent = - NotificationContent.TweetNotification(TweetNotification(itemTweetId)) - - val multiTweetNotificationContent: NotificationContent = - NotificationContent.MultiTweetNotification(MultiTweetNotification(tweetIds)) - - val unknownNotificationContent: NotificationContent = - NotificationContent.UnknownNotification(UnknownNotification()) - - val reportTweetClick: TweetActionInfo = - TweetActionInfo.ClientTweetReport(ClientTweetReport(isReportTweetDone = false)) - - val reportTweetDone: TweetActionInfo = - TweetActionInfo.ClientTweetReport(ClientTweetReport(isReportTweetDone = true)) - - val reportTweetWithReportFlowId: TweetActionInfo = - TweetActionInfo.ClientTweetReport( - ClientTweetReport(isReportTweetDone = true, reportFlowId = Some(reportFlowId))) - - val reportTweetWithoutReportFlowId: TweetActionInfo = - TweetActionInfo.ClientTweetReport( - ClientTweetReport(isReportTweetDone = true, reportFlowId = None)) - - val reportTweetSubmit: TweetActionInfo = - TweetActionInfo.ServerTweetReport( - ServerTweetReport(reportFlowId = Some(reportFlowId), reportType = Some(ReportType.Abuse))) - - val notificationTabProductSurfaceInfo: ProductSurfaceInfo = - ProductSurfaceInfo.NotificationTabInfo(NotificationTabInfo(notificationId = notificationId)) - - val clientOpenLinkWithUrl: TweetActionInfo = - TweetActionInfo.ClientTweetOpenLink(ClientTweetOpenLink(url = Some("go/url"))) - - val clientOpenLinkWithoutUrl: TweetActionInfo = - TweetActionInfo.ClientTweetOpenLink(ClientTweetOpenLink(url = None)) - - val clientTakeScreenshot: TweetActionInfo = - TweetActionInfo.ClientTweetTakeScreenshot( - ClientTweetTakeScreenshot(percentVisibleHeight100k = Some(100))) - - // client-event event_namespace - val ceLingerEventNamespace: EventNamespace = EventNamespace( - component = Some("stream"), - element = Some("linger"), - action = Some("results") - ) - val ceRenderEventNamespace: EventNamespace = EventNamespace( - component = Some("stream"), - action = Some("results") - ) - val ceTweetDetailsEventNamespace1: EventNamespace = EventNamespace( - page = Some("tweet"), - section = None, - component = Some("tweet"), - element = None, - action = Some("impression") - ) - val ceGalleryEventNamespace: EventNamespace = EventNamespace( - component = Some("gallery"), - element = Some("photo"), - action = Some("impression") - ) - val ceFavoriteEventNamespace: EventNamespace = EventNamespace(action = Some("favorite")) - val ceHomeFavoriteEventNamespace: EventNamespace = - EventNamespace(page = Some("home"), action = Some("favorite")) - val ceHomeLatestFavoriteEventNamespace: EventNamespace = - EventNamespace(page = Some("home_latest"), action = Some("favorite")) - val ceSearchFavoriteEventNamespace: EventNamespace = - EventNamespace(page = Some("search"), action = Some("favorite")) - val ceClickReplyEventNamespace: EventNamespace = EventNamespace(action = Some("reply")) - val ceReplyEventNamespace: EventNamespace = EventNamespace(action = Some("send_reply")) - val ceRetweetEventNamespace: EventNamespace = EventNamespace(action = Some("retweet")) - val ceVideoPlayback25: EventNamespace = EventNamespace(action = Some("playback_25")) - val ceVideoPlayback50: EventNamespace = EventNamespace(action = Some("playback_50")) - val ceVideoPlayback75: EventNamespace = EventNamespace(action = Some("playback_75")) - val ceVideoPlayback95: EventNamespace = EventNamespace(action = Some("playback_95")) - val ceVideoPlayFromTap: EventNamespace = EventNamespace(action = Some("play_from_tap")) - val ceVideoQualityView: EventNamespace = EventNamespace(action = Some("video_quality_view")) - val ceVideoView: EventNamespace = EventNamespace(action = Some("video_view")) - val ceVideoMrcView: EventNamespace = EventNamespace(action = Some("video_mrc_view")) - val ceVideoViewThreshold: EventNamespace = EventNamespace(action = Some("view_threshold")) - val ceVideoCtaUrlClick: EventNamespace = EventNamespace(action = Some("cta_url_click")) - val ceVideoCtaWatchClick: EventNamespace = EventNamespace(action = Some("cta_watch_click")) - val cePhotoExpand: EventNamespace = - EventNamespace(element = Some("platform_photo_card"), action = Some("click")) - val ceCardClick: EventNamespace = - EventNamespace(element = Some("platform_card"), action = Some("click")) - val ceCardOpenApp: EventNamespace = EventNamespace(action = Some("open_app")) - val ceCardAppInstallAttempt: EventNamespace = EventNamespace(action = Some("install_app")) - val cePollCardVote1: EventNamespace = - EventNamespace(element = Some("platform_card"), action = Some("vote")) - val cePollCardVote2: EventNamespace = - EventNamespace(element = Some("platform_forward_card"), action = Some("vote")) - val ceMentionClick: EventNamespace = - EventNamespace(element = Some("mention"), action = Some("click")) - val ceVideoPlaybackStart: EventNamespace = EventNamespace(action = Some("playback_start")) - val ceVideoPlaybackComplete: EventNamespace = EventNamespace(action = Some("playback_complete")) - val ceClickHashtag: EventNamespace = EventNamespace(action = Some("hashtag_click")) - val ceTopicFollow1: EventNamespace = - EventNamespace(element = Some("topic"), action = Some("follow")) - val ceOpenLink: EventNamespace = EventNamespace(action = Some("open_link")) - val ceTakeScreenshot: EventNamespace = EventNamespace(action = Some("take_screenshot")) - val ceTopicFollow2: EventNamespace = - EventNamespace(element = Some("social_proof"), action = Some("follow")) - val ceTopicFollow3: EventNamespace = - EventNamespace(element = Some("feedback_follow_topic"), action = Some("click")) - val ceTopicUnfollow1: EventNamespace = - EventNamespace(element = Some("topic"), action = Some("unfollow")) - val ceTopicUnfollow2: EventNamespace = - EventNamespace(element = Some("social_proof"), action = Some("unfollow")) - val ceTopicUnfollow3: EventNamespace = - EventNamespace(element = Some("feedback_unfollow_topic"), action = Some("click")) - val ceTopicNotInterestedIn1: EventNamespace = - EventNamespace(element = Some("topic"), action = Some("not_interested")) - val ceTopicNotInterestedIn2: EventNamespace = - EventNamespace(element = Some("feedback_not_interested_in_topic"), action = Some("click")) - val ceTopicUndoNotInterestedIn1: EventNamespace = - EventNamespace(element = Some("topic"), action = Some("un_not_interested")) - val ceTopicUndoNotInterestedIn2: EventNamespace = - EventNamespace(element = Some("feedback_not_interested_in_topic"), action = Some("undo")) - val ceProfileFollowAttempt: EventNamespace = - EventNamespace(action = Some("follow_attempt")) - val ceTweetFavoriteAttempt: EventNamespace = - EventNamespace(action = Some("favorite_attempt")) - val ceTweetRetweetAttempt: EventNamespace = - EventNamespace(action = Some("retweet_attempt")) - val ceTweetReplyAttempt: EventNamespace = - EventNamespace(action = Some("reply_attempt")) - val ceClientCTALoginClick: EventNamespace = - EventNamespace(action = Some("login")) - val ceClientCTALoginStart: EventNamespace = - EventNamespace(page = Some("login"), action = Some("show")) - val ceClientCTALoginSuccess: EventNamespace = - EventNamespace(page = Some("login"), action = Some("success")) - val ceClientCTASignupClick: EventNamespace = - EventNamespace(action = Some("signup")) - val ceClientCTASignupSuccess: EventNamespace = - EventNamespace(page = Some("signup"), action = Some("success")) - val ceNotificationOpen: EventNamespace = EventNamespace( - page = Some("notification"), - section = Some("status_bar"), - component = None, - action = Some("open")) - val ceNotificationClick: EventNamespace = EventNamespace( - page = Some("ntab"), - section = Some("all"), - component = Some("urt"), - element = Some("users_liked_your_tweet"), - action = Some("navigate")) - val ceTypeaheadClick: EventNamespace = - EventNamespace(element = Some("typeahead"), action = Some("click")) - val ceTweetReport: EventNamespace = EventNamespace(element = Some("report_tweet")) - def ceEventNamespace(element: String, action: String): EventNamespace = - EventNamespace(element = Some(element), action = Some(action)) - def ceTweetReportFlow(page: String, action: String): EventNamespace = - EventNamespace(element = Some("ticket"), page = Some(page), action = Some(action)) - val ceNotificationSeeLessOften: EventNamespace = EventNamespace( - page = Some("ntab"), - section = Some("all"), - component = Some("urt"), - action = Some("see_less_often")) - val ceNotificationDismiss: EventNamespace = EventNamespace( - page = Some("notification"), - section = Some("status_bar"), - component = None, - action = Some("dismiss")) - val ceSearchResultsRelevant: EventNamespace = EventNamespace( - page = Some("search"), - component = Some("did_you_find_it_module"), - element = Some("is_relevant"), - action = Some("click") - ) - val ceSearchResultsNotRelevant: EventNamespace = EventNamespace( - page = Some("search"), - component = Some("did_you_find_it_module"), - element = Some("not_relevant"), - action = Some("click") - ) - val ceTweetRelevantToSearch: EventNamespace = EventNamespace( - page = Some("search"), - component = Some("relevance_prompt_module"), - element = Some("is_relevant"), - action = Some("click")) - val ceTweetNotRelevantToSearch: EventNamespace = EventNamespace( - page = Some("search"), - component = Some("relevance_prompt_module"), - element = Some("not_relevant"), - action = Some("click")) - val ceProfileBlock: EventNamespace = - EventNamespace(page = Some("profile"), action = Some("block")) - val ceProfileUnblock: EventNamespace = - EventNamespace(page = Some("profile"), action = Some("unblock")) - val ceProfileMute: EventNamespace = - EventNamespace(page = Some("profile"), action = Some("mute_user")) - val ceProfileReport: EventNamespace = - EventNamespace(page = Some("profile"), action = Some("report")) - val ceProfileShow: EventNamespace = - EventNamespace(page = Some("profile"), action = Some("show")) - val ceProfileFollow: EventNamespace = - EventNamespace(action = Some("follow")) - val ceProfileClick: EventNamespace = - EventNamespace(action = Some("profile_click")) - val ceTweetFollowAuthor1: EventNamespace = EventNamespace( - action = Some("click"), - element = Some("follow") - ) - val ceTweetFollowAuthor2: EventNamespace = EventNamespace( - action = Some("follow") - ) - val ceTweetUnfollowAuthor1: EventNamespace = EventNamespace( - action = Some("click"), - element = Some("unfollow") - ) - val ceTweetUnfollowAuthor2: EventNamespace = EventNamespace( - action = Some("unfollow") - ) - val ceTweetBlockAuthor: EventNamespace = EventNamespace( - page = Some("profile"), - section = Some("tweets"), - component = Some("tweet"), - action = Some("click"), - element = Some("block") - ) - val ceTweetUnblockAuthor: EventNamespace = EventNamespace( - section = Some("tweets"), - component = Some("tweet"), - action = Some("click"), - element = Some("unblock") - ) - val ceTweetMuteAuthor: EventNamespace = EventNamespace( - component = Some("suggest_sc_tweet"), - action = Some("click"), - element = Some("mute") - ) - val ceTweetClick: EventNamespace = - EventNamespace(element = Some("tweet"), action = Some("click")) - val ceTweetClickProfile: EventNamespace = EventNamespace( - component = Some("tweet"), - element = Some("user"), - action = Some("profile_click")) - val ceAppExit: EventNamespace = - EventNamespace(page = Some("app"), action = Some("become_inactive")) - - // UUA client_event_namespace - val uuaLingerClientEventNamespace: ClientEventNamespace = ClientEventNamespace( - component = Some("stream"), - element = Some("linger"), - action = Some("results") - ) - val uuaRenderClientEventNamespace: ClientEventNamespace = ClientEventNamespace( - component = Some("stream"), - action = Some("results") - ) - val ceTweetDetailsClientEventNamespace1: ClientEventNamespace = ClientEventNamespace( - page = Some("tweet"), - section = None, - component = Some("tweet"), - element = None, - action = Some("impression") - ) - val ceTweetDetailsClientEventNamespace2: ClientEventNamespace = ClientEventNamespace( - page = Some("tweet"), - section = None, - component = Some("suggest_ranked_list_tweet"), - element = None, - action = Some("impression") - ) - val ceTweetDetailsClientEventNamespace3: ClientEventNamespace = ClientEventNamespace( - page = Some("tweet"), - section = None, - component = None, - element = None, - action = Some("impression") - ) - val ceTweetDetailsClientEventNamespace4: ClientEventNamespace = ClientEventNamespace( - page = Some("tweet"), - section = None, - component = None, - element = None, - action = Some("show") - ) - val ceTweetDetailsClientEventNamespace5: ClientEventNamespace = ClientEventNamespace( - page = Some("tweet"), - section = Some("landing"), - component = None, - element = None, - action = Some("show") - ) - val ceGalleryClientEventNamespace: ClientEventNamespace = ClientEventNamespace( - component = Some("gallery"), - element = Some("photo"), - action = Some("impression") - ) - val uuaFavoriteClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("favorite")) - val uuaHomeFavoriteClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(page = Some("home"), action = Some("favorite")) - val uuaSearchFavoriteClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(page = Some("search"), action = Some("favorite")) - val uuaHomeLatestFavoriteClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(page = Some("home_latest"), action = Some("favorite")) - val uuaClickReplyClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("reply")) - val uuaReplyClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("send_reply")) - val uuaRetweetClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("retweet")) - val uuaVideoPlayback25ClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("playback_25")) - val uuaVideoPlayback50ClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("playback_50")) - val uuaVideoPlayback75ClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("playback_75")) - val uuaVideoPlayback95ClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("playback_95")) - val uuaOpenLinkClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("open_link")) - val uuaTakeScreenshotClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("take_screenshot")) - val uuaVideoPlayFromTapClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("play_from_tap")) - val uuaVideoQualityViewClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("video_quality_view")) - val uuaVideoViewClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("video_view")) - val uuaVideoMrcViewClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("video_mrc_view")) - val uuaVideoViewThresholdClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("view_threshold")) - val uuaVideoCtaUrlClickClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("cta_url_click")) - val uuaVideoCtaWatchClickClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("cta_watch_click")) - val uuaPhotoExpandClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(element = Some("platform_photo_card"), action = Some("click")) - val uuaCardClickClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(element = Some("platform_card"), action = Some("click")) - val uuaCardOpenAppClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("open_app")) - val uuaCardAppInstallAttemptClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("install_app")) - val uuaPollCardVote1ClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(element = Some("platform_card"), action = Some("vote")) - val uuaPollCardVote2ClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(element = Some("platform_forward_card"), action = Some("vote")) - val uuaMentionClickClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(element = Some("mention"), action = Some("click")) - val uuaVideoPlaybackStartClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("playback_start")) - val uuaVideoPlaybackCompleteClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("playback_complete")) - val uuaClickHashtagClientEventNamespace: ClientEventNamespace = - ClientEventNamespace(action = Some("hashtag_click")) - val uuaTopicFollowClientEventNamespace1: ClientEventNamespace = - ClientEventNamespace(element = Some("topic"), action = Some("follow")) - val uuaTopicFollowClientEventNamespace2: ClientEventNamespace = - ClientEventNamespace(element = Some("social_proof"), action = Some("follow")) - val uuaTopicFollowClientEventNamespace3: ClientEventNamespace = - ClientEventNamespace(element = Some("feedback_follow_topic"), action = Some("click")) - val uuaTopicUnfollowClientEventNamespace1: ClientEventNamespace = - ClientEventNamespace(element = Some("topic"), action = Some("unfollow")) - val uuaTopicUnfollowClientEventNamespace2: ClientEventNamespace = - ClientEventNamespace(element = Some("social_proof"), action = Some("unfollow")) - val uuaTopicUnfollowClientEventNamespace3: ClientEventNamespace = - ClientEventNamespace(element = Some("feedback_unfollow_topic"), action = Some("click")) - val uuaTopicNotInterestedInClientEventNamespace1: ClientEventNamespace = - ClientEventNamespace(element = Some("topic"), action = Some("not_interested")) - val uuaTopicNotInterestedInClientEventNamespace2: ClientEventNamespace = - ClientEventNamespace( - element = Some("feedback_not_interested_in_topic"), - action = Some("click")) - val uuaTopicUndoNotInterestedInClientEventNamespace1: ClientEventNamespace = - ClientEventNamespace(element = Some("topic"), action = Some("un_not_interested")) - val uuaTopicUndoNotInterestedInClientEventNamespace2: ClientEventNamespace = - ClientEventNamespace( - element = Some("feedback_not_interested_in_topic"), - action = Some("undo")) - val uuaProfileFollowAttempt: ClientEventNamespace = - ClientEventNamespace(action = Some("follow_attempt")) - val uuaTweetFavoriteAttempt: ClientEventNamespace = - ClientEventNamespace(action = Some("favorite_attempt")) - val uuaTweetRetweetAttempt: ClientEventNamespace = - ClientEventNamespace(action = Some("retweet_attempt")) - val uuaTweetReplyAttempt: ClientEventNamespace = - ClientEventNamespace(action = Some("reply_attempt")) - val uuaClientCTALoginClick: ClientEventNamespace = - ClientEventNamespace(action = Some("login")) - val uuaClientCTALoginStart: ClientEventNamespace = - ClientEventNamespace(page = Some("login"), action = Some("show")) - val uuaClientCTALoginSuccess: ClientEventNamespace = - ClientEventNamespace(page = Some("login"), action = Some("success")) - val uuaClientCTASignupClick: ClientEventNamespace = - ClientEventNamespace(action = Some("signup")) - val uuaClientCTASignupSuccess: ClientEventNamespace = - ClientEventNamespace(page = Some("signup"), action = Some("success")) - val uuaNotificationOpen: ClientEventNamespace = - ClientEventNamespace( - page = Some("notification"), - section = Some("status_bar"), - component = None, - action = Some("open")) - val uuaNotificationClick: ClientEventNamespace = - ClientEventNamespace( - page = Some("ntab"), - section = Some("all"), - component = Some("urt"), - element = Some("users_liked_your_tweet"), - action = Some("navigate")) - val uuaTweetReport: ClientEventNamespace = ClientEventNamespace(element = Some("report_tweet")) - val uuaTweetFollowAuthor1: ClientEventNamespace = - ClientEventNamespace(element = Some("follow"), action = Some("click")) - val uuaTweetFollowAuthor2: ClientEventNamespace = - ClientEventNamespace(action = Some("follow")) - val uuaTweetUnfollowAuthor1: ClientEventNamespace = - ClientEventNamespace(element = Some("unfollow"), action = Some("click")) - val uuaTweetUnfollowAuthor2: ClientEventNamespace = - ClientEventNamespace(action = Some("unfollow")) - val uuaNotificationSeeLessOften: ClientEventNamespace = ClientEventNamespace( - page = Some("ntab"), - section = Some("all"), - component = Some("urt"), - action = Some("see_less_often")) - def uuaClientEventNamespace(element: String, action: String): ClientEventNamespace = - ClientEventNamespace(element = Some(element), action = Some(action)) - def uuaTweetReportFlow(page: String, action: String): ClientEventNamespace = - ClientEventNamespace(element = Some("ticket"), page = Some(page), action = Some(action)) - val uuaTweetClick: ClientEventNamespace = - ClientEventNamespace(element = Some("tweet"), action = Some("click")) - def uuaTweetClickProfile: ClientEventNamespace = ClientEventNamespace( - component = Some("tweet"), - element = Some("user"), - action = Some("profile_click")) - val uuaNotificationDismiss: ClientEventNamespace = ClientEventNamespace( - page = Some("notification"), - section = Some("status_bar"), - component = None, - action = Some("dismiss")) - val uuaTypeaheadClick: ClientEventNamespace = - ClientEventNamespace(element = Some("typeahead"), action = Some("click")) - val uuaSearchResultsRelevant: ClientEventNamespace = ClientEventNamespace( - page = Some("search"), - component = Some("did_you_find_it_module"), - element = Some("is_relevant"), - action = Some("click") - ) - val uuaSearchResultsNotRelevant: ClientEventNamespace = ClientEventNamespace( - page = Some("search"), - component = Some("did_you_find_it_module"), - element = Some("not_relevant"), - action = Some("click") - ) - val uuaTweetRelevantToSearch: ClientEventNamespace = ClientEventNamespace( - page = Some("search"), - component = Some("relevance_prompt_module"), - element = Some("is_relevant"), - action = Some("click")) - val uuaTweetNotRelevantToSearch: ClientEventNamespace = ClientEventNamespace( - page = Some("search"), - component = Some("relevance_prompt_module"), - element = Some("not_relevant"), - action = Some("click")) - val uuaProfileBlock: ClientEventNamespace = - ClientEventNamespace(page = Some("profile"), action = Some("block")) - val uuaProfileUnblock: ClientEventNamespace = - ClientEventNamespace(page = Some("profile"), action = Some("unblock")) - val uuaProfileMute: ClientEventNamespace = - ClientEventNamespace(page = Some("profile"), action = Some("mute_user")) - val uuaProfileReport: ClientEventNamespace = - ClientEventNamespace(page = Some("profile"), action = Some("report")) - val uuaProfileShow: ClientEventNamespace = - ClientEventNamespace(page = Some("profile"), action = Some("show")) - val uuaProfileFollow: ClientEventNamespace = - ClientEventNamespace(action = Some("follow")) - val uuaProfileClick: ClientEventNamespace = - ClientEventNamespace(action = Some("profile_click")) - val uuaTweetBlockAuthor: ClientEventNamespace = ClientEventNamespace( - page = Some("profile"), - section = Some("tweets"), - component = Some("tweet"), - action = Some("click"), - element = Some("block") - ) - val uuaTweetUnblockAuthor: ClientEventNamespace = ClientEventNamespace( - section = Some("tweets"), - component = Some("tweet"), - action = Some("click"), - element = Some("unblock") - ) - val uuaTweetMuteAuthor: ClientEventNamespace = ClientEventNamespace( - component = Some("suggest_sc_tweet"), - action = Some("click"), - element = Some("mute") - ) - val uuaAppExit: ClientEventNamespace = - ClientEventNamespace(page = Some("app"), action = Some("become_inactive")) - - // helper methods for creating client-events and UUA objects - def mkLogEvent( - eventName: String = "", - eventNamespace: Option[EventNamespace], - eventDetails: Option[EventDetails] = None, - logBase: Option[LogBase] = None, - pushNotificationDetails: Option[NotificationDetails] = None, - reportDetails: Option[ReportDetails] = None, - searchDetails: Option[SearchDetails] = None, - performanceDetails: Option[PerformanceDetails] = None - ): LogEvent = LogEvent( - eventName = eventName, - eventNamespace = eventNamespace, - eventDetails = eventDetails, - logBase = logBase, - notificationDetails = pushNotificationDetails, - reportDetails = reportDetails, - searchDetails = searchDetails, - performanceDetails = performanceDetails - ) - - def actionTowardDefaultTweetEvent( - eventNamespace: Option[EventNamespace], - impressionDetails: Option[ImpressionDetails] = None, - suggestionDetails: Option[SuggestionDetails] = None, - itemId: Option[Long] = Some(itemTweetId), - mediaDetailsV2: Option[MediaDetailsV2] = None, - clientMediaEvent: Option[ClientMediaEvent] = None, - itemTypeOpt: Option[ItemType] = Some(ItemType.Tweet), - authorId: Option[Long] = None, - isFollowedByActingUser: Option[Boolean] = None, - isFollowingActingUser: Option[Boolean] = None, - notificationTabDetails: Option[NotificationTabDetails] = None, - reportDetails: Option[ReportDetails] = None, - logBase: LogBase = logBase, - tweetPosition: Option[Int] = None, - promotedId: Option[String] = None, - url: Option[String] = None, - targets: Option[Seq[LogEventItem]] = None, - percentVisibleHeight100k: Option[Int] = None, - searchDetails: Option[SearchDetails] = None, - cardDetails: Option[CardDetails] = None - ): LogEvent = - mkLogEvent( - eventName = "action_toward_default_tweet_event", - eventNamespace = eventNamespace, - reportDetails = reportDetails, - eventDetails = Some( - EventDetails( - url = url, - items = Some( - Seq(LogEventItem( - id = itemId, - percentVisibleHeight100k = percentVisibleHeight100k, - itemType = itemTypeOpt, - impressionDetails = impressionDetails, - suggestionDetails = suggestionDetails, - mediaDetailsV2 = mediaDetailsV2, - clientMediaEvent = clientMediaEvent, - cardDetails = cardDetails, - tweetDetails = authorId.map { id => LogEventTweetDetails(authorId = Some(id)) }, - isViewerFollowsTweetAuthor = isFollowedByActingUser, - isTweetAuthorFollowsViewer = isFollowingActingUser, - notificationTabDetails = notificationTabDetails, - position = tweetPosition, - promotedId = promotedId - ))), - targets = targets - ) - ), - logBase = Some(logBase), - searchDetails = searchDetails - ) - - def actionTowardReplyEvent( - eventNamespace: Option[EventNamespace], - inReplyToTweetId: Long = inReplyToTweetId, - impressionDetails: Option[ImpressionDetails] = None - ): LogEvent = - mkLogEvent( - eventName = "action_toward_reply_event", - eventNamespace = eventNamespace, - eventDetails = Some( - EventDetails( - items = Some( - Seq( - LogEventItem( - id = Some(itemTweetId), - itemType = Some(ItemType.Tweet), - impressionDetails = impressionDetails, - tweetDetails = - Some(LogEventTweetDetails(inReplyToTweetId = Some(inReplyToTweetId))) - )) - ) - ) - ), - logBase = Some(logBase) - ) - - def actionTowardRetweetEvent( - eventNamespace: Option[EventNamespace], - inReplyToTweetId: Option[Long] = None, - impressionDetails: Option[ImpressionDetails] = None - ): LogEvent = - mkLogEvent( - eventName = "action_toward_retweet_event", - eventNamespace = eventNamespace, - eventDetails = Some( - EventDetails( - items = Some( - Seq(LogEventItem( - id = Some(itemTweetId), - itemType = Some(ItemType.Tweet), - impressionDetails = impressionDetails, - tweetDetails = Some(LogEventTweetDetails( - retweetingTweetId = Some(retweetingTweetId), - inReplyToTweetId = inReplyToTweetId)) - ))) - ) - ), - logBase = Some(logBase) - ) - - def actionTowardQuoteEvent( - eventNamespace: Option[EventNamespace], - inReplyToTweetId: Option[Long] = None, - quotedAuthorId: Option[Long] = None, - impressionDetails: Option[ImpressionDetails] = None - ): LogEvent = - mkLogEvent( - eventName = "action_toward_quote_event", - eventNamespace = eventNamespace, - eventDetails = Some( - EventDetails( - items = Some( - Seq( - LogEventItem( - id = Some(itemTweetId), - itemType = Some(ItemType.Tweet), - impressionDetails = impressionDetails, - tweetDetails = Some( - LogEventTweetDetails( - quotedTweetId = Some(quotedTweetId), - inReplyToTweetId = inReplyToTweetId, - quotedAuthorId = quotedAuthorId)) - )) - ) - ) - ), - logBase = Some(logBase) - ) - - def actionTowardRetweetEventWithReplyAndQuote( - eventNamespace: Option[EventNamespace], - inReplyToTweetId: Long = inReplyToTweetId, - impressionDetails: Option[ImpressionDetails] = None - ): LogEvent = mkLogEvent( - eventName = "action_toward_retweet_event_with_reply_and_quote", - eventNamespace = eventNamespace, - eventDetails = Some( - EventDetails( - items = Some( - Seq(LogEventItem( - id = Some(itemTweetId), - itemType = Some(ItemType.Tweet), - impressionDetails = impressionDetails, - tweetDetails = Some( - LogEventTweetDetails( - retweetingTweetId = Some(retweetingTweetId), - quotedTweetId = Some(quotedTweetId), - inReplyToTweetId = Some(inReplyToTweetId), - )) - ))) - ) - ), - logBase = Some(logBase) - ) - - def pushNotificationEvent( - eventNamespace: Option[EventNamespace], - itemId: Option[Long] = Some(itemTweetId), - itemTypeOpt: Option[ItemType] = Some(ItemType.Tweet), - notificationDetails: Option[NotificationDetails], - ): LogEvent = - mkLogEvent( - eventName = "push_notification_open", - eventNamespace = eventNamespace, - eventDetails = Some( - EventDetails( - items = Some( - Seq( - LogEventItem( - id = itemId, - itemType = itemTypeOpt, - )))) - ), - logBase = Some(logBase), - pushNotificationDetails = notificationDetails - ) - - def actionTowardNotificationEvent( - eventNamespace: Option[EventNamespace], - notificationTabDetails: Option[NotificationTabDetails], - ): LogEvent = - mkLogEvent( - eventName = "notification_event", - eventNamespace = eventNamespace, - eventDetails = Some( - EventDetails(items = - Some(Seq(LogEventItem(notificationTabDetails = notificationTabDetails))))), - logBase = Some(logBase) - ) - - def profileClickEvent(eventNamespace: Option[EventNamespace]): LogEvent = - mkLogEvent( - eventName = "profile_click", - eventNamespace = eventNamespace, - eventDetails = Some( - EventDetails(items = Some(Seq( - LogEventItem(id = Some(userId), itemType = Some(ItemType.User)), - LogEventItem( - id = Some(itemTweetId), - itemType = Some(ItemType.Tweet), - tweetDetails = Some(LogEventTweetDetails(authorId = Some(authorId)))) - )))), - logBase = Some(logBase) - ) - - def actionTowardProfileEvent( - eventName: String, - eventNamespace: Option[EventNamespace] - ): LogEvent = - mkLogEvent( - eventName = eventName, - eventNamespace = eventNamespace, - eventDetails = Some( - EventDetails(items = Some( - Seq( - LogEventItem(id = Some(itemProfileId), itemType = Some(ItemType.User)) - )))), - logBase = Some(logBase) - ) - - def tweetActionTowardAuthorEvent( - eventName: String, - eventNamespace: Option[EventNamespace] - ): LogEvent = - mkLogEvent( - eventName = eventName, - eventNamespace = eventNamespace, - eventDetails = Some( - EventDetails(items = Some(Seq( - LogEventItem(id = Some(userId), itemType = Some(ItemType.User)), - LogEventItem( - id = Some(itemTweetId), - itemType = Some(ItemType.Tweet), - tweetDetails = Some(LogEventTweetDetails(authorId = Some(authorId)))) - )))), - logBase = Some(logBase) - ) - - def actionTowardsTypeaheadEvent( - eventNamespace: Option[EventNamespace], - targets: Option[Seq[LogEventItem]], - searchQuery: String - ): LogEvent = - mkLogEvent( - eventNamespace = eventNamespace, - eventDetails = Some(EventDetails(targets = targets)), - logBase = Some(logBase), - searchDetails = Some(SearchDetails(query = Some(searchQuery))) - ) - def actionTowardSearchResultPageEvent( - eventNamespace: Option[EventNamespace], - searchDetails: Option[SearchDetails], - items: Option[Seq[LogEventItem]] = None - ): LogEvent = - mkLogEvent( - eventNamespace = eventNamespace, - eventDetails = Some(EventDetails(items = items)), - logBase = Some(logBase), - searchDetails = searchDetails - ) - - def actionTowardsUasEvent( - eventNamespace: Option[EventNamespace], - clientAppId: Option[Long], - duration: Option[Long] - ): LogEvent = - mkLogEvent( - eventNamespace = eventNamespace, - logBase = Some(logBase.copy(clientAppId = clientAppId)), - performanceDetails = Some(PerformanceDetails(durationMs = duration)) - ) - - def mkUUAEventMetadata( - clientEventNamespace: Option[ClientEventNamespace], - traceId: Option[Long] = None, - requestJoinId: Option[Long] = None, - clientAppId: Option[Long] = None - ): EventMetadata = EventMetadata( - sourceTimestampMs = 1001L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ClientEvents, - clientEventNamespace = clientEventNamespace, - traceId = traceId, - requestJoinId = requestJoinId, - clientAppId = clientAppId - ) - - def mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - inReplyToTweetId: Option[Long] = None, - tweetActionInfo: Option[TweetActionInfo] = None, - topicId: Option[Long] = None, - authorInfo: Option[AuthorInfo] = None, - productSurface: Option[ProductSurface] = None, - productSurfaceInfo: Option[ProductSurfaceInfo] = None, - tweetPosition: Option[Int] = None, - promotedId: Option[String] = None, - traceIdOpt: Option[Long] = None, - requestJoinIdOpt: Option[Long] = None, - guestIdMarketingOpt: Option[Long] = None - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = - UserIdentifier(userId = Some(userId), guestIdMarketing = guestIdMarketingOpt), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = itemTweetId, - inReplyToTweetId = inReplyToTweetId, - tweetActionInfo = tweetActionInfo, - actionTweetTopicSocialProofId = topicId, - actionTweetAuthorInfo = authorInfo, - tweetPosition = tweetPosition, - promotedId = promotedId - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata( - clientEventNamespace = clientEventNamespace, - traceId = traceIdOpt, - requestJoinId = requestJoinIdOpt - ), - productSurface = productSurface, - productSurfaceInfo = productSurfaceInfo - ) - - def mkExpectedUUAForActionTowardReplyEvent( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - tweetActionInfo: Option[TweetActionInfo] = None, - authorInfo: Option[AuthorInfo] = None, - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = itemTweetId, - inReplyToTweetId = Some(inReplyToTweetId), - tweetActionInfo = tweetActionInfo, - actionTweetAuthorInfo = authorInfo - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForActionTowardRetweetEvent( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - inReplyToTweetId: Option[Long] = None, - tweetActionInfo: Option[TweetActionInfo] = None, - authorInfo: Option[AuthorInfo] = None, - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = itemTweetId, - retweetingTweetId = Some(retweetingTweetId), - inReplyToTweetId = inReplyToTweetId, - tweetActionInfo = tweetActionInfo, - actionTweetAuthorInfo = authorInfo - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForActionTowardQuoteEvent( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - inReplyToTweetId: Option[Long] = None, - quotedAuthorId: Option[Long] = None, - tweetActionInfo: Option[TweetActionInfo] = None, - authorInfo: Option[AuthorInfo] = None, - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = itemTweetId, - quotedTweetId = Some(quotedTweetId), - quotedAuthorId = quotedAuthorId, - inReplyToTweetId = inReplyToTweetId, - tweetActionInfo = tweetActionInfo, - actionTweetAuthorInfo = authorInfo - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForActionTowardQuotingEvent( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - inReplyToTweetId: Option[Long] = None, - tweetActionInfo: Option[TweetActionInfo] = None, - authorInfo: Option[AuthorInfo] = None, - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = quotedTweetId, - quotingTweetId = Some(itemTweetId), - inReplyToTweetId = inReplyToTweetId, - tweetActionInfo = tweetActionInfo, - actionTweetAuthorInfo = authorInfo - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoted( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - inReplyToTweetId: Long = inReplyToTweetId, - tweetActionInfo: Option[TweetActionInfo] = None, - authorInfo: Option[AuthorInfo] = None, - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = itemTweetId, - retweetingTweetId = Some(retweetingTweetId), - quotedTweetId = Some(quotedTweetId), - inReplyToTweetId = Some(inReplyToTweetId), - tweetActionInfo = tweetActionInfo, - actionTweetAuthorInfo = authorInfo - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoting( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - inReplyToTweetId: Long = inReplyToTweetId, - tweetActionInfo: Option[TweetActionInfo] = None, - authorInfo: Option[AuthorInfo] = None, - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = quotedTweetId, - quotingTweetId = Some(itemTweetId), - tweetActionInfo = tweetActionInfo, - actionTweetAuthorInfo = authorInfo - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForActionTowardTopicEvent( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - topicId: Long, - traceId: Option[Long] = None, - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TopicInfo( - TopicInfo( - actionTopicId = topicId, - ) - ), - actionType = actionType, - eventMetadata = - mkUUAEventMetadata(clientEventNamespace = clientEventNamespace, traceId = traceId) - ) - - def mkExpectedUUAForNotificationEvent( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - notificationContent: NotificationContent, - productSurface: Option[ProductSurface], - productSurfaceInfo: Option[ProductSurfaceInfo], - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.NotificationInfo( - NotificationInfo( - actionNotificationId = notificationId, - content = notificationContent - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace), - productSurface = productSurface, - productSurfaceInfo = productSurfaceInfo - ) - - def mkExpectedUUAForProfileClick( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - authorInfo: Option[AuthorInfo] = None - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = itemTweetId, - actionTweetAuthorInfo = authorInfo - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForTweetActionTowardAuthor( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - authorInfo: Option[AuthorInfo] = None, - tweetActionInfo: Option[TweetActionInfo] = None - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = itemTweetId, - actionTweetAuthorInfo = authorInfo, - tweetActionInfo = tweetActionInfo - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForProfileAction( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - actionProfileId: Long - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.ProfileInfo( - ProfileInfo( - actionProfileId = actionProfileId - ) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForTypeaheadAction( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - typeaheadActionInfo: TypeaheadActionInfo, - searchQuery: String, - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TypeaheadInfo( - TypeaheadInfo(actionQuery = searchQuery, typeaheadActionInfo = typeaheadActionInfo) - ), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace), - productSurface = Some(ProductSurface.SearchTypeahead), - productSurfaceInfo = - Some(ProductSurfaceInfo.SearchTypeaheadInfo(SearchTypeaheadInfo(query = searchQuery))) - ) - def mkExpectedUUAForFeedbackSubmitAction( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - feedbackPromptInfo: FeedbackPromptInfo, - searchQuery: String - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.FeedbackPromptInfo(feedbackPromptInfo), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace), - productSurface = Some(ProductSurface.SearchResultsPage), - productSurfaceInfo = - Some(ProductSurfaceInfo.SearchResultsPageInfo(SearchResultsPageInfo(query = searchQuery))) - ) - - def mkExpectedUUAForActionTowardCTAEvent( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - guestIdMarketingOpt: Option[Long] - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = - UserIdentifier(userId = Some(userId), guestIdMarketing = guestIdMarketingOpt), - item = Item.CtaInfo(CTAInfo()), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def mkExpectedUUAForUasEvent( - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - clientAppId: Option[Long], - duration: Option[Long] - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.UasInfo(UASInfo(timeSpentMs = duration.get)), - actionType = actionType, - eventMetadata = - mkUUAEventMetadata(clientEventNamespace = clientEventNamespace, clientAppId = clientAppId) - ) - - def mkExpectedUUAForCardEvent( - id: Option[Long], - clientEventNamespace: Option[ClientEventNamespace], - actionType: ActionType, - itemType: Option[ItemType], - authorId: Option[Long], - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.CardInfo( - CardInfo( - id = id, - itemType = itemType, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = authorId)))), - actionType = actionType, - eventMetadata = mkUUAEventMetadata(clientEventNamespace = clientEventNamespace) - ) - - def timelineTopicControllerData(topicId: Long = topicId): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.TimelinesTopic( - TimelinesTopicControllerData.V1( - TimelinesTopicControllerDataV1( - topicId = topicId, - topicTypesBitmap = 1 - ) - ))) - - def homeTweetControllerData( - topicId: Long = topicId, - traceId: Long = traceId - ): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.HomeTweets( - HomeTweetsControllerData.V1( - HomeTweetsControllerDataV1( - topicId = Some(topicId), - traceId = Some(traceId) - )))) - - def homeTweetControllerDataV2( - injectedPosition: Option[Int] = None, - requestJoinId: Option[Long] = None, - traceId: Option[Long] = None - ): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.HomeTweets( - HomeTweetsControllerData.V1( - HomeTweetsControllerDataV1( - injectedPosition = injectedPosition, - traceId = traceId, - requestJoinId = requestJoinId - )))) - - // mock client-events - val ddgEvent: LogEvent = mkLogEvent( - eventName = "ddg", - eventNamespace = Some( - EventNamespace( - page = Some("ddg"), - action = Some("experiment") - ) - ) - ) - - val qigRankerEvent: LogEvent = mkLogEvent( - eventName = "qig_ranker", - eventNamespace = Some( - EventNamespace( - page = Some("qig_ranker"), - ) - ) - ) - - val timelineMixerEvent: LogEvent = mkLogEvent( - eventName = "timelinemixer", - eventNamespace = Some( - EventNamespace( - page = Some("timelinemixer"), - ) - ) - ) - - val timelineServiceEvent: LogEvent = mkLogEvent( - eventName = "timelineservice", - eventNamespace = Some( - EventNamespace( - page = Some("timelineservice"), - ) - ) - ) - - val tweetConcServiceEvent: LogEvent = mkLogEvent( - eventName = "tweetconvosvc", - eventNamespace = Some( - EventNamespace( - page = Some("tweetconvosvc"), - ) - ) - ) - - val renderNonTweetItemTypeEvent: LogEvent = mkLogEvent( - eventName = "render non-tweet item-type", - eventNamespace = Some(ceRenderEventNamespace), - eventDetails = Some( - EventDetails( - items = Some( - Seq(LogEventItem(itemType = Some(ItemType.Event))) - ) - ) - ) - ) - - val renderDefaultTweetWithTopicIdEvent: LogEvent = actionTowardDefaultTweetEvent( - eventNamespace = Some(ceRenderEventNamespace), - suggestionDetails = - Some(SuggestionDetails(decodedControllerData = Some(timelineTopicControllerData()))) - ) - - def renderDefaultTweetUserFollowStatusEvent( - authorId: Option[Long], - isFollowedByActingUser: Boolean = false, - isFollowingActingUser: Boolean = false - ): LogEvent = actionTowardDefaultTweetEvent( - eventNamespace = Some(ceRenderEventNamespace), - authorId = authorId, - isFollowedByActingUser = Some(isFollowedByActingUser), - isFollowingActingUser = Some(isFollowingActingUser) - ) - - val lingerDefaultTweetEvent: LogEvent = actionTowardDefaultTweetEvent( - eventNamespace = Some(ceLingerEventNamespace), - impressionDetails = Some( - ImpressionDetails( - visibilityStart = Some(100L), - visibilityEnd = Some(105L) - )) - ) - - val lingerReplyEvent: LogEvent = actionTowardReplyEvent( - eventNamespace = Some(ceLingerEventNamespace), - impressionDetails = Some( - ImpressionDetails( - visibilityStart = Some(100L), - visibilityEnd = Some(105L) - )) - ) - - val lingerRetweetEvent: LogEvent = actionTowardRetweetEvent( - eventNamespace = Some(ceLingerEventNamespace), - impressionDetails = Some( - ImpressionDetails( - visibilityStart = Some(100L), - visibilityEnd = Some(105L) - )) - ) - - val lingerQuoteEvent: LogEvent = actionTowardQuoteEvent( - eventNamespace = Some(ceLingerEventNamespace), - impressionDetails = Some( - ImpressionDetails( - visibilityStart = Some(100L), - visibilityEnd = Some(105L) - )) - ) - - val lingerRetweetWithReplyAndQuoteEvent: LogEvent = actionTowardRetweetEventWithReplyAndQuote( - eventNamespace = Some(ceLingerEventNamespace), - impressionDetails = Some( - ImpressionDetails( - visibilityStart = Some(100L), - visibilityEnd = Some(105L) - )) - ) - - val replyToDefaultTweetOrReplyEvent: LogEvent = actionTowardReplyEvent( - eventNamespace = Some(ceReplyEventNamespace), - // since the action is reply, item.id = inReplyToTweetId - inReplyToTweetId = itemTweetId, - ) - - val replyToRetweetEvent: LogEvent = actionTowardRetweetEvent( - eventNamespace = Some(ceReplyEventNamespace), - // since the action is reply, item.id = inReplyToTweetId - inReplyToTweetId = Some(itemTweetId), - ) - - val replyToQuoteEvent: LogEvent = actionTowardQuoteEvent( - eventNamespace = Some(ceReplyEventNamespace), - // since the action is reply, item.id = inReplyToTweetId - inReplyToTweetId = Some(itemTweetId), - ) - - val replyToRetweetWithReplyAndQuoteEvent: LogEvent = actionTowardRetweetEventWithReplyAndQuote( - eventNamespace = Some(ceReplyEventNamespace), - // since the action is reply, item.id = inReplyToTweetId - inReplyToTweetId = itemTweetId, - ) - - // expected UUA corresponding to mock client-events - val expectedTweetRenderDefaultTweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaRenderClientEventNamespace), - actionType = ActionType.ClientTweetRenderImpression - ) - - val expectedTweetRenderReplyUUA: UnifiedUserAction = mkExpectedUUAForActionTowardReplyEvent( - clientEventNamespace = Some(uuaRenderClientEventNamespace), - actionType = ActionType.ClientTweetRenderImpression - ) - - val expectedTweetRenderRetweetUUA: UnifiedUserAction = mkExpectedUUAForActionTowardRetweetEvent( - clientEventNamespace = Some(uuaRenderClientEventNamespace), - actionType = ActionType.ClientTweetRenderImpression - ) - - val expectedTweetRenderQuoteUUA1: UnifiedUserAction = mkExpectedUUAForActionTowardQuoteEvent( - clientEventNamespace = Some(uuaRenderClientEventNamespace), - actionType = ActionType.ClientTweetRenderImpression, - quotedAuthorId = Some(quotedAuthorId), - ) - val expectedTweetRenderQuoteUUA2: UnifiedUserAction = mkExpectedUUAForActionTowardQuotingEvent( - clientEventNamespace = Some(uuaRenderClientEventNamespace), - actionType = ActionType.ClientTweetRenderImpression, - authorInfo = Some(AuthorInfo(authorId = Some(quotedAuthorId))) - ) - - val expectedTweetRenderRetweetWithReplyAndQuoteUUA1: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoted( - clientEventNamespace = Some(uuaRenderClientEventNamespace), - actionType = ActionType.ClientTweetRenderImpression - ) - val expectedTweetRenderRetweetWithReplyAndQuoteUUA2: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoting( - clientEventNamespace = Some(uuaRenderClientEventNamespace), - actionType = ActionType.ClientTweetRenderImpression - ) - - val expectedTweetRenderDefaultTweetWithTopicIdUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaRenderClientEventNamespace), - actionType = ActionType.ClientTweetRenderImpression, - topicId = Some(topicId) - ) - - val expectedTweetDetailImpressionUUA1: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(ceTweetDetailsClientEventNamespace1), - actionType = ActionType.ClientTweetDetailsImpression - ) - - val expectedTweetGalleryImpressionUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(ceGalleryClientEventNamespace), - actionType = ActionType.ClientTweetGalleryImpression - ) - - def expectedTweetRenderDefaultTweetWithAuthorInfoUUA( - authorInfo: Option[AuthorInfo] = None - ): UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaRenderClientEventNamespace), - actionType = ActionType.ClientTweetRenderImpression, - authorInfo = authorInfo - ) - - val expectedTweetLingerDefaultTweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaLingerClientEventNamespace), - actionType = ActionType.ClientTweetLingerImpression, - tweetActionInfo = Some( - TweetActionInfo.ClientTweetLingerImpression( - ClientTweetLingerImpression( - lingerStartTimestampMs = 100L, - lingerEndTimestampMs = 105L - )) - ) - ) - - val expectedTweetLingerReplyUUA: UnifiedUserAction = mkExpectedUUAForActionTowardReplyEvent( - clientEventNamespace = Some(uuaLingerClientEventNamespace), - actionType = ActionType.ClientTweetLingerImpression, - tweetActionInfo = Some( - TweetActionInfo.ClientTweetLingerImpression( - ClientTweetLingerImpression( - lingerStartTimestampMs = 100L, - lingerEndTimestampMs = 105L - )) - ) - ) - - val expectedTweetLingerRetweetUUA: UnifiedUserAction = mkExpectedUUAForActionTowardRetweetEvent( - clientEventNamespace = Some(uuaLingerClientEventNamespace), - actionType = ActionType.ClientTweetLingerImpression, - tweetActionInfo = Some( - TweetActionInfo.ClientTweetLingerImpression( - ClientTweetLingerImpression( - lingerStartTimestampMs = 100L, - lingerEndTimestampMs = 105L - )) - ) - ) - - val expectedTweetLingerQuoteUUA: UnifiedUserAction = mkExpectedUUAForActionTowardQuoteEvent( - clientEventNamespace = Some(uuaLingerClientEventNamespace), - actionType = ActionType.ClientTweetLingerImpression, - tweetActionInfo = Some( - TweetActionInfo.ClientTweetLingerImpression( - ClientTweetLingerImpression( - lingerStartTimestampMs = 100L, - lingerEndTimestampMs = 105L - )) - ) - ) - - val expectedTweetLingerRetweetWithReplyAndQuoteUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoted( - clientEventNamespace = Some(uuaLingerClientEventNamespace), - actionType = ActionType.ClientTweetLingerImpression, - tweetActionInfo = Some( - TweetActionInfo.ClientTweetLingerImpression( - ClientTweetLingerImpression( - lingerStartTimestampMs = 100L, - lingerEndTimestampMs = 105L - )) - ) - ) - - val expectedTweetClickQuoteUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoted( - clientEventNamespace = Some( - ClientEventNamespace( - action = Some("quote") - )), - actionType = ActionType.ClientTweetClickQuote - ) - - def expectedTweetQuoteUUA(action: String): UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoted( - clientEventNamespace = Some( - ClientEventNamespace( - action = Some(action) - )), - actionType = ActionType.ClientTweetQuote - ) - - val expectedTweetFavoriteDefaultTweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav - ) - - val expectedHomeTweetEventWithControllerDataSuggestType: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaHomeFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav, - productSurface = Some(ProductSurface.HomeTimeline), - productSurfaceInfo = Some( - ProductSurfaceInfo.HomeTimelineInfo( - HomeTimelineInfo(suggestionType = Some("Test_type"), injectedPosition = Some(1)))), - traceIdOpt = Some(traceId), - requestJoinIdOpt = Some(requestJoinId) - ) - - val expectedHomeTweetEventWithControllerData: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaHomeFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav, - productSurface = Some(ProductSurface.HomeTimeline), - productSurfaceInfo = - Some(ProductSurfaceInfo.HomeTimelineInfo(HomeTimelineInfo(injectedPosition = Some(1)))), - traceIdOpt = Some(traceId), - requestJoinIdOpt = Some(requestJoinId) - ) - - val expectedSearchTweetEventWithControllerData: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaSearchFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav, - productSurface = Some(ProductSurface.SearchResultsPage), - productSurfaceInfo = - Some(ProductSurfaceInfo.SearchResultsPageInfo(SearchResultsPageInfo(query = "twitter"))), - traceIdOpt = Some(traceId), - requestJoinIdOpt = Some(requestJoinId) - ) - - val expectedHomeTweetEventWithSuggestType: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaHomeFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav, - productSurface = Some(ProductSurface.HomeTimeline), - productSurfaceInfo = Some( - ProductSurfaceInfo.HomeTimelineInfo(HomeTimelineInfo(suggestionType = Some("Test_type")))) - ) - - val expectedHomeLatestTweetEventWithControllerDataSuggestType: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaHomeLatestFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav, - productSurface = Some(ProductSurface.HomeTimeline), - productSurfaceInfo = Some( - ProductSurfaceInfo.HomeTimelineInfo( - HomeTimelineInfo(suggestionType = Some("Test_type"), injectedPosition = Some(1)))), - traceIdOpt = Some(traceId), - requestJoinIdOpt = Some(requestJoinId) - ) - - val expectedHomeLatestTweetEventWithControllerData: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaHomeLatestFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav, - productSurface = Some(ProductSurface.HomeTimeline), - productSurfaceInfo = - Some(ProductSurfaceInfo.HomeTimelineInfo(HomeTimelineInfo(injectedPosition = Some(1)))), - traceIdOpt = Some(traceId), - requestJoinIdOpt = Some(requestJoinId) - ) - - val expectedHomeLatestTweetEventWithSuggestType: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaHomeLatestFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav, - productSurface = Some(ProductSurface.HomeTimeline), - productSurfaceInfo = Some( - ProductSurfaceInfo.HomeTimelineInfo(HomeTimelineInfo(suggestionType = Some("Test_type")))) - ) - - val expectedTweetFavoriteReplyUUA: UnifiedUserAction = mkExpectedUUAForActionTowardReplyEvent( - clientEventNamespace = Some(uuaFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav - ) - - val expectedTweetFavoriteRetweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEvent( - clientEventNamespace = Some(uuaFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav - ) - - val expectedTweetFavoriteQuoteUUA: UnifiedUserAction = mkExpectedUUAForActionTowardQuoteEvent( - clientEventNamespace = Some(uuaFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav) - - val expectedTweetFavoriteRetweetWithReplyAndQuoteUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoted( - clientEventNamespace = Some(uuaFavoriteClientEventNamespace), - actionType = ActionType.ClientTweetFav - ) - - val expectedTweetClickReplyDefaultTweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaClickReplyClientEventNamespace), - actionType = ActionType.ClientTweetClickReply - ) - - val expectedTweetClickReplyReplyUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardReplyEvent( - clientEventNamespace = Some(uuaClickReplyClientEventNamespace), - actionType = ActionType.ClientTweetClickReply - ) - - val expectedTweetClickReplyRetweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEvent( - clientEventNamespace = Some(uuaClickReplyClientEventNamespace), - actionType = ActionType.ClientTweetClickReply - ) - - val expectedTweetClickReplyQuoteUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardQuoteEvent( - clientEventNamespace = Some(uuaClickReplyClientEventNamespace), - actionType = ActionType.ClientTweetClickReply - ) - - val expectedTweetClickReplyRetweetWithReplyAndQuoteUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoted( - clientEventNamespace = Some(uuaClickReplyClientEventNamespace), - actionType = ActionType.ClientTweetClickReply - ) - - val expectedTweetReplyDefaultTweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaReplyClientEventNamespace), - actionType = ActionType.ClientTweetReply, - inReplyToTweetId = Some(itemTweetId) - ) - - val expectedTweetReplyRetweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEvent( - clientEventNamespace = Some(uuaReplyClientEventNamespace), - actionType = ActionType.ClientTweetReply, - inReplyToTweetId = Some(itemTweetId) - ) - - val expectedTweetReplyQuoteUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardQuoteEvent( - clientEventNamespace = Some(uuaReplyClientEventNamespace), - actionType = ActionType.ClientTweetReply, - inReplyToTweetId = Some(itemTweetId) - ) - - val expectedTweetReplyRetweetWithReplyAndQuoteUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoted( - clientEventNamespace = Some(uuaReplyClientEventNamespace), - actionType = ActionType.ClientTweetReply, - inReplyToTweetId = itemTweetId - ) - - val expectedTweetRetweetDefaultTweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardDefaultTweetEvent( - clientEventNamespace = Some(uuaRetweetClientEventNamespace), - actionType = ActionType.ClientTweetRetweet - ) - - val expectedTweetRetweetReplyUUA: UnifiedUserAction = mkExpectedUUAForActionTowardReplyEvent( - clientEventNamespace = Some(uuaRetweetClientEventNamespace), - actionType = ActionType.ClientTweetRetweet - ) - - val expectedTweetRetweetRetweetUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEvent( - clientEventNamespace = Some(uuaRetweetClientEventNamespace), - actionType = ActionType.ClientTweetRetweet - ) - - val expectedTweetRetweetQuoteUUA: UnifiedUserAction = mkExpectedUUAForActionTowardQuoteEvent( - clientEventNamespace = Some(uuaRetweetClientEventNamespace), - actionType = ActionType.ClientTweetRetweet - ) - - val expectedTweetRetweetRetweetWithReplyAndQuoteUUA: UnifiedUserAction = - mkExpectedUUAForActionTowardRetweetEventWithReplyAndQuoted( - clientEventNamespace = Some(uuaRetweetClientEventNamespace), - actionType = ActionType.ClientTweetRetweet - ) - } - - trait EmailNotificationEventFixture extends CommonFixture { - val timestamp = 1001L - val pageUrlStatus = - "https://twitter.com/a/status/3?cn=a%3D%3D&refsrc=email" - val tweetIdStatus = 3L - - val pageUrlEvent = - "https://twitter.com/i/events/2?cn=a%3D%3D&refsrc=email" - val tweetIdEvent = 2L - - val pageUrlNoArgs = "https://twitter.com/i/events/1" - val tweetIdNoArgs = 1L - - val logBase1: LogBase = LogBase( - transactionId = "test", - ipAddress = "127.0.0.1", - userId = Some(userId), - guestId = Some(2L), - timestamp = timestamp, - page = Some(pageUrlStatus), - ) - - val logBase2: LogBase = LogBase( - transactionId = "test", - ipAddress = "127.0.0.1", - userId = Some(userId), - guestId = Some(2L), - timestamp = timestamp - ) - - val notificationEvent: NotificationScribe = NotificationScribe( - `type` = NotificationScribeType.Click, - impressionId = Some("1234"), - userId = Some(userId), - timestamp = timestamp, - logBase = Some(logBase1) - ) - - val notificationEventWOTweetId: NotificationScribe = NotificationScribe( - `type` = NotificationScribeType.Click, - impressionId = Some("1234"), - userId = Some(userId), - timestamp = timestamp, - logBase = Some(logBase2) - ) - - val notificationEventWOImpressionId: NotificationScribe = NotificationScribe( - `type` = NotificationScribeType.Click, - userId = Some(userId), - timestamp = timestamp, - logBase = Some(logBase1) - ) - - val expectedUua: UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = tweetIdStatus, - ) - ), - actionType = ActionType.ClientTweetEmailClick, - eventMetadata = EventMetadata( - sourceTimestampMs = timestamp, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.EmailNotificationEvents, - traceId = None - ), - productSurfaceInfo = Some( - ProductSurfaceInfo.EmailNotificationInfo(EmailNotificationInfo(notificationId = "1234"))), - productSurface = Some(ProductSurface.EmailNotification) - ) - } - - trait UserModificationEventFixture extends CommonFixture { - val timestamp = 1001L - val userName = "A" - val screenName = "B" - val description = "this is A" - val location = "US" - val url = s"https://www.twitter.com/${userName}" - - val baseUserModification = UserModification( - forUserId = Some(userId), - userId = Some(userId), - ) - - val userCreate = baseUserModification.copy( - create = Some( - User( - id = userId, - createdAtMsec = timestamp, - updatedAtMsec = timestamp, - userType = UserType.Normal, - profile = Some( - Profile( - name = userName, - screenName = screenName, - description = description, - auth = null.asInstanceOf[Auth], - location = location, - url = url - )) - )), - ) - - val updateDiffs = Seq( - UpdateDiffItem(fieldName = "user_name", before = Some("abc"), after = Some("def")), - UpdateDiffItem(fieldName = "description", before = Some("d1"), after = Some("d2")), - ) - val userUpdate = baseUserModification.copy( - updatedAtMsec = Some(timestamp), - update = Some(updateDiffs), - success = Some(true) - ) - - val expectedUuaUserCreate: UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.ProfileInfo( - ProfileInfo( - actionProfileId = userId, - name = Some(userName), - handle = Some(screenName), - description = Some(description) - ) - ), - actionType = ActionType.ServerUserCreate, - eventMetadata = EventMetadata( - sourceTimestampMs = timestamp, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerGizmoduckUserModificationEvents, - ) - ) - - val expectedUuaUserUpdate: UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.ProfileInfo( - ProfileInfo( - actionProfileId = userId, - profileActionInfo = Some( - ProfileActionInfo.ServerUserUpdate( - ServerUserUpdate(updates = updateDiffs, success = Some(true)))) - ) - ), - actionType = ActionType.ServerUserUpdate, - eventMetadata = EventMetadata( - sourceTimestampMs = timestamp, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerGizmoduckUserModificationEvents, - ) - ) - } - - trait AdsCallbackEngagementsFixture extends CommonFixture { - - val timestamp = 1001L - val engagementId = 123 - val accountTimeZone = "PST" - val advertiserId = 2002L - val displayLocation: DisplayLocation = DisplayLocation(value = 1) - val trendId = 1002 - - val authorInfo: AuthorInfo = AuthorInfo(authorId = Some(advertiserId)) - val openLinkWithUrl: TweetActionInfo = - TweetActionInfo.ServerPromotedTweetOpenLink(ServerPromotedTweetOpenLink(url = Some("go/url"))) - val openLinkWithoutUrl: TweetActionInfo = - TweetActionInfo.ServerPromotedTweetOpenLink(ServerPromotedTweetOpenLink(url = None)) - - def createTweetInfoItem( - authorInfo: Option[AuthorInfo] = None, - actionInfo: Option[TweetActionInfo] = None - ): Item = { - Item.TweetInfo( - TweetInfo( - actionTweetId = itemTweetId, - actionTweetAuthorInfo = authorInfo, - tweetActionInfo = actionInfo)) - } - - val trendInfoItem: Item = Item.TrendInfo(TrendInfo(actionTrendId = trendId)) - - val organicTweetId = Some(100001L) - val promotedTweetId = Some(200002L) - - val organicTweetVideoUuid = Some("organic_video_1") - val organicTweetVideoOwnerId = Some(123L) - - val promotedTweetVideoUuid = Some("promoted_video_1") - val promotedTweetVideoOwnerId = Some(345L) - - val prerollAdUuid = Some("preroll_ad_1") - val prerollAdOwnerId = Some(567L) - - val amplifyDetailsPrerollAd = Some( - AmplifyDetails( - videoOwnerId = prerollAdOwnerId, - videoUuid = prerollAdUuid, - prerollOwnerId = prerollAdOwnerId, - prerollUuid = prerollAdUuid - )) - - val tweetActionInfoPrerollAd = Some( - TweetActionInfo.TweetVideoWatch( - TweetVideoWatch( - isMonetizable = Some(true), - videoOwnerId = prerollAdOwnerId, - videoUuid = prerollAdUuid, - prerollOwnerId = prerollAdOwnerId, - prerollUuid = prerollAdUuid - ) - ) - ) - - val amplifyDetailsPromotedTweetWithoutAd = Some( - AmplifyDetails( - videoOwnerId = promotedTweetVideoOwnerId, - videoUuid = promotedTweetVideoUuid - )) - - val tweetActionInfoPromotedTweetWithoutAd = Some( - TweetActionInfo.TweetVideoWatch( - TweetVideoWatch( - isMonetizable = Some(true), - videoOwnerId = promotedTweetVideoOwnerId, - videoUuid = promotedTweetVideoUuid, - ) - ) - ) - - val amplifyDetailsPromotedTweetWithAd = Some( - AmplifyDetails( - videoOwnerId = promotedTweetVideoOwnerId, - videoUuid = promotedTweetVideoUuid, - prerollOwnerId = prerollAdOwnerId, - prerollUuid = prerollAdUuid - )) - - val tweetActionInfoPromotedTweetWithAd = Some( - TweetActionInfo.TweetVideoWatch( - TweetVideoWatch( - isMonetizable = Some(true), - videoOwnerId = promotedTweetVideoOwnerId, - videoUuid = promotedTweetVideoUuid, - prerollOwnerId = prerollAdOwnerId, - prerollUuid = prerollAdUuid - ) - ) - ) - - val amplifyDetailsOrganicTweetWithAd = Some( - AmplifyDetails( - videoOwnerId = organicTweetVideoOwnerId, - videoUuid = organicTweetVideoUuid, - prerollOwnerId = prerollAdOwnerId, - prerollUuid = prerollAdUuid - )) - - val tweetActionInfoOrganicTweetWithAd = Some( - TweetActionInfo.TweetVideoWatch( - TweetVideoWatch( - isMonetizable = Some(true), - videoOwnerId = organicTweetVideoOwnerId, - videoUuid = organicTweetVideoUuid, - prerollOwnerId = prerollAdOwnerId, - prerollUuid = prerollAdUuid - ) - ) - ) - - def createExpectedUua( - actionType: ActionType, - item: Item - ): UnifiedUserAction = { - UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = item, - actionType = actionType, - eventMetadata = EventMetadata( - sourceTimestampMs = timestamp, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerAdsCallbackEngagements - ) - ) - } - - def createExpectedUuaWithProfileInfo( - actionType: ActionType - ): UnifiedUserAction = { - UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.ProfileInfo(ProfileInfo(actionProfileId = advertiserId)), - actionType = actionType, - eventMetadata = EventMetadata( - sourceTimestampMs = timestamp, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerAdsCallbackEngagements - ) - ) - } - - def createSpendServerEvent( - engagementType: EngagementType, - url: Option[String] = None - ): SpendServerEvent = { - SpendServerEvent( - engagementEvent = Some( - EngagementEvent( - clientInfo = Some(ClientInfo(userId64 = Some(userId))), - engagementId = engagementId, - engagementEpochTimeMilliSec = timestamp, - engagementType = engagementType, - accountTimeZone = accountTimeZone, - url = url, - impressionData = Some( - ImpressionDataNeededAtEngagementTime( - advertiserId = advertiserId, - promotedTweetId = Some(itemTweetId), - displayLocation = displayLocation, - promotedTrendId = Some(trendId))) - ))) - } - - def createExpectedVideoUua( - actionType: ActionType, - tweetActionInfo: Option[TweetActionInfo], - actionTweetId: Option[Long] - ): UnifiedUserAction = { - UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = actionTweetId.get, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(advertiserId))), - tweetActionInfo = tweetActionInfo - ) - ), - actionType = actionType, - eventMetadata = EventMetadata( - sourceTimestampMs = timestamp, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerAdsCallbackEngagements - ) - ) - } - - def createVideoSpendServerEvent( - engagementType: EngagementType, - amplifyDetails: Option[AmplifyDetails], - promotedTweetId: Option[Long], - organicTweetId: Option[Long] - ): SpendServerEvent = { - SpendServerEvent( - engagementEvent = Some( - EngagementEvent( - clientInfo = Some(ClientInfo(userId64 = Some(userId))), - engagementId = engagementId, - engagementEpochTimeMilliSec = timestamp, - engagementType = engagementType, - accountTimeZone = accountTimeZone, - impressionData = Some( - ImpressionDataNeededAtEngagementTime( - advertiserId = advertiserId, - promotedTweetId = promotedTweetId, - displayLocation = displayLocation, - organicTweetId = organicTweetId)), - cardEngagement = Some( - CardEvent( - amplifyDetails = amplifyDetails - ) - ) - ))) - } - } - - trait InteractionEventsFixtures extends CommonFixture { - val timestamp = 123456L - val tweetId = 1L - val engagingUserId = 11L - - val baseInteractionEvent: InteractionEvent = InteractionEvent( - targetId = tweetId, - targetType = InteractionTargetType.Tweet, - engagingUserId = engagingUserId, - eventSource = EventSource.ClientEvent, - timestampMillis = timestamp, - interactionType = Some(InteractionType.TweetRenderImpression), - details = InteractionDetails.TweetRenderImpression(TweetImpression()), - additionalEngagingUserIdentifiers = UserIdentifierIE(), - engagingContext = EngagingContext.ClientEventContext( - ClientEventContext( - clientEventNamespace = ContextualEventNamespace(), - clientType = ClientType.Iphone, - displayLocation = DisplayLocation(1), - isTweetDetailsImpression = Some(false))) - ) - - val loggedOutInteractionEvent: InteractionEvent = baseInteractionEvent.copy(engagingUserId = 0L) - - val detailImpressionInteractionEvent: InteractionEvent = baseInteractionEvent.copy( - engagingContext = EngagingContext.ClientEventContext( - ClientEventContext( - clientEventNamespace = ContextualEventNamespace(), - clientType = ClientType.Iphone, - displayLocation = DisplayLocation(1), - isTweetDetailsImpression = Some(true))) - ) - - val expectedBaseKeyedUuaTweet: KeyedUuaTweet = KeyedUuaTweet( - tweetId = tweetId, - actionType = ActionType.ClientTweetRenderImpression, - userIdentifier = UserIdentifier(userId = Some(engagingUserId)), - eventMetadata = EventMetadata( - sourceTimestampMs = timestamp, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ClientEvents - ) - ) - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TlsFavsAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TlsFavsAdapterSpec.docx new file mode 100644 index 000000000..72810cc30 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TlsFavsAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TlsFavsAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TlsFavsAdapterSpec.scala deleted file mode 100644 index a627cac95..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TlsFavsAdapterSpec.scala +++ /dev/null @@ -1,205 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.context.thriftscala.Viewer -import com.twitter.inject.Test -import com.twitter.timelineservice.thriftscala._ -import com.twitter.unified_user_actions.adapter.tls_favs_event.TlsFavsAdapter -import com.twitter.unified_user_actions.thriftscala._ -import com.twitter.util.Time - -class TlsFavsAdapterSpec extends Test { - trait Fixture { - - val frozenTime = Time.fromMilliseconds(1658949273000L) - - val favEventNoRetweet = ContextualizedFavoriteEvent( - event = FavoriteEventUnion.Favorite( - FavoriteEvent( - userId = 91L, - tweetId = 1L, - tweetUserId = 101L, - eventTimeMs = 1001L - ) - ), - context = LogEventContext(hostname = "", traceId = 31L) - ) - val favEventRetweet = ContextualizedFavoriteEvent( - event = FavoriteEventUnion.Favorite( - FavoriteEvent( - userId = 92L, - tweetId = 2L, - tweetUserId = 102L, - eventTimeMs = 1002L, - retweetId = Some(22L) - ) - ), - context = LogEventContext(hostname = "", traceId = 32L) - ) - val unfavEventNoRetweet = ContextualizedFavoriteEvent( - event = FavoriteEventUnion.Unfavorite( - UnfavoriteEvent( - userId = 93L, - tweetId = 3L, - tweetUserId = 103L, - eventTimeMs = 1003L - ) - ), - context = LogEventContext(hostname = "", traceId = 33L) - ) - val unfavEventRetweet = ContextualizedFavoriteEvent( - event = FavoriteEventUnion.Unfavorite( - UnfavoriteEvent( - userId = 94L, - tweetId = 4L, - tweetUserId = 104L, - eventTimeMs = 1004L, - retweetId = Some(44L) - ) - ), - context = LogEventContext(hostname = "", traceId = 34L) - ) - val favEventWithLangAndCountry = ContextualizedFavoriteEvent( - event = FavoriteEventUnion.Favorite( - FavoriteEvent( - userId = 91L, - tweetId = 1L, - tweetUserId = 101L, - eventTimeMs = 1001L, - viewerContext = - Some(Viewer(requestCountryCode = Some("us"), requestLanguageCode = Some("en"))) - ) - ), - context = LogEventContext(hostname = "", traceId = 31L) - ) - - val expectedUua1 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(91L)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = 1L, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(101L))), - ) - ), - actionType = ActionType.ServerTweetFav, - eventMetadata = EventMetadata( - sourceTimestampMs = 1001L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerTlsFavs, - traceId = Some(31L) - ) - ) - val expectedUua2 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(92L)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = 2L, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(102L))), - retweetingTweetId = Some(22L) - ) - ), - actionType = ActionType.ServerTweetFav, - eventMetadata = EventMetadata( - sourceTimestampMs = 1002L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerTlsFavs, - traceId = Some(32L) - ) - ) - val expectedUua3 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(93L)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = 3L, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(103L))), - ) - ), - actionType = ActionType.ServerTweetUnfav, - eventMetadata = EventMetadata( - sourceTimestampMs = 1003L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerTlsFavs, - traceId = Some(33L) - ) - ) - val expectedUua4 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(94L)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = 4L, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(104L))), - retweetingTweetId = Some(44L) - ) - ), - actionType = ActionType.ServerTweetUnfav, - eventMetadata = EventMetadata( - sourceTimestampMs = 1004L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerTlsFavs, - traceId = Some(34L) - ) - ) - val expectedUua5 = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(91L)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = 1L, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(101L))), - ) - ), - actionType = ActionType.ServerTweetFav, - eventMetadata = EventMetadata( - sourceTimestampMs = 1001L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerTlsFavs, - language = Some("EN"), - countryCode = Some("US"), - traceId = Some(31L) - ) - ) - } - - test("fav event with no retweet") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val actual = TlsFavsAdapter.adaptEvent(favEventNoRetweet) - assert(Seq(expectedUua1) === actual) - } - } - } - - test("fav event with a retweet") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val actual = TlsFavsAdapter.adaptEvent(favEventRetweet) - assert(Seq(expectedUua2) === actual) - } - } - } - - test("unfav event with no retweet") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val actual = TlsFavsAdapter.adaptEvent(unfavEventNoRetweet) - assert(Seq(expectedUua3) === actual) - } - } - } - - test("unfav event with a retweet") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val actual = TlsFavsAdapter.adaptEvent(unfavEventRetweet) - assert(Seq(expectedUua4) === actual) - } - } - } - - test("fav event with language and country") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val actual = TlsFavsAdapter.adaptEvent(favEventWithLangAndCountry) - assert(Seq(expectedUua5) === actual) - } - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TopicsIdUtilsSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TopicsIdUtilsSpec.docx new file mode 100644 index 000000000..ba86ad5e2 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TopicsIdUtilsSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TopicsIdUtilsSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TopicsIdUtilsSpec.scala deleted file mode 100644 index 3ad5a9ed5..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TopicsIdUtilsSpec.scala +++ /dev/null @@ -1,545 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.clientapp.thriftscala._ -import com.twitter.clientapp.thriftscala.SuggestionDetails -import com.twitter.guide.scribing.thriftscala._ -import com.twitter.guide.scribing.thriftscala.{SemanticCoreInterest => SemanticCoreInterestV1} -import com.twitter.guide.scribing.thriftscala.{SimClusterInterest => SimClusterInterestV1} -import com.twitter.guide.scribing.thriftscala.TopicModuleMetadata.SemanticCoreInterest -import com.twitter.guide.scribing.thriftscala.TopicModuleMetadata.SimClusterInterest -import com.twitter.guide.scribing.thriftscala.TransparentGuideDetails.TopicMetadata -import com.twitter.logbase.thriftscala.LogBase -import com.twitter.scrooge.TFieldBlob -import com.twitter.suggests.controller_data.home_hitl_topic_annotation_prompt.thriftscala.HomeHitlTopicAnnotationPromptControllerData -import com.twitter.suggests.controller_data.home_hitl_topic_annotation_prompt.v1.thriftscala.{ - HomeHitlTopicAnnotationPromptControllerData => HomeHitlTopicAnnotationPromptControllerDataV1 -} -import com.twitter.suggests.controller_data.home_topic_annotation_prompt.thriftscala.HomeTopicAnnotationPromptControllerData -import com.twitter.suggests.controller_data.home_topic_annotation_prompt.v1.thriftscala.{ - HomeTopicAnnotationPromptControllerData => HomeTopicAnnotationPromptControllerDataV1 -} -import com.twitter.suggests.controller_data.home_topic_follow_prompt.thriftscala.HomeTopicFollowPromptControllerData -import com.twitter.suggests.controller_data.home_topic_follow_prompt.v1.thriftscala.{ - HomeTopicFollowPromptControllerData => HomeTopicFollowPromptControllerDataV1 -} -import com.twitter.suggests.controller_data.home_tweets.thriftscala.HomeTweetsControllerData -import com.twitter.suggests.controller_data.home_tweets.v1.thriftscala.{ - HomeTweetsControllerData => HomeTweetsControllerDataV1 -} -import com.twitter.suggests.controller_data.search_response.item_types.thriftscala.ItemTypesControllerData -import com.twitter.suggests.controller_data.search_response.thriftscala.SearchResponseControllerData -import com.twitter.suggests.controller_data.search_response.topic_follow_prompt.thriftscala.SearchTopicFollowPromptControllerData -import com.twitter.suggests.controller_data.search_response.tweet_types.thriftscala.TweetTypesControllerData -import com.twitter.suggests.controller_data.search_response.v1.thriftscala.{ - SearchResponseControllerData => SearchResponseControllerDataV1 -} -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.suggests.controller_data.timelines_topic.thriftscala.TimelinesTopicControllerData -import com.twitter.suggests.controller_data.timelines_topic.v1.thriftscala.{ - TimelinesTopicControllerData => TimelinesTopicControllerDataV1 -} -import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} -import org.apache.thrift.protocol.TField -import org.junit.runner.RunWith -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers -import org.scalatestplus.junit.JUnitRunner -import com.twitter.util.mock.Mockito -import org.mockito.Mockito.when -import org.scalatest.prop.TableDrivenPropertyChecks - -@RunWith(classOf[JUnitRunner]) -class TopicsIdUtilsSpec - extends AnyFunSuite - with Matchers - with Mockito - with TableDrivenPropertyChecks { - import com.twitter.unified_user_actions.adapter.client_event.TopicIdUtils._ - - trait Fixture { - def buildLogBase(userId: Long): LogBase = { - val logBase = mock[LogBase] - when(logBase.country).thenReturn(Some("US")) - when(logBase.userId).thenReturn(Some(userId)) - when(logBase.timestamp).thenReturn(100L) - when(logBase.guestId).thenReturn(Some(1L)) - when(logBase.userAgent).thenReturn(None) - when(logBase.language).thenReturn(Some("en")) - logBase - } - - def buildItemForTimeline( - itemId: Long, - itemType: ItemType, - topicId: Long, - fn: Long => ControllerData.V2 - ): Item = { - val item = Item( - id = Some(itemId), - itemType = Some(itemType), - suggestionDetails = Some(SuggestionDetails(decodedControllerData = Some(fn(topicId)))) - ) - item - } - - def buildClientEventForHomeSearchTimeline( - itemId: Long, - itemType: ItemType, - topicId: Long, - fn: Long => ControllerData.V2, - userId: Long = 1L, - eventNamespaceOpt: Option[EventNamespace] = None, - ): LogEvent = { - val logEvent = mock[LogEvent] - when(logEvent.eventNamespace).thenReturn(eventNamespaceOpt) - val eventsDetails = mock[EventDetails] - when(eventsDetails.items) - .thenReturn(Some(Seq(buildItemForTimeline(itemId, itemType, topicId, fn)))) - val logbase = buildLogBase(userId) - when(logEvent.logBase).thenReturn(Some(logbase)) - when(logEvent.eventDetails).thenReturn(Some(eventsDetails)) - logEvent - } - - def buildClientEventForHomeTweetsTimeline( - itemId: Long, - itemType: ItemType, - topicId: Long, - topicIds: Set[Long], - fn: (Long, Set[Long]) => ControllerData.V2, - userId: Long = 1L, - eventNamespaceOpt: Option[EventNamespace] = None, - ): LogEvent = { - val logEvent = mock[LogEvent] - when(logEvent.eventNamespace).thenReturn(eventNamespaceOpt) - val eventsDetails = mock[EventDetails] - when(eventsDetails.items) - .thenReturn(Some(Seq(buildItemForHomeTimeline(itemId, itemType, topicId, topicIds, fn)))) - val logbase = buildLogBase(userId) - when(logEvent.logBase).thenReturn(Some(logbase)) - when(logEvent.eventDetails).thenReturn(Some(eventsDetails)) - logEvent - } - - def buildClientEventForGuide( - itemId: Long, - itemType: ItemType, - topicId: Long, - fn: Long => TopicMetadata, - userId: Long = 1L, - eventNamespaceOpt: Option[EventNamespace] = None, - ): LogEvent = { - val logEvent = mock[LogEvent] - when(logEvent.eventNamespace).thenReturn(eventNamespaceOpt) - val logbase = buildLogBase(userId) - when(logEvent.logBase).thenReturn(Some(logbase)) - val eventDetails = mock[EventDetails] - val item = buildItemForGuide(itemId, itemType, topicId, fn) - when(eventDetails.items).thenReturn(Some(Seq(item))) - when(logEvent.eventDetails).thenReturn(Some(eventDetails)) - logEvent - } - - def buildClientEventForOnboarding( - itemId: Long, - topicId: Long, - userId: Long = 1L - ): LogEvent = { - val logEvent = mock[LogEvent] - val logbase = buildLogBase(userId) - when(logEvent.logBase).thenReturn(Some(logbase)) - when(logEvent.eventNamespace).thenReturn(Some(buildNamespaceForOnboarding)) - val eventDetails = mock[EventDetails] - val item = buildItemForOnboarding(itemId, topicId) - when(eventDetails.items) - .thenReturn(Some(Seq(item))) - when(logEvent.eventDetails).thenReturn(Some(eventDetails)) - logEvent - } - - def buildClientEventForOnboardingBackend( - topicId: Long, - userId: Long = 1L - ): LogEvent = { - val logEvent = mock[LogEvent] - val logbase = buildLogBase(userId) - when(logEvent.logBase).thenReturn(Some(logbase)) - when(logEvent.eventNamespace).thenReturn(Some(buildNamespaceForOnboardingBackend)) - val eventDetails = buildEventDetailsForOnboardingBackend(topicId) - when(logEvent.eventDetails).thenReturn(Some(eventDetails)) - logEvent - } - - def defaultNamespace: EventNamespace = { - EventNamespace(Some("iphone"), None, None, None, None, Some("favorite")) - } - - def buildNamespaceForOnboardingBackend: EventNamespace = { - EventNamespace( - Some("iphone"), - Some("onboarding_backend"), - Some("subtasks"), - Some("topics_selector"), - Some("removed"), - Some("selected")) - } - - def buildNamespaceForOnboarding: EventNamespace = { - EventNamespace( - Some("iphone"), - Some("onboarding"), - Some("topics_selector"), - None, - Some("topic"), - Some("follow") - ) - } - - def buildItemForHomeTimeline( - itemId: Long, - itemType: ItemType, - topicId: Long, - topicIds: Set[Long], - fn: (Long, Set[Long]) => ControllerData.V2 - ): Item = { - val item = Item( - id = Some(itemId), - itemType = Some(itemType), - suggestionDetails = - Some(SuggestionDetails(decodedControllerData = Some(fn(topicId, topicIds)))) - ) - item - } - - def buildItemForGuide( - itemId: Long, - itemType: ItemType, - topicId: Long, - fn: Long => TopicMetadata - ): Item = { - val item = mock[Item] - when(item.id).thenReturn(Some(itemId)) - when(item.itemType).thenReturn(Some(itemType)) - when(item.suggestionDetails) - .thenReturn(Some(SuggestionDetails(suggestionType = Some("ErgTweet")))) - val guideItemDetails = mock[GuideItemDetails] - when(guideItemDetails.transparentGuideDetails).thenReturn(Some(fn(topicId))) - when(item.guideItemDetails).thenReturn(Some(guideItemDetails)) - item - } - - def buildItemForOnboarding( - itemId: Long, - topicId: Long - ): Item = { - val item = Item( - id = Some(itemId), - itemType = None, - description = Some(s"id=$topicId,row=1") - ) - item - } - - def buildEventDetailsForOnboardingBackend( - topicId: Long - ): EventDetails = { - val eventDetails = mock[EventDetails] - val item = Item( - id = Some(topicId) - ) - val itemTmp = buildItemForOnboarding(10, topicId) - when(eventDetails.items).thenReturn(Some(Seq(itemTmp))) - when(eventDetails.targets).thenReturn(Some(Seq(item))) - eventDetails - } - - def topicMetadataInGuide(topicId: Long): TopicMetadata = - TopicMetadata( - SemanticCoreInterest( - SemanticCoreInterestV1(domainId = "131", entityId = topicId.toString) - ) - ) - - def simClusterMetadataInGuide(simclusterId: Long = 1L): TopicMetadata = - TopicMetadata( - SimClusterInterest( - SimClusterInterestV1(simclusterId.toString) - ) - ) - - def timelineTopicControllerData(topicId: Long): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.TimelinesTopic( - TimelinesTopicControllerData.V1( - TimelinesTopicControllerDataV1( - topicId = topicId, - topicTypesBitmap = 1 - ) - ))) - - def homeTweetControllerData(topicId: Long): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.HomeTweets( - HomeTweetsControllerData.V1( - HomeTweetsControllerDataV1( - topicId = Some(topicId) - )))) - - def homeTopicFollowPromptControllerData(topicId: Long): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.HomeTopicFollowPrompt(HomeTopicFollowPromptControllerData.V1( - HomeTopicFollowPromptControllerDataV1(Some(topicId))))) - - def homeTopicAnnotationPromptControllerData(topicId: Long): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.HomeTopicAnnotationPrompt(HomeTopicAnnotationPromptControllerData.V1( - HomeTopicAnnotationPromptControllerDataV1(tweetId = 1L, topicId = topicId)))) - - def homeHitlTopicAnnotationPromptControllerData(topicId: Long): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.HomeHitlTopicAnnotationPrompt( - HomeHitlTopicAnnotationPromptControllerData.V1( - HomeHitlTopicAnnotationPromptControllerDataV1(tweetId = 2L, topicId = topicId)))) - - def searchTopicFollowPromptControllerData(topicId: Long): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.SearchResponse( - SearchResponseControllerData.V1( - SearchResponseControllerDataV1( - Some(ItemTypesControllerData.TopicFollowControllerData( - SearchTopicFollowPromptControllerData(Some(topicId)) - )), - None - )))) - - def searchTweetTypesControllerData(topicId: Long): ControllerData.V2 = - ControllerData.V2( - ControllerDataV2.SearchResponse( - SearchResponseControllerData.V1( - SearchResponseControllerDataV1( - Some(ItemTypesControllerData.TweetTypesControllerData( - TweetTypesControllerData(None, Some(topicId)) - )), - None - ) - ))) - - //used for creating logged out user client events - def buildLogBaseWithoutUserId(guestId: Long): LogBase = - LogBase( - ipAddress = "120.10.10.20", - guestId = Some(guestId), - userAgent = None, - transactionId = "", - country = Some("US"), - timestamp = 100L, - language = Some("en") - ) - } - - test("getTopicId should correctly find topic id from item for home timeline and search") { - new Fixture { - - val testData = Table( - ("ItemType", "topicId", "controllerData"), - (ItemType.Tweet, 1L, timelineTopicControllerData(1L)), - (ItemType.User, 2L, timelineTopicControllerData(2L)), - (ItemType.Topic, 3L, homeTweetControllerData(3L)), - (ItemType.Topic, 4L, homeTopicFollowPromptControllerData(4L)), - (ItemType.Topic, 5L, searchTopicFollowPromptControllerData(5L)), - (ItemType.Topic, 6L, homeHitlTopicAnnotationPromptControllerData(6L)) - ) - - forEvery(testData) { - (itemType: ItemType, topicId: Long, controllerDataV2: ControllerData.V2) => - getTopicId( - buildItemForTimeline(1, itemType, topicId, _ => controllerDataV2), - defaultNamespace) shouldEqual Some(topicId) - } - } - } - - test("getTopicId should correctly find topic id from item for guide events") { - new Fixture { - getTopicId( - buildItemForGuide(1, ItemType.Tweet, 100, topicMetadataInGuide), - defaultNamespace - ) shouldEqual Some(100) - } - } - - test("getTopicId should correctly find topic id for onboarding events") { - new Fixture { - getTopicId( - buildItemForOnboarding(1, 100), - buildNamespaceForOnboarding - ) shouldEqual Some(100) - } - } - - test("should return TopicId From HomeSearch") { - val testData = Table( - ("controllerData", "topicId"), - ( - ControllerData.V2( - ControllerDataV2.HomeTweets( - HomeTweetsControllerData.V1(HomeTweetsControllerDataV1(topicId = Some(1L)))) - ), - Some(1L)), - ( - ControllerData.V2( - ControllerDataV2.HomeTopicFollowPrompt(HomeTopicFollowPromptControllerData - .V1(HomeTopicFollowPromptControllerDataV1(topicId = Some(2L))))), - Some(2L)), - ( - ControllerData.V2( - ControllerDataV2.TimelinesTopic( - TimelinesTopicControllerData.V1( - TimelinesTopicControllerDataV1(topicId = 3L, topicTypesBitmap = 100) - ))), - Some(3L)), - ( - ControllerData.V2( - ControllerDataV2.SearchResponse( - SearchResponseControllerData.V1(SearchResponseControllerDataV1(itemTypesControllerData = - Some(ItemTypesControllerData.TopicFollowControllerData( - SearchTopicFollowPromptControllerData(topicId = Some(4L)))))))), - Some(4L)), - ( - ControllerData.V2( - ControllerDataV2.SearchResponse( - SearchResponseControllerData.V1( - SearchResponseControllerDataV1(itemTypesControllerData = Some(ItemTypesControllerData - .TweetTypesControllerData(TweetTypesControllerData(topicId = Some(5L)))))))), - Some(5L)), - ( - ControllerData.V2( - ControllerDataV2 - .SearchResponse(SearchResponseControllerData.V1(SearchResponseControllerDataV1()))), - None) - ) - - forEvery(testData) { (controllerDataV2: ControllerData.V2, topicId: Option[Long]) => - getTopicIdFromHomeSearch( - Item(suggestionDetails = Some( - SuggestionDetails(decodedControllerData = Some(controllerDataV2))))) shouldEqual topicId - } - } - - test("test TopicId From Onboarding") { - val testData = Table( - ("Item", "EventNamespace", "topicId"), - ( - Item(description = Some("id=11,key=value")), - EventNamespace( - page = Some("onboarding"), - section = Some("section has topic"), - component = Some("component has topic"), - element = Some("element has topic") - ), - Some(11L)), - ( - Item(description = Some("id=22,key=value")), - EventNamespace( - page = Some("onboarding"), - section = Some("section has topic") - ), - Some(22L)), - ( - Item(description = Some("id=33,key=value")), - EventNamespace( - page = Some("onboarding"), - component = Some("component has topic") - ), - Some(33L)), - ( - Item(description = Some("id=44,key=value")), - EventNamespace( - page = Some("onboarding"), - element = Some("element has topic") - ), - Some(44L)), - ( - Item(description = Some("id=678,key=value")), - EventNamespace( - page = Some("onXYZboarding"), - section = Some("section has topic"), - component = Some("component has topic"), - element = Some("element has topic") - ), - None), - ( - Item(description = Some("id=678,key=value")), - EventNamespace( - page = Some("page has onboarding"), - section = Some("section has topPic"), - component = Some("component has topPic"), - element = Some("element has topPic") - ), - None), - ( - Item(description = Some("key=value,id=678")), - EventNamespace( - page = Some("page has onboarding"), - section = Some("section has topic"), - component = Some("component has topic"), - element = Some("element has topic") - ), - None) - ) - - forEvery(testData) { (item: Item, eventNamespace: EventNamespace, topicId: Option[Long]) => - getTopicFromOnboarding(item, eventNamespace) shouldEqual topicId - } - } - - test("test from Guide") { - val testData = Table( - ("guideItemDetails", "topicId"), - ( - GuideItemDetails(transparentGuideDetails = Some( - TransparentGuideDetails.TopicMetadata( - TopicModuleMetadata.TttInterest(tttInterest = TttInterest.unsafeEmpty)))), - None), - ( - GuideItemDetails(transparentGuideDetails = Some( - TransparentGuideDetails.TopicMetadata( - TopicModuleMetadata.SimClusterInterest(simClusterInterest = - com.twitter.guide.scribing.thriftscala.SimClusterInterest.unsafeEmpty)))), - None), - ( - GuideItemDetails(transparentGuideDetails = Some( - TransparentGuideDetails.TopicMetadata(TopicModuleMetadata.UnknownUnionField(field = - TFieldBlob(new TField(), Array.empty[Byte]))))), - None), - ( - GuideItemDetails(transparentGuideDetails = Some( - TransparentGuideDetails.TopicMetadata( - TopicModuleMetadata.SemanticCoreInterest( - com.twitter.guide.scribing.thriftscala.SemanticCoreInterest.unsafeEmpty - .copy(domainId = "131", entityId = "1"))))), - Some(1L)), - ) - - forEvery(testData) { (guideItemDetails: GuideItemDetails, topicId: Option[Long]) => - getTopicFromGuide(Item(guideItemDetails = Some(guideItemDetails))) shouldEqual topicId - } - } - - test("getTopicId should return topicIds") { - getTopicId( - item = Item(suggestionDetails = Some( - SuggestionDetails(decodedControllerData = Some( - ControllerData.V2( - ControllerDataV2.HomeTweets( - HomeTweetsControllerData.V1(HomeTweetsControllerDataV1(topicId = Some(1L)))) - ))))), - namespace = EventNamespace( - page = Some("onboarding"), - section = Some("section has topic"), - component = Some("component has topic"), - element = Some("element has topic") - ) - ) shouldEqual Some(1L) - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TweetypieEventAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TweetypieEventAdapterSpec.docx new file mode 100644 index 000000000..0050a0666 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TweetypieEventAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TweetypieEventAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TweetypieEventAdapterSpec.scala deleted file mode 100644 index c23b5db54..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/TweetypieEventAdapterSpec.scala +++ /dev/null @@ -1,852 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.inject.Test -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.tweetypie.thriftscala.AdditionalFieldDeleteEvent -import com.twitter.tweetypie.thriftscala.AdditionalFieldUpdateEvent -import com.twitter.tweetypie.thriftscala.AuditDeleteTweet -import com.twitter.tweetypie.thriftscala.DeviceSource -import com.twitter.tweetypie.thriftscala.EditControl -import com.twitter.tweetypie.thriftscala.EditControlEdit -import com.twitter.tweetypie.thriftscala.Language -import com.twitter.tweetypie.thriftscala.Place -import com.twitter.tweetypie.thriftscala.PlaceType -import com.twitter.tweetypie.thriftscala.QuotedTweet -import com.twitter.tweetypie.thriftscala.QuotedTweetDeleteEvent -import com.twitter.tweetypie.thriftscala.QuotedTweetTakedownEvent -import com.twitter.tweetypie.thriftscala.Reply -import com.twitter.tweetypie.thriftscala.Share -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.tweetypie.thriftscala.TweetCoreData -import com.twitter.tweetypie.thriftscala.TweetCreateEvent -import com.twitter.tweetypie.thriftscala.TweetDeleteEvent -import com.twitter.tweetypie.thriftscala.TweetEvent -import com.twitter.tweetypie.thriftscala.TweetEventData -import com.twitter.tweetypie.thriftscala.TweetEventFlags -import com.twitter.tweetypie.thriftscala.TweetPossiblySensitiveUpdateEvent -import com.twitter.tweetypie.thriftscala.TweetScrubGeoEvent -import com.twitter.tweetypie.thriftscala.TweetTakedownEvent -import com.twitter.tweetypie.thriftscala.TweetUndeleteEvent -import com.twitter.tweetypie.thriftscala.UserScrubGeoEvent -import com.twitter.unified_user_actions.adapter.tweetypie_event.TweetypieEventAdapter -import com.twitter.unified_user_actions.thriftscala._ -import com.twitter.util.Time -import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatest.prop.TableFor1 -import org.scalatest.prop.TableFor2 -import org.scalatest.prop.TableFor3 - -class TweetypieEventAdapterSpec extends Test with TableDrivenPropertyChecks { - trait Fixture { - val frozenTime: Time = Time.fromMilliseconds(1658949273000L) - - val tweetDeleteEventTime: Time = Time.fromMilliseconds(1658949253000L) - - val tweetId = 1554576940856246272L - val timestamp: Long = SnowflakeId.unixTimeMillisFromId(tweetId) - val userId = 1L - val user: User = User( - id = userId, - createdAtMsec = 1000L, - updatedAtMsec = 1000L, - userType = UserType.Normal, - ) - - val actionedTweetId = 1554576940756246333L - val actionedTweetTimestamp: Long = SnowflakeId.unixTimeMillisFromId(actionedTweetId) - val actionedTweetAuthorId = 2L - - val actionedByActionedTweetId = 1554566940756246272L - val actionedByActionedTweetTimestamp: Long = - SnowflakeId.unixTimeMillisFromId(actionedByActionedTweetId) - val actionedByActionedTweetAuthorId = 3L - - val tweetEventFlags: TweetEventFlags = TweetEventFlags(timestampMs = timestamp) - val language: Option[Language] = Some(Language("EN-US", false)) - val deviceSource: Option[DeviceSource] = Some( - DeviceSource( - id = 0, - parameter = "", - internalName = "", - name = "name", - url = "url", - display = "display", - clientAppId = Option(100L))) - val place: Option[Place] = Some( - Place( - id = "id", - `type` = PlaceType.City, - fullName = "San Francisco", - name = "SF", - countryCode = Some("US"), - )) - - // for TweetDeleteEvent - val auditDeleteTweet = Some( - AuditDeleteTweet( - clientApplicationId = Option(200L) - )) - - val tweetCoreData: TweetCoreData = - TweetCoreData(userId, text = "text", createdVia = "created_via", createdAtSecs = timestamp) - val baseTweet: Tweet = Tweet( - tweetId, - coreData = Some(tweetCoreData), - language = language, - deviceSource = deviceSource, - place = place) - - def getCreateTweetCoreData(userId: Long, timestamp: Long): TweetCoreData = - tweetCoreData.copy(userId = userId, createdAtSecs = timestamp) - def getRetweetTweetCoreData( - userId: Long, - retweetedTweetId: Long, - retweetedAuthorId: Long, - parentStatusId: Long, - timestamp: Long - ): TweetCoreData = tweetCoreData.copy( - userId = userId, - share = Some( - Share( - sourceStatusId = retweetedTweetId, - sourceUserId = retweetedAuthorId, - parentStatusId = parentStatusId - )), - createdAtSecs = timestamp - ) - def getReplyTweetCoreData( - userId: Long, - repliedTweetId: Long, - repliedAuthorId: Long, - timestamp: Long - ): TweetCoreData = tweetCoreData.copy( - userId = userId, - reply = Some( - Reply( - inReplyToStatusId = Some(repliedTweetId), - inReplyToUserId = repliedAuthorId, - ) - ), - createdAtSecs = timestamp) - def getQuoteTweetCoreData(userId: Long, timestamp: Long): TweetCoreData = - tweetCoreData.copy(userId = userId, createdAtSecs = timestamp) - - def getTweet(tweetId: Long, userId: Long, timestamp: Long): Tweet = - baseTweet.copy(id = tweetId, coreData = Some(getCreateTweetCoreData(userId, timestamp))) - - def getRetweet( - tweetId: Long, - userId: Long, - timestamp: Long, - retweetedTweetId: Long, - retweetedUserId: Long, - parentStatusId: Option[Long] = None - ): Tweet = - baseTweet.copy( - id = tweetId, - coreData = Some( - getRetweetTweetCoreData( - userId, - retweetedTweetId, - retweetedUserId, - parentStatusId.getOrElse(retweetedTweetId), - timestamp))) - - def getQuote( - tweetId: Long, - userId: Long, - timestamp: Long, - quotedTweetId: Long, - quotedUserId: Long - ): Tweet = - baseTweet.copy( - id = tweetId, - coreData = Some(getQuoteTweetCoreData(userId, timestamp)), - quotedTweet = Some(QuotedTweet(quotedTweetId, quotedUserId))) - - def getReply( - tweetId: Long, - userId: Long, - repliedTweetId: Long, - repliedAuthorId: Long, - timestamp: Long - ): Tweet = - baseTweet.copy( - id = tweetId, - coreData = Some(getReplyTweetCoreData(userId, repliedTweetId, repliedAuthorId, timestamp)), - ) - - // ignored tweet events - val additionalFieldUpdateEvent: TweetEvent = TweetEvent( - TweetEventData.AdditionalFieldUpdateEvent(AdditionalFieldUpdateEvent(baseTweet)), - tweetEventFlags) - val additionalFieldDeleteEvent: TweetEvent = TweetEvent( - TweetEventData.AdditionalFieldDeleteEvent( - AdditionalFieldDeleteEvent(Map(tweetId -> Seq.empty)) - ), - tweetEventFlags - ) - val tweetUndeleteEvent: TweetEvent = TweetEvent( - TweetEventData.TweetUndeleteEvent(TweetUndeleteEvent(baseTweet)), - tweetEventFlags - ) - val tweetScrubGeoEvent: TweetEvent = TweetEvent( - TweetEventData.TweetScrubGeoEvent(TweetScrubGeoEvent(tweetId, userId)), - tweetEventFlags) - val tweetTakedownEvent: TweetEvent = TweetEvent( - TweetEventData.TweetTakedownEvent(TweetTakedownEvent(tweetId, userId)), - tweetEventFlags - ) - val userScrubGeoEvent: TweetEvent = TweetEvent( - TweetEventData.UserScrubGeoEvent(UserScrubGeoEvent(userId = userId, maxTweetId = tweetId)), - tweetEventFlags - ) - val tweetPossiblySensitiveUpdateEvent: TweetEvent = TweetEvent( - TweetEventData.TweetPossiblySensitiveUpdateEvent( - TweetPossiblySensitiveUpdateEvent( - tweetId = tweetId, - userId = userId, - nsfwAdmin = false, - nsfwUser = false)), - tweetEventFlags - ) - val quotedTweetDeleteEvent: TweetEvent = TweetEvent( - TweetEventData.QuotedTweetDeleteEvent( - QuotedTweetDeleteEvent( - quotingTweetId = tweetId, - quotingUserId = userId, - quotedTweetId = tweetId, - quotedUserId = userId)), - tweetEventFlags - ) - val quotedTweetTakedownEvent: TweetEvent = TweetEvent( - TweetEventData.QuotedTweetTakedownEvent( - QuotedTweetTakedownEvent( - quotingTweetId = tweetId, - quotingUserId = userId, - quotedTweetId = tweetId, - quotedUserId = userId, - takedownCountryCodes = Seq.empty, - takedownReasons = Seq.empty - ) - ), - tweetEventFlags - ) - val replyOnlyTweet = - getReply(tweetId, userId, actionedTweetId, actionedTweetAuthorId, timestamp) - val replyAndRetweetTweet = replyOnlyTweet.copy(coreData = replyOnlyTweet.coreData.map( - _.copy(share = Some( - Share( - sourceStatusId = actionedTweetId, - sourceUserId = actionedTweetAuthorId, - parentStatusId = actionedTweetId - ))))) - val replyRetweetPresentEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = replyAndRetweetTweet, - user = user, - sourceTweet = - Some(getTweet(actionedTweetId, actionedTweetAuthorId, actionedTweetTimestamp)) - )), - tweetEventFlags - ) - - def getExpectedUUA( - userId: Long, - actionTweetId: Long, - actionTweetAuthorId: Long, - sourceTimestampMs: Long, - actionType: ActionType, - replyingTweetId: Option[Long] = None, - quotingTweetId: Option[Long] = None, - retweetingTweetId: Option[Long] = None, - inReplyToTweetId: Option[Long] = None, - quotedTweetId: Option[Long] = None, - retweetedTweetId: Option[Long] = None, - editedTweetId: Option[Long] = None, - appId: Option[Long] = None, - ): UnifiedUserAction = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(userId)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = actionTweetId, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(actionTweetAuthorId))), - replyingTweetId = replyingTweetId, - quotingTweetId = quotingTweetId, - retweetingTweetId = retweetingTweetId, - inReplyToTweetId = inReplyToTweetId, - quotedTweetId = quotedTweetId, - retweetedTweetId = retweetedTweetId, - editedTweetId = editedTweetId - ) - ), - actionType = actionType, - eventMetadata = EventMetadata( - sourceTimestampMs = sourceTimestampMs, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerTweetypieEvents, - language = None, - countryCode = Some("US"), - clientAppId = appId, - ) - ) - - /* Note: This is a deprecated field {ActionTweetType}. - * We keep this here to document the behaviors of each unit test. - /* - * Types of tweets on which actions can take place. - * Note that retweets are not included because actions can NOT take place - * on retweets. They can only take place on source tweets of retweets, - * which are one of the ActionTweetTypes listed below. - */ - enum ActionTweetType { - /* Is a standard (non-retweet, non-reply, non-quote) tweet */ - Default = 0 - - /* - * Is a tweet in a reply chain (this includes tweets - * without a leading @mention, as long as they are in reply - * to some tweet id) - */ - Reply = 1 - - /* Is a retweet with comment */ - Quote = 2 - }(persisted='true', hasPersonalData='false') - */ - - // tweet create - val tweetCreateEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getTweet(tweetId, userId, timestamp), - user = user, - ) - ), - tweetEventFlags) - val expectedUUACreate = getExpectedUUA( - userId = userId, - actionTweetId = tweetId, - /* @see comment above for ActionTweetType - actionTweetType = Some(ActionTweetType.Default), - */ - actionTweetAuthorId = userId, - sourceTimestampMs = timestamp, - actionType = ActionType.ServerTweetCreate, - appId = deviceSource.flatMap(_.clientAppId) - ) - - // tweet reply to a default - val tweetReplyDefaultEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getReply(tweetId, userId, actionedTweetId, actionedTweetAuthorId, timestamp), - user = user - ) - ), - tweetEventFlags - ) - val expectedUUAReplyDefault = getExpectedUUA( - userId = userId, - actionTweetId = actionedTweetId, - /* @see comment above for ActionTweetType - actionTweetType = None, - */ - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = timestamp, - actionType = ActionType.ServerTweetReply, - replyingTweetId = Some(tweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - // tweet reply to a reply - val tweetReplyToReplyEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getReply(tweetId, userId, actionedTweetId, actionedTweetAuthorId, timestamp), - user = user - ) - ), - tweetEventFlags - ) - // tweet reply to a quote - val tweetReplyToQuoteEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getReply(tweetId, userId, actionedTweetId, actionedTweetAuthorId, timestamp), - user = user - ) - ), - tweetEventFlags - ) - // tweet quote a default - val tweetQuoteDefaultEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getQuote(tweetId, userId, timestamp, actionedTweetId, actionedTweetAuthorId), - user = user, - quotedTweet = - Some(getTweet(actionedTweetId, actionedTweetAuthorId, actionedTweetTimestamp)) - ) - ), - tweetEventFlags - ) - val expectedUUAQuoteDefault: UnifiedUserAction = getExpectedUUA( - userId = userId, - actionTweetId = actionedTweetId, - /* @see comment above for ActionTweetType - actionTweetType = Some(ActionTweetType.Default), - */ - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = timestamp, - actionType = ActionType.ServerTweetQuote, - quotingTweetId = Some(tweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - // tweet quote a reply - val tweetQuoteReplyEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getQuote(tweetId, userId, timestamp, actionedTweetId, actionedTweetAuthorId), - user = user, - quotedTweet = Some( - getReply( - tweetId = actionedTweetId, - userId = actionedTweetAuthorId, - repliedTweetId = actionedByActionedTweetId, - repliedAuthorId = actionedByActionedTweetAuthorId, - timestamp = actionedTweetTimestamp - )) - ) - ), - tweetEventFlags - ) - val expectedUUAQuoteReply: UnifiedUserAction = getExpectedUUA( - userId = userId, - actionTweetId = actionedTweetId, - /* @see comment above for ActionTweetType - actionTweetType = Some(ActionTweetType.Reply), - */ - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = timestamp, - actionType = ActionType.ServerTweetQuote, - quotingTweetId = Some(tweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - // tweet quote a quote - val tweetQuoteQuoteEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getQuote(tweetId, userId, timestamp, actionedTweetId, actionedTweetAuthorId), - user = user, - quotedTweet = Some( - getQuote( - tweetId = actionedTweetId, - userId = actionedTweetAuthorId, - timestamp = actionedTweetTimestamp, - quotedTweetId = actionedByActionedTweetId, - quotedUserId = actionedByActionedTweetAuthorId, - )) - ) - ), - tweetEventFlags - ) - val expectedUUAQuoteQuote: UnifiedUserAction = getExpectedUUA( - userId = userId, - actionTweetId = actionedTweetId, - /* @see comment above for ActionTweetType - actionTweetType = Some(ActionTweetType.Quote), - */ - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = timestamp, - actionType = ActionType.ServerTweetQuote, - quotingTweetId = Some(tweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - // tweet retweet a default - val tweetRetweetDefaultEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getRetweet(tweetId, userId, timestamp, actionedTweetId, actionedTweetAuthorId), - user = user, - sourceTweet = - Some(getTweet(actionedTweetId, actionedTweetAuthorId, actionedTweetTimestamp)) - ) - ), - tweetEventFlags - ) - val expectedUUARetweetDefault: UnifiedUserAction = getExpectedUUA( - userId = userId, - actionTweetId = actionedTweetId, - /* @see comment above for ActionTweetType - actionTweetType = Some(ActionTweetType.Default), - */ - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = timestamp, - actionType = ActionType.ServerTweetRetweet, - retweetingTweetId = Some(tweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - // tweet retweet a reply - val tweetRetweetReplyEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getRetweet(tweetId, userId, timestamp, actionedTweetId, actionedTweetAuthorId), - user = user, - sourceTweet = Some( - getReply( - actionedTweetId, - actionedTweetAuthorId, - actionedByActionedTweetId, - actionedByActionedTweetAuthorId, - actionedTweetTimestamp)) - ) - ), - tweetEventFlags - ) - val expectedUUARetweetReply: UnifiedUserAction = getExpectedUUA( - userId = userId, - actionTweetId = actionedTweetId, - /* @see comment above for ActionTweetType - actionTweetType = Some(ActionTweetType.Reply), - */ - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = timestamp, - actionType = ActionType.ServerTweetRetweet, - retweetingTweetId = Some(tweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - // tweet retweet a quote - val tweetRetweetQuoteEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getRetweet(tweetId, userId, timestamp, actionedTweetId, actionedTweetAuthorId), - user = user, - sourceTweet = Some( - getQuote( - actionedTweetId, - actionedTweetAuthorId, - actionedTweetTimestamp, - actionedByActionedTweetId, - actionedByActionedTweetAuthorId - )) - ) - ), - tweetEventFlags - ) - val expectedUUARetweetQuote: UnifiedUserAction = getExpectedUUA( - userId = userId, - actionTweetId = actionedTweetId, - /* @see comment above for ActionTweetType - actionTweetType = Some(ActionTweetType.Quote), - */ - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = timestamp, - actionType = ActionType.ServerTweetRetweet, - retweetingTweetId = Some(tweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - // tweet retweet a retweet - val tweetRetweetRetweetEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getRetweet( - tweetId, - userId, - timestamp, - actionedByActionedTweetId, - actionedByActionedTweetAuthorId, - Some(actionedTweetId)), - user = user, - sourceTweet = Some( - getTweet( - actionedByActionedTweetId, - actionedByActionedTweetAuthorId, - actionedByActionedTweetTimestamp, - )) - ) - ), - tweetEventFlags - ) - val expectedUUARetweetRetweet: UnifiedUserAction = getExpectedUUA( - userId = userId, - actionTweetId = actionedByActionedTweetId, - /* @see comment above for ActionTweetType - actionTweetType = Some(ActionTweetType.Default), - */ - actionTweetAuthorId = actionedByActionedTweetAuthorId, - sourceTimestampMs = timestamp, - actionType = ActionType.ServerTweetRetweet, - retweetingTweetId = Some(tweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - // delete a tweet - val tweetDeleteEvent: TweetEvent = TweetEvent( - TweetEventData.TweetDeleteEvent( - TweetDeleteEvent( - tweet = getTweet(tweetId, userId, timestamp), - user = Some(user), - audit = auditDeleteTweet - ) - ), - tweetEventFlags.copy(timestampMs = tweetDeleteEventTime.inMilliseconds) - ) - val expectedUUADeleteDefault: UnifiedUserAction = getExpectedUUA( - userId = user.id, - actionTweetId = tweetId, - actionTweetAuthorId = userId, - sourceTimestampMs = tweetDeleteEventTime.inMilliseconds, - actionType = ActionType.ServerTweetDelete, - appId = auditDeleteTweet.flatMap(_.clientApplicationId) - ) - // delete a reply - Unreply - val tweetUnreplyEvent: TweetEvent = TweetEvent( - TweetEventData.TweetDeleteEvent( - TweetDeleteEvent( - tweet = getReply(tweetId, userId, actionedTweetId, actionedTweetAuthorId, timestamp), - user = Some(user), - audit = auditDeleteTweet - ) - ), - tweetEventFlags.copy(timestampMs = tweetDeleteEventTime.inMilliseconds) - ) - val expectedUUAUnreply: UnifiedUserAction = getExpectedUUA( - userId = user.id, - actionTweetId = actionedTweetId, - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = tweetDeleteEventTime.inMilliseconds, - actionType = ActionType.ServerTweetUnreply, - replyingTweetId = Some(tweetId), - appId = auditDeleteTweet.flatMap(_.clientApplicationId) - ) - // delete a quote - Unquote - val tweetUnquoteEvent: TweetEvent = TweetEvent( - TweetEventData.TweetDeleteEvent( - TweetDeleteEvent( - tweet = getQuote(tweetId, userId, timestamp, actionedTweetId, actionedTweetAuthorId), - user = Some(user), - audit = auditDeleteTweet - ) - ), - tweetEventFlags.copy(timestampMs = tweetDeleteEventTime.inMilliseconds) - ) - val expectedUUAUnquote: UnifiedUserAction = getExpectedUUA( - userId = user.id, - actionTweetId = actionedTweetId, - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = tweetDeleteEventTime.inMilliseconds, - actionType = ActionType.ServerTweetUnquote, - quotingTweetId = Some(tweetId), - appId = auditDeleteTweet.flatMap(_.clientApplicationId) - ) - // delete a retweet / unretweet - val tweetUnretweetEvent: TweetEvent = TweetEvent( - TweetEventData.TweetDeleteEvent( - TweetDeleteEvent( - tweet = getRetweet( - tweetId, - userId, - timestamp, - actionedTweetId, - actionedTweetAuthorId, - Some(actionedTweetId)), - user = Some(user), - audit = auditDeleteTweet - ) - ), - tweetEventFlags.copy(timestampMs = tweetDeleteEventTime.inMilliseconds) - ) - val expectedUUAUnretweet: UnifiedUserAction = getExpectedUUA( - userId = user.id, - actionTweetId = actionedTweetId, - actionTweetAuthorId = actionedTweetAuthorId, - sourceTimestampMs = tweetDeleteEventTime.inMilliseconds, - actionType = ActionType.ServerTweetUnretweet, - retweetingTweetId = Some(tweetId), - appId = auditDeleteTweet.flatMap(_.clientApplicationId) - ) - // edit a tweet, the new tweet from edit is a default tweet (not reply/quote/retweet) - val regularTweetFromEditEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getTweet( - tweetId, - userId, - timestamp - ).copy(editControl = - Some(EditControl.Edit(EditControlEdit(initialTweetId = actionedTweetId)))), - user = user, - ) - ), - tweetEventFlags - ) - val expectedUUARegularTweetFromEdit: UnifiedUserAction = getExpectedUUA( - userId = user.id, - actionTweetId = tweetId, - actionTweetAuthorId = userId, - sourceTimestampMs = tweetEventFlags.timestampMs, - actionType = ActionType.ServerTweetEdit, - editedTweetId = Some(actionedTweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - // edit a tweet, the new tweet from edit is a Quote - val quoteFromEditEvent: TweetEvent = TweetEvent( - TweetEventData.TweetCreateEvent( - TweetCreateEvent( - tweet = getQuote( - tweetId, - userId, - timestamp, - actionedTweetId, - actionedTweetAuthorId - ).copy(editControl = - Some(EditControl.Edit(EditControlEdit(initialTweetId = actionedByActionedTweetId)))), - user = user, - ) - ), - tweetEventFlags - ) - val expectedUUAQuoteFromEdit: UnifiedUserAction = getExpectedUUA( - userId = user.id, - actionTweetId = tweetId, - actionTweetAuthorId = userId, - sourceTimestampMs = tweetEventFlags.timestampMs, - actionType = ActionType.ServerTweetEdit, - editedTweetId = Some(actionedByActionedTweetId), - quotedTweetId = Some(actionedTweetId), - appId = deviceSource.flatMap(_.clientAppId) - ) - } - - test("ignore non-TweetCreate / non-TweetDelete events") { - new Fixture { - val ignoredTweetEvents: TableFor1[TweetEvent] = Table( - "ignoredTweetEvents", - additionalFieldUpdateEvent, - additionalFieldDeleteEvent, - tweetUndeleteEvent, - tweetScrubGeoEvent, - tweetTakedownEvent, - userScrubGeoEvent, - tweetPossiblySensitiveUpdateEvent, - quotedTweetDeleteEvent, - quotedTweetTakedownEvent - ) - forEvery(ignoredTweetEvents) { tweetEvent: TweetEvent => - val actual = TweetypieEventAdapter.adaptEvent(tweetEvent) - assert(actual.isEmpty) - } - } - } - - test("ignore invalid TweetCreate events") { - new Fixture { - val ignoredTweetEvents: TableFor2[String, TweetEvent] = Table( - ("invalidType", "event"), - ("replyAndRetweetBothPresent", replyRetweetPresentEvent) - ) - forEvery(ignoredTweetEvents) { (_, event) => - val actual = TweetypieEventAdapter.adaptEvent(event) - assert(actual.isEmpty) - } - } - } - - test("TweetypieCreateEvent") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val actual = TweetypieEventAdapter.adaptEvent(tweetCreateEvent) - assert(Seq(expectedUUACreate) == actual) - } - } - } - - test("TweetypieReplyEvent") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val tweetReplies: TableFor3[String, TweetEvent, UnifiedUserAction] = Table( - ("actionTweetType", "event", "expected"), - ("Default", tweetReplyDefaultEvent, expectedUUAReplyDefault), - ("Reply", tweetReplyToReplyEvent, expectedUUAReplyDefault), - ("Quote", tweetReplyToQuoteEvent, expectedUUAReplyDefault), - ) - forEvery(tweetReplies) { (_: String, event: TweetEvent, expected: UnifiedUserAction) => - val actual = TweetypieEventAdapter.adaptEvent(event) - assert(Seq(expected) === actual) - } - } - } - } - - test("TweetypieQuoteEvent") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val tweetQuotes: TableFor3[String, TweetEvent, UnifiedUserAction] = Table( - ("actionTweetType", "event", "expected"), - ("Default", tweetQuoteDefaultEvent, expectedUUAQuoteDefault), - ("Reply", tweetQuoteReplyEvent, expectedUUAQuoteReply), - ("Quote", tweetQuoteQuoteEvent, expectedUUAQuoteQuote), - ) - forEvery(tweetQuotes) { (_: String, event: TweetEvent, expected: UnifiedUserAction) => - val actual = TweetypieEventAdapter.adaptEvent(event) - assert(Seq(expected) === actual) - } - } - } - } - - test("TweetypieRetweetEvent") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val tweetRetweets: TableFor3[String, TweetEvent, UnifiedUserAction] = Table( - ("actionTweetType", "event", "expected"), - ("Default", tweetRetweetDefaultEvent, expectedUUARetweetDefault), - ("Reply", tweetRetweetReplyEvent, expectedUUARetweetReply), - ("Quote", tweetRetweetQuoteEvent, expectedUUARetweetQuote), - ("Retweet", tweetRetweetRetweetEvent, expectedUUARetweetRetweet), - ) - forEvery(tweetRetweets) { (_: String, event: TweetEvent, expected: UnifiedUserAction) => - val actual = TweetypieEventAdapter.adaptEvent(event) - assert(Seq(expected) === actual) - } - } - } - } - - test("TweetypieDeleteEvent") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val tweetDeletes: TableFor3[String, TweetEvent, UnifiedUserAction] = Table( - ("actionTweetType", "event", "expected"), - ("Default", tweetDeleteEvent, expectedUUADeleteDefault), - ("Reply", tweetUnreplyEvent, expectedUUAUnreply), - ("Quote", tweetUnquoteEvent, expectedUUAUnquote), - ("Retweet", tweetUnretweetEvent, expectedUUAUnretweet), - ) - forEvery(tweetDeletes) { (_: String, event: TweetEvent, expected: UnifiedUserAction) => - val actual = TweetypieEventAdapter.adaptEvent(event) - assert(Seq(expected) === actual) - } - } - } - } - - test("TweetypieEditEvent") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - val tweetEdits: TableFor3[String, TweetEvent, UnifiedUserAction] = Table( - ("actionTweetType", "event", "expected"), - ("RegularTweetFromEdit", regularTweetFromEditEvent, expectedUUARegularTweetFromEdit), - ("QuoteFromEdit", quoteFromEditEvent, expectedUUAQuoteFromEdit) - ) - forEvery(tweetEdits) { (_: String, event: TweetEvent, expected: UnifiedUserAction) => - val actual = TweetypieEventAdapter.adaptEvent(event) - assert(Seq(expected) === actual) - } - } - } - } - -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/UserModificationAdapterSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/UserModificationAdapterSpec.docx new file mode 100644 index 000000000..ccc29ec37 Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/UserModificationAdapterSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/UserModificationAdapterSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/UserModificationAdapterSpec.scala deleted file mode 100644 index 238e5dc7e..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/UserModificationAdapterSpec.scala +++ /dev/null @@ -1,25 +0,0 @@ -package unified_user_actions.adapter.src.test.scala.com.twitter.unified_user_actions.adapter - -import com.twitter.inject.Test -import com.twitter.unified_user_actions.adapter.TestFixtures.UserModificationEventFixture -import com.twitter.unified_user_actions.adapter.user_modification.UserModificationAdapter -import com.twitter.util.Time -import org.scalatest.prop.TableDrivenPropertyChecks - -class UserModificationAdapterSpec extends Test with TableDrivenPropertyChecks { - test("User Create") { - new UserModificationEventFixture { - Time.withTimeAt(frozenTime) { _ => - assert(UserModificationAdapter.adaptEvent(userCreate) === Seq(expectedUuaUserCreate)) - } - } - } - - test("User Update") { - new UserModificationEventFixture { - Time.withTimeAt(frozenTime) { _ => - assert(UserModificationAdapter.adaptEvent(userUpdate) === Seq(expectedUuaUserUpdate)) - } - } - } -} diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/VideoClientEventUtilsSpec.docx b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/VideoClientEventUtilsSpec.docx new file mode 100644 index 000000000..2df1bfcbe Binary files /dev/null and b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/VideoClientEventUtilsSpec.docx differ diff --git a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/VideoClientEventUtilsSpec.scala b/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/VideoClientEventUtilsSpec.scala deleted file mode 100644 index c22ca795a..000000000 --- a/unified_user_actions/adapter/src/test/scala/com/twitter/unified_user_actions/adapter/VideoClientEventUtilsSpec.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.unified_user_actions.adapter - -import com.twitter.clientapp.thriftscala.AmplifyDetails -import com.twitter.clientapp.thriftscala.MediaDetails -import com.twitter.clientapp.thriftscala.MediaType -import com.twitter.mediaservices.commons.thriftscala.MediaCategory -import com.twitter.unified_user_actions.adapter.client_event.VideoClientEventUtils.getVideoMetadata -import com.twitter.unified_user_actions.adapter.client_event.VideoClientEventUtils.videoIdFromMediaIdentifier -import com.twitter.unified_user_actions.thriftscala._ -import com.twitter.util.mock.Mockito -import com.twitter.video.analytics.thriftscala._ -import org.junit.runner.RunWith -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers -import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatestplus.junit.JUnitRunner - -@RunWith(classOf[JUnitRunner]) -class VideoClientEventUtilsSpec - extends AnyFunSuite - with Matchers - with Mockito - with TableDrivenPropertyChecks { - - trait Fixture { - val mediaDetails = Seq[MediaDetails]( - MediaDetails( - contentId = Some("456"), - mediaType = Some(MediaType.ConsumerVideo), - dynamicAds = Some(false)), - MediaDetails( - contentId = Some("123"), - mediaType = Some(MediaType.ConsumerVideo), - dynamicAds = Some(false)), - MediaDetails( - contentId = Some("789"), - mediaType = Some(MediaType.ConsumerVideo), - dynamicAds = Some(false)) - ) - - val videoMetadata: TweetActionInfo = TweetActionInfo.TweetVideoWatch( - TweetVideoWatch(mediaType = Some(MediaType.ConsumerVideo), isMonetizable = Some(false))) - - val videoMetadataWithAmplifyDetailsVideoType: TweetActionInfo = TweetActionInfo.TweetVideoWatch( - TweetVideoWatch( - mediaType = Some(MediaType.ConsumerVideo), - isMonetizable = Some(false), - videoType = Some("content"))) - - val validMediaIdentifier: MediaIdentifier = MediaIdentifier.MediaPlatformIdentifier( - MediaPlatformIdentifier(mediaId = 123L, mediaCategory = MediaCategory.TweetVideo)) - - val invalidMediaIdentifier: MediaIdentifier = MediaIdentifier.AmplifyCardIdentifier( - AmplifyCardIdentifier(vmapUrl = "", contentId = "") - ) - } - - test("findVideoMetadata") { - new Fixture { - val testData = Table( - ("testType", "mediaId", "mediaItems", "amplifyDetails", "expectedOutput"), - ("emptyMediaDetails", "123", Seq[MediaDetails](), None, None), - ("mediaIdNotFound", "111", mediaDetails, None, None), - ("mediaIdFound", "123", mediaDetails, None, Some(videoMetadata)), - ( - "mediaIdFound", - "123", - mediaDetails, - Some(AmplifyDetails(videoType = Some("content"))), - Some(videoMetadataWithAmplifyDetailsVideoType)) - ) - - forEvery(testData) { - ( - _: String, - mediaId: String, - mediaItems: Seq[MediaDetails], - amplifyDetails: Option[AmplifyDetails], - expectedOutput: Option[TweetActionInfo] - ) => - val actual = getVideoMetadata(mediaId, mediaItems, amplifyDetails) - assert(expectedOutput === actual) - } - } - } - - test("videoIdFromMediaIdentifier") { - new Fixture { - val testData = Table( - ("testType", "mediaIdentifier", "expectedOutput"), - ("validMediaIdentifierType", validMediaIdentifier, Some("123")), - ("invalidMediaIdentifierType", invalidMediaIdentifier, None) - ) - - forEvery(testData) { - (_: String, mediaIdentifier: MediaIdentifier, expectedOutput: Option[String]) => - val actual = videoIdFromMediaIdentifier(mediaIdentifier) - assert(expectedOutput === actual) - } - } - } -} diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/BUILD b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/BUILD deleted file mode 100644 index 46e4c1c23..000000000 --- a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - # Our runtime is using Java 11, but for compatibility with other internal libraries that - # are still on Java 8, we'll make our target platform to be Java 8 as well until everyone can - # migrate. - platform = "java8", - tags = ["bazel-compatible"], -) diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/BUILD.docx b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/BUILD.docx new file mode 100644 index 000000000..01e4c69f8 Binary files /dev/null and b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/BUILD.docx differ diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Clusters.docx b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Clusters.docx new file mode 100644 index 000000000..7c799005a Binary files /dev/null and b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Clusters.docx differ diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Clusters.scala b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Clusters.scala deleted file mode 100644 index fd9c29aee..000000000 --- a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Clusters.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.unified_user_actions.client.config - -sealed trait ClusterConfig { - val name: String - val environment: EnvironmentConfig -} - -object Clusters { - /* - * Our production cluster for external consumption. Our SLAs are enforced. - */ - case object ProdCluster extends ClusterConfig { - override val name: String = Constants.UuaKafkaProdClusterName - override val environment: EnvironmentConfig = Environments.Prod - } - - /* - * Our staging cluster for external development and pre-releases. No SLAs are enforced. - */ - case object StagingCluster extends ClusterConfig { - override val name: String = Constants.UuaKafkaStagingClusterName - override val environment: EnvironmentConfig = Environments.Staging - } -} diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Constants.docx b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Constants.docx new file mode 100644 index 000000000..e52bcbe76 Binary files /dev/null and b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Constants.docx differ diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Constants.scala b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Constants.scala deleted file mode 100644 index c3f8244b2..000000000 --- a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Constants.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.unified_user_actions.client.config - -object Constants { - val UuaKafkaTopicName = "unified_user_actions" - val UuaEngagementOnlyKafkaTopicName = "unified_user_actions_engagements" - val UuaKafkaProdClusterName = "/s/kafka/bluebird-1" - val UuaKafkaStagingClusterName = "/s/kafka/custdevel" - val UuaProdEnv = "prod" - val UuaStagingEnv = "staging" -} diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Environments.docx b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Environments.docx new file mode 100644 index 000000000..433cc527b Binary files /dev/null and b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Environments.docx differ diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Environments.scala b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Environments.scala deleted file mode 100644 index 9e24363fe..000000000 --- a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/Environments.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.unified_user_actions.client.config - -sealed trait EnvironmentConfig { - val name: String -} - -object Environments { - case object Prod extends EnvironmentConfig { - override val name: String = Constants.UuaProdEnv - } - - case object Staging extends EnvironmentConfig { - override val name: String = Constants.UuaStagingEnv - } -} diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/KafkaConfigs.docx b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/KafkaConfigs.docx new file mode 100644 index 000000000..93033e0d8 Binary files /dev/null and b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/KafkaConfigs.docx differ diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/KafkaConfigs.scala b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/KafkaConfigs.scala deleted file mode 100644 index 54b4378f2..000000000 --- a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config/KafkaConfigs.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.unified_user_actions.client.config - -sealed trait ClientConfig { - val cluster: ClusterConfig - val topic: String - val environment: EnvironmentConfig -} - -class AbstractClientConfig(isEngagementOnly: Boolean, env: EnvironmentConfig) extends ClientConfig { - override val cluster: ClusterConfig = { - env match { - case Environments.Prod => Clusters.ProdCluster - case Environments.Staging => Clusters.StagingCluster - case _ => Clusters.ProdCluster - } - } - - override val topic: String = { - if (isEngagementOnly) Constants.UuaEngagementOnlyKafkaTopicName - else Constants.UuaKafkaTopicName - } - - override val environment: EnvironmentConfig = env -} - -object KafkaConfigs { - - /* - * Unified User Actions Kafka config with all events (engagements and impressions). - * Use this config when you mainly need impression data and data volume is not an issue. - */ - case object ProdUnifiedUserActions - extends AbstractClientConfig(isEngagementOnly = false, env = Environments.Prod) - - /* - * Unified User Actions Kafka config with engagements events only. - * Use this config when you only need engagement data. The data volume should be a lot smaller - * than our main config. - */ - case object ProdUnifiedUserActionsEngagementOnly - extends AbstractClientConfig(isEngagementOnly = true, env = Environments.Prod) - - /* - * Staging Environment for integration and testing. This is not a production config. - * - * Unified User Actions Kafka config with all events (engagements and impressions). - * Use this config when you mainly need impression data and data volume is not an issue. - */ - case object StagingUnifiedUserActions - extends AbstractClientConfig(isEngagementOnly = false, env = Environments.Staging) - - /* - * Staging Environment for integration and testing. This is not a production config. - * - * Unified User Actions Kafka config with engagements events only. - * Use this config when you only need engagement data. The data volume should be a lot smaller - * than our main config. - */ - case object StagingUnifiedUserActionsEngagementOnly - extends AbstractClientConfig(isEngagementOnly = true, env = Environments.Staging) -} diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/BUILD b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/BUILD deleted file mode 100644 index b57b14ead..000000000 --- a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = [ - "UnifiedUserActionsSourceScrooge.scala", - ], - compiler_option_sets = ["fatal_warnings"], - # Our runtime is using Java 11, but for compatibility with other internal libraries that - # are still on Java 8, we'll make our target platform to be Java 8 as well until everyone can - # migrate. - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/src/jvm/com/twitter/summingbird:core", - "3rdparty/src/jvm/com/twitter/summingbird:storm", - "3rdparty/src/jvm/com/twitter/tormenta:core", - "src/scala/com/twitter/summingbird_internal/sources/common", - "src/scala/com/twitter/tormenta_internal/scheme", - "src/scala/com/twitter/tormenta_internal/spout:kafka2", - "unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/BUILD.docx b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/BUILD.docx new file mode 100644 index 000000000..c878c71c1 Binary files /dev/null and b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/BUILD.docx differ diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/UnifiedUserActionsSourceScrooge.docx b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/UnifiedUserActionsSourceScrooge.docx new file mode 100644 index 000000000..fae27c4e8 Binary files /dev/null and b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/UnifiedUserActionsSourceScrooge.docx differ diff --git a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/UnifiedUserActionsSourceScrooge.scala b/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/UnifiedUserActionsSourceScrooge.scala deleted file mode 100644 index 603517087..000000000 --- a/unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/summingbird/UnifiedUserActionsSourceScrooge.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.unified_user_actions.client.summingbird - -import com.twitter.summingbird.TimeExtractor -import com.twitter.summingbird.storm.Storm -import com.twitter.summingbird_internal.sources.AppId -import com.twitter.summingbird_internal.sources.SourceFactory -import com.twitter.tormenta_internal.spout.Kafka2ScroogeSpoutWrapper -import com.twitter.unified_user_actions.client.config.ClientConfig -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.unified_user_actions.client.config.KafkaConfigs - -case class UnifiedUserActionsSourceScrooge( - appId: AppId, - parallelism: Int, - kafkaConfig: ClientConfig = KafkaConfigs.ProdUnifiedUserActions, - skipToLatest: Boolean = false, - enableTls: Boolean = true) - extends SourceFactory[Storm, UnifiedUserAction] { - - override def name: String = "UnifiedUserActionsSource" - override def description: String = "Unified User Actions (UUA) events" - - // The event timestamps from summingbird's perspective (client), is our internally - // outputted timestamps (producer). This ensures time-continuity between the client and the - // producer. - val timeExtractor: TimeExtractor[UnifiedUserAction] = TimeExtractor { e => - e.eventMetadata.receivedTimestampMs - } - - override def source = { - Storm.source( - Kafka2ScroogeSpoutWrapper( - codec = UnifiedUserAction, - cluster = kafkaConfig.cluster.name, - topic = kafkaConfig.topic, - appId = appId.get, - skipToLatest = skipToLatest, - enableTls = enableTls - ), - Some(parallelism) - )(timeExtractor) - } -} diff --git a/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/BUILD.bazel b/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/BUILD.bazel deleted file mode 100644 index 3b7e20ff0..000000000 --- a/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -junit_tests( - sources = ["**/*.scala"], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "finatra/inject/inject-core/src/test/scala:test-deps", - "unified_user_actions/client/src/main/scala/com/twitter/unified_user_actions/client/config", - ], -) diff --git a/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/BUILD.docx b/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/BUILD.docx new file mode 100644 index 000000000..259adac89 Binary files /dev/null and b/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/BUILD.docx differ diff --git a/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/KafkaConfigsSpec.docx b/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/KafkaConfigsSpec.docx new file mode 100644 index 000000000..0b9fb8861 Binary files /dev/null and b/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/KafkaConfigsSpec.docx differ diff --git a/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/KafkaConfigsSpec.scala b/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/KafkaConfigsSpec.scala deleted file mode 100644 index 14c741789..000000000 --- a/unified_user_actions/client/src/test/scala/com/twitter/unified_user_actions/client/config/KafkaConfigsSpec.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.unified_user_actions.client.config - -import com.twitter.inject.Test - -class KafkaConfigsSpec extends Test { - test("configs should be correct") { - val states = Seq( - ( - KafkaConfigs.ProdUnifiedUserActions, - Constants.UuaProdEnv, - Constants.UuaKafkaTopicName, - Constants.UuaKafkaProdClusterName), - ( - KafkaConfigs.ProdUnifiedUserActionsEngagementOnly, - Constants.UuaProdEnv, - Constants.UuaEngagementOnlyKafkaTopicName, - Constants.UuaKafkaProdClusterName), - ( - KafkaConfigs.StagingUnifiedUserActions, - Constants.UuaStagingEnv, - Constants.UuaKafkaTopicName, - Constants.UuaKafkaStagingClusterName), - ( - KafkaConfigs.StagingUnifiedUserActionsEngagementOnly, - Constants.UuaStagingEnv, - Constants.UuaEngagementOnlyKafkaTopicName, - Constants.UuaKafkaStagingClusterName) - ) - - states.foreach { - case (actual, expectedEnv, expectedTopic, expectedClusterName) => - assert(expectedEnv == actual.environment.name, s"in $actual") - assert(expectedTopic == actual.topic, s"in $actual") - assert(expectedClusterName == actual.cluster.name, s"in $actual") - case _ => - } - } -} diff --git a/unified_user_actions/enricher/BUILD.bazel b/unified_user_actions/enricher/BUILD.bazel deleted file mode 100644 index 1624a57d4..000000000 --- a/unified_user_actions/enricher/BUILD.bazel +++ /dev/null @@ -1 +0,0 @@ -# This prevents SQ query from grabbing //:all since it traverses up once to find a BUILD diff --git a/unified_user_actions/enricher/BUILD.docx b/unified_user_actions/enricher/BUILD.docx new file mode 100644 index 000000000..3fb6236dc Binary files /dev/null and b/unified_user_actions/enricher/BUILD.docx differ diff --git a/unified_user_actions/enricher/README.docx b/unified_user_actions/enricher/README.docx new file mode 100644 index 000000000..3cc77f9d2 Binary files /dev/null and b/unified_user_actions/enricher/README.docx differ diff --git a/unified_user_actions/enricher/README.md b/unified_user_actions/enricher/README.md deleted file mode 100644 index 0b9314bdb..000000000 --- a/unified_user_actions/enricher/README.md +++ /dev/null @@ -1,24 +0,0 @@ -## Aurora deploy - -## From master branch - -``` -aurora workflow build unified_user_actions/service/deploy/uua-partitioner-staging.workflow -``` - -## From your own branch - -``` -git push origin / -aurora workflow build --build-branch=/ unified_user_actions/service/deploy/uua-partitioner-staging.workflow -``` - -* Check build status: - * Dev - * https://workflows.twitter.biz/workflow/discode/uua-partitioner-staging/ - -## Monitor output topic EPS - * Prod - * unified_user_actions: https://monitoring.twitter.biz/tiny/2942881 - * Dev - * unified_user_action_sample1: https://monitoring.twitter.biz/tiny/2942879 diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/BUILD b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/BUILD deleted file mode 100644 index c9697053a..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/BUILD +++ /dev/null @@ -1,5 +0,0 @@ -scala_library( - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [], -) diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/BUILD.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/BUILD.docx new file mode 100644 index 000000000..414e6044a Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/Exceptions.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/Exceptions.docx new file mode 100644 index 000000000..b5ae53695 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/Exceptions.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/Exceptions.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/Exceptions.scala deleted file mode 100644 index a27eaa0b9..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/Exceptions.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.unified_user_actions.enricher - -/** - * When this exception is thrown, it means that an assumption in the enricher services - * was violated and it needs to be fixed before a production deployment. - */ -abstract class FatalException(msg: String) extends Exception(msg) - -class ImplementationException(msg: String) extends FatalException(msg) - -object Exceptions { - def require(requirement: Boolean, message: String): Unit = { - if (!requirement) - throw new ImplementationException("requirement failed: " + message) - } -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/BUILD b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/BUILD deleted file mode 100644 index 1336f18ff..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator:base", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner:base", - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/BUILD.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/BUILD.docx new file mode 100644 index 000000000..4f763f048 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentDriver.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentDriver.docx new file mode 100644 index 000000000..1ff49a211 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentDriver.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentDriver.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentDriver.scala deleted file mode 100644 index 54999d810..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentDriver.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.unified_user_actions.enricher.driver - -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageType.Hydration -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageType.Repartition -import com.twitter.util.Future -import EnrichmentPlanUtils._ -import com.twitter.unified_user_actions.enricher.Exceptions -import com.twitter.unified_user_actions.enricher.ImplementationException -import com.twitter.unified_user_actions.enricher.hydrator.Hydrator -import com.twitter.unified_user_actions.enricher.partitioner.Partitioner - -/** - * A driver that will execute on a key, value tuple and produce an output to a Kafka topic. - * - * The output Kafka topic will depend on the current enrichment plan. In one scenario, the driver - * will output to a partitioned Kafka topic if the output needs to be repartitioned (after it has - * been hydrated 0 or more times as necessary). In another scenario, the driver will output to - * the final topic if there's no more work to be done. - * - * @param finalOutputTopic The final output Kafka topic - * @param partitionedTopic The intermediate Kafka topic used for repartitioning based on [[EnrichmentKey]] - * @param hydrator A hydrator that knows how to populate the metadata based on the current plan / instruction. - * @param partitioner A partitioner that knows how to transform the current uua event into an [[EnrichmentKey]]. - */ -class EnrichmentDriver( - finalOutputTopic: Option[String], - partitionedTopic: String, - hydrator: Hydrator, - partitioner: Partitioner) { - - /** - * A driver that does the following when being executed. - * It checks if we are done with enrichment plan, if not: - * - is the current stage repartitioning? - * -> remap the output key, update plan accordingly then return with the new partition key - * - is the current stage hydration? - * -> use the hydrator to hydrate the envelop, update the plan accordingly, then proceed - * recursively unless the next stage is repartitioning or this is the last stage. - */ - def execute( - key: Option[EnrichmentKey], - envelop: Future[EnrichmentEnvelop] - ): Future[(Option[EnrichmentKey], EnrichmentEnvelop)] = { - envelop.flatMap { envelop => - val plan = envelop.plan - if (plan.isEnrichmentComplete) { - val topic = finalOutputTopic.getOrElse( - throw new ImplementationException( - "A final output Kafka topic is supposed to be used but " + - "no final output topic was provided.")) - Future.value((key, envelop.copy(plan = plan.markLastStageCompletedWithOutputTopic(topic)))) - } else { - val currentStage = plan.getCurrentStage - - currentStage.stageType match { - case Repartition => - Exceptions.require( - currentStage.instructions.size == 1, - s"re-partitioning needs exactly 1 instruction but ${currentStage.instructions.size} was provided") - - val instruction = currentStage.instructions.head - val outputKey = partitioner.repartition(instruction, envelop) - val outputValue = envelop.copy( - plan = plan.markStageCompletedWithOutputTopic( - stage = currentStage, - outputTopic = partitionedTopic) - ) - Future.value((outputKey, outputValue)) - case Hydration => - Exceptions.require( - currentStage.instructions.nonEmpty, - "hydration needs at least one instruction") - - // Hydration is either initialized or completed after this, failure state - // will have to be handled upstream. Any unhandled exception will abort the entire - // stage. - // This is so that if the error in unrecoverable, the hydrator can choose to return an - // un-hydrated envelop to tolerate the error. - val finalEnvelop = currentStage.instructions.foldLeft(Future.value(envelop)) { - (curEnvelop, instruction) => - curEnvelop.flatMap(e => hydrator.hydrate(instruction, key, e)) - } - - val outputValue = finalEnvelop.map(e => - e.copy( - plan = plan.markStageCompleted(stage = currentStage) - )) - - // continue executing other stages if it can (locally) until a terminal state - execute(key, outputValue) - case _ => - throw new ImplementationException(s"Invalid / unsupported stage type $currentStage") - } - } - } - } -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentPlanUtils.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentPlanUtils.docx new file mode 100644 index 000000000..7b94c729f Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentPlanUtils.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentPlanUtils.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentPlanUtils.scala deleted file mode 100644 index 20f1093bc..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver/EnrichmentPlanUtils.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.unified_user_actions.enricher.driver - -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentPlan -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStage -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageStatus.Completion -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageStatus.Initialized - -object EnrichmentPlanUtils { - implicit class EnrichmentPlanStatus(plan: EnrichmentPlan) { - - /** - * Check each stage of the plan to know if we are done - */ - def isEnrichmentComplete: Boolean = { - plan.stages.forall(stage => stage.status == Completion) - } - - /** - * Get the next stage in the enrichment process. Note, if there is none this will throw - * an exception. - */ - def getCurrentStage: EnrichmentStage = { - val next = plan.stages.find(stage => stage.status == Initialized) - next match { - case Some(stage) => stage - case None => throw new IllegalStateException("check for plan completion first") - } - } - def getLastCompletedStage: EnrichmentStage = { - val completed = plan.stages.reverse.find(stage => stage.status == Completion) - completed match { - case Some(stage) => stage - case None => throw new IllegalStateException("check for plan completion first") - } - } - - /** - * Copy the current plan with the requested stage marked as complete - */ - def markStageCompletedWithOutputTopic( - stage: EnrichmentStage, - outputTopic: String - ): EnrichmentPlan = { - plan.copy( - stages = plan.stages.map(s => - if (s == stage) s.copy(status = Completion, outputTopic = Some(outputTopic)) else s) - ) - } - - def markStageCompleted( - stage: EnrichmentStage - ): EnrichmentPlan = { - plan.copy( - stages = plan.stages.map(s => if (s == stage) s.copy(status = Completion) else s) - ) - } - - /** - * Copy the current plan with the last stage marked as necessary - */ - def markLastStageCompletedWithOutputTopic( - outputTopic: String - ): EnrichmentPlan = { - val last = plan.stages.last - plan.copy( - stages = plan.stages.map(s => - if (s == last) s.copy(status = Completion, outputTopic = Some(outputTopic)) else s) - ) - } - } -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD deleted file mode 100644 index 09d16ff7a..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "featureswitches/dynmap/src/main/scala/com/twitter/dynmap:dynmap-core", - "featureswitches/dynmap/src/main/scala/com/twitter/dynmap/json:dynmap-json", - "graphql/thrift/src/main/thrift/com/twitter/graphql:graphql-scala", - "util/util-core:scala", - ], -) diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD.docx new file mode 100644 index 000000000..2b250dd06 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlRspParser.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlRspParser.docx new file mode 100644 index 000000000..d648976cc Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlRspParser.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlRspParser.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlRspParser.scala deleted file mode 100644 index 965c1ddbb..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlRspParser.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.unified_user_actions.enricher.graphql - -import com.google.common.util.concurrent.RateLimiter -import com.twitter.dynmap.DynMap -import com.twitter.dynmap.json.DynMapJson -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.util.logging.Logging -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -/** - * @param dm The DynMap parsed from the returned Json string - */ -case class GraphqlRspErrors(dm: DynMap) extends Exception { - override def toString: String = dm.toString() -} - -object GraphqlRspParser extends Logging { - private val rateLimiter = RateLimiter.create(1.0) // at most 1 log message per second - private def rateLimitedLogError(e: Throwable): Unit = - if (rateLimiter.tryAcquire()) { - error(e.getMessage, e) - } - - /** - * GraphQL's response is a Json string. - * This function first parses the raw response as a Json string, then it checks if the returned - * object has the "data" field which means the response is expected. The response could also - * return a valid Json string but with errors inside it as a list of "errors". - */ - def toDynMap( - rsp: String, - invalidRspCounter: Counter = NullStatsReceiver.NullCounter, - failedReqCounter: Counter = NullStatsReceiver.NullCounter - ): Try[DynMap] = { - val rawRsp: Try[DynMap] = DynMapJson.fromJsonString(rsp) - rawRsp match { - case Return(r) => - if (r.getMapOpt("data").isDefined) Return(r) - else { - invalidRspCounter.incr() - rateLimitedLogError(GraphqlRspErrors(r)) - Throw(GraphqlRspErrors(r)) - } - case Throw(e) => - rateLimitedLogError(e) - failedReqCounter.incr() - Throw(e) - } - } - - /** - * Similar to `toDynMap` above, but returns an Option - */ - def toDynMapOpt( - rsp: String, - invalidRspCounter: Counter = NullStatsReceiver.NullCounter, - failedReqCounter: Counter = NullStatsReceiver.NullCounter - ): Option[DynMap] = - toDynMap( - rsp = rsp, - invalidRspCounter = invalidRspCounter, - failedReqCounter = failedReqCounter).toOption -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD deleted file mode 100644 index b1579c1e1..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - name = "hcache", - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "util/util-cache-guava/src/main/scala", - "util/util-cache/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD.docx new file mode 100644 index 000000000..c7f2fe530 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCache.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCache.docx new file mode 100644 index 000000000..808b00c67 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCache.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCache.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCache.scala deleted file mode 100644 index 6cae67422..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCache.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.unified_user_actions.enricher.hcache - -import com.google.common.cache.Cache -import com.twitter.cache.FutureCache -import com.twitter.cache.guava.GuavaCache -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Future - -/** - * A local cache implementation using GuavaCache. - * Underneath it uses a customized version of the EvictingCache to 1) deal with Futures, 2) add more stats. - */ -class LocalCache[K, V]( - underlying: Cache[K, Future[V]], - statsReceiver: StatsReceiver = NullStatsReceiver) { - - private[this] val cache = new GuavaCache(underlying) - private[this] val evictingCache: FutureCache[K, V] = - ObservedEvictingCache(underlying = cache, statsReceiver = statsReceiver) - - def getOrElseUpdate(key: K)(fn: => Future[V]): Future[V] = evictingCache.getOrElseUpdate(key)(fn) - - def get(key: K): Option[Future[V]] = evictingCache.get(key) - - def evict(key: K, value: Future[V]): Boolean = evictingCache.evict(key, value) - - def set(key: K, value: Future[V]): Unit = evictingCache.set(key, value) - - def reset(): Unit = - underlying.invalidateAll() - - def size: Int = evictingCache.size -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/ObservedEvictingCache.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/ObservedEvictingCache.docx new file mode 100644 index 000000000..24d089ba8 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/ObservedEvictingCache.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/ObservedEvictingCache.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/ObservedEvictingCache.scala deleted file mode 100644 index 8c7e60029..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache/ObservedEvictingCache.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.unified_user_actions.enricher.hcache - -import com.twitter.cache.FutureCache -import com.twitter.cache.FutureCacheProxy -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Future -import scala.annotation.nowarn - -/** - * Adds stats and reuse the main logic of the EvictingCache. - */ -class ObservedEvictingCache[K, V](underlying: FutureCache[K, V], scopedStatsReceiver: StatsReceiver) - extends FutureCacheProxy[K, V](underlying) { - import ObservedEvictingCache._ - - private[this] val getsCounter = scopedStatsReceiver.counter(StatsNames.Gets) - private[this] val setsCounter = scopedStatsReceiver.counter(StatsNames.Sets) - private[this] val hitsCounter = scopedStatsReceiver.counter(StatsNames.Hits) - private[this] val missesCounter = scopedStatsReceiver.counter(StatsNames.Misses) - private[this] val evictionsCounter = scopedStatsReceiver.counter(StatsNames.Evictions) - private[this] val failedFuturesCounter = scopedStatsReceiver.counter(StatsNames.FailedFutures) - - @nowarn("cat=unused") - private[this] val cacheSizeGauge = scopedStatsReceiver.addGauge(StatsNames.Size)(underlying.size) - - private[this] def evictOnFailure(k: K, f: Future[V]): Future[V] = { - f.onFailure { _ => - failedFuturesCounter.incr() - evict(k, f) - } - f // we return the original future to make evict(k, f) easier to work with. - } - - override def set(k: K, v: Future[V]): Unit = { - setsCounter.incr() - super.set(k, v) - evictOnFailure(k, v) - } - - override def getOrElseUpdate(k: K)(v: => Future[V]): Future[V] = { - getsCounter.incr() - - var computeWasEvaluated = false - def computeWithTracking: Future[V] = v.onSuccess { _ => - computeWasEvaluated = true - missesCounter.incr() - } - - evictOnFailure( - k, - super.getOrElseUpdate(k)(computeWithTracking).onSuccess { _ => - if (!computeWasEvaluated) hitsCounter.incr() - } - ).interruptible() - } - - override def get(key: K): Option[Future[V]] = { - getsCounter.incr() - val value = super.get(key) - value match { - case Some(_) => hitsCounter.incr() - case _ => missesCounter.incr() - } - value - } - - override def evict(key: K, value: Future[V]): Boolean = { - val evicted = super.evict(key, value) - if (evicted) evictionsCounter.incr() - evicted - } -} - -object ObservedEvictingCache { - object StatsNames { - val Gets = "gets" - val Hits = "hits" - val Misses = "misses" - val Sets = "sets" - val Evictions = "evictions" - val FailedFutures = "failed_futures" - val Size = "size" - } - - /** - * Wraps an underlying FutureCache, ensuring that failed Futures that are set in - * the cache are evicted later. - */ - def apply[K, V](underlying: FutureCache[K, V], statsReceiver: StatsReceiver): FutureCache[K, V] = - new ObservedEvictingCache[K, V](underlying, statsReceiver) -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/AbstractHydrator.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/AbstractHydrator.docx new file mode 100644 index 000000000..f1e8d1ae0 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/AbstractHydrator.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/AbstractHydrator.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/AbstractHydrator.scala deleted file mode 100644 index e8444e965..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/AbstractHydrator.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.unified_user_actions.enricher.hydrator -import com.google.common.util.concurrent.RateLimiter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.unified_user_actions.enricher.FatalException -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.util.Future -import com.twitter.util.logging.Logging - -abstract class AbstractHydrator(scopedStatsReceiver: StatsReceiver) extends Hydrator with Logging { - - object StatsNames { - val Exceptions = "exceptions" - val EmptyKeys = "empty_keys" - val Hydrations = "hydrations" - } - - private val exceptionsCounter = scopedStatsReceiver.counter(StatsNames.Exceptions) - private val emptyKeysCounter = scopedStatsReceiver.counter(StatsNames.EmptyKeys) - private val hydrationsCounter = scopedStatsReceiver.counter(StatsNames.Hydrations) - - // at most 1 log message per second - private val rateLimiter = RateLimiter.create(1.0) - - private def rateLimitedLogError(e: Throwable): Unit = - if (rateLimiter.tryAcquire()) { - error(e.getMessage, e) - } - - protected def safelyHydrate( - instruction: EnrichmentInstruction, - keyOpt: EnrichmentKey, - envelop: EnrichmentEnvelop - ): Future[EnrichmentEnvelop] - - override def hydrate( - instruction: EnrichmentInstruction, - keyOpt: Option[EnrichmentKey], - envelop: EnrichmentEnvelop - ): Future[EnrichmentEnvelop] = { - keyOpt - .map(key => { - safelyHydrate(instruction, key, envelop) - .onSuccess(_ => hydrationsCounter.incr()) - .rescue { - case e: FatalException => Future.exception(e) - case e => - rateLimitedLogError(e) - exceptionsCounter.incr() - Future.value(envelop) - } - }).getOrElse({ - emptyKeysCounter.incr() - Future.value(envelop) - }) - } -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD deleted file mode 100644 index 3f4bb6780..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - name = "default", - sources = [ - "AbstractHydrator.scala", - "DefaultHydrator.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - ":base", - "featureswitches/dynmap/src/main/scala/com/twitter/dynmap:dynmap-core", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache", - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - ], -) - -scala_library( - name = "noop", - sources = ["NoopHydrator.scala"], - tags = ["bazel-compatible"], - dependencies = [ - ":base", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher", - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - ], -) - -scala_library( - name = "base", - sources = ["Hydrator.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - ], -) diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD.docx new file mode 100644 index 000000000..ba6e2f64a Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydrator.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydrator.docx new file mode 100644 index 000000000..84a118884 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydrator.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydrator.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydrator.scala deleted file mode 100644 index ac5802070..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydrator.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.unified_user_actions.enricher.hydrator -import com.twitter.dynmap.DynMap -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.graphql.thriftscala.AuthHeaders -import com.twitter.graphql.thriftscala.Authentication -import com.twitter.graphql.thriftscala.Document -import com.twitter.graphql.thriftscala.GraphQlRequest -import com.twitter.graphql.thriftscala.GraphqlExecutionService -import com.twitter.graphql.thriftscala.Variables -import com.twitter.unified_user_actions.enricher.ImplementationException -import com.twitter.unified_user_actions.enricher.graphql.GraphqlRspParser -import com.twitter.unified_user_actions.enricher.hcache.LocalCache -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentIdType -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.unified_user_actions.thriftscala.AuthorInfo -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.util.Future - -class DefaultHydrator( - cache: LocalCache[EnrichmentKey, DynMap], - graphqlClient: GraphqlExecutionService.FinagledClient, - scopedStatsReceiver: StatsReceiver = NullStatsReceiver) - extends AbstractHydrator(scopedStatsReceiver) { - - private def constructGraphqlReq( - enrichmentKey: EnrichmentKey - ): GraphQlRequest = - enrichmentKey.keyType match { - case EnrichmentIdType.TweetId => - GraphQlRequest( - // see go/graphiql/M5sHxua-RDiRtTn48CAhng - document = Document.DocumentId("M5sHxua-RDiRtTn48CAhng"), - operationName = Some("TweetHydration"), - variables = Some( - Variables.JsonEncodedVariables(s"""{"rest_id": "${enrichmentKey.id}"}""") - ), - authentication = Authentication.AuthHeaders( - AuthHeaders() - ) - ) - case _ => - throw new ImplementationException( - s"Missing implementation for hydration of type ${enrichmentKey.keyType}") - } - - private def hydrateAuthorInfo(item: Item.TweetInfo, authorId: Option[Long]): Item.TweetInfo = { - item.tweetInfo.actionTweetAuthorInfo match { - case Some(_) => item - case _ => - item.copy(tweetInfo = item.tweetInfo.copy( - actionTweetAuthorInfo = Some(AuthorInfo(authorId = authorId)) - )) - } - } - - override protected def safelyHydrate( - instruction: EnrichmentInstruction, - key: EnrichmentKey, - envelop: EnrichmentEnvelop - ): Future[EnrichmentEnvelop] = { - instruction match { - case EnrichmentInstruction.TweetEnrichment => - val dynMapFuture = cache.getOrElseUpdate(key) { - graphqlClient - .graphql(constructGraphqlReq(enrichmentKey = key)) - .map { body => - body.response.flatMap { r => - GraphqlRspParser.toDynMapOpt(r) - }.get - } - } - - dynMapFuture.map(map => { - val authorIdOpt = - map.getLongOpt("data.tweet_result_by_rest_id.result.core.user.legacy.id_str") - - val hydratedEnvelop = envelop.uua.item match { - case item: Item.TweetInfo => - envelop.copy(uua = envelop.uua.copy(item = hydrateAuthorInfo(item, authorIdOpt))) - case _ => envelop - } - hydratedEnvelop - }) - case _ => Future.value(envelop) - } - } -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/Hydrator.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/Hydrator.docx new file mode 100644 index 000000000..ad345401c Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/Hydrator.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/Hydrator.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/Hydrator.scala deleted file mode 100644 index c03bc7df6..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/Hydrator.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.unified_user_actions.enricher.hydrator - -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.util.Future - -trait Hydrator { - def hydrate( - instruction: EnrichmentInstruction, - key: Option[EnrichmentKey], - envelop: EnrichmentEnvelop - ): Future[EnrichmentEnvelop] -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydrator.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydrator.docx new file mode 100644 index 000000000..98ffe6ecf Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydrator.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydrator.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydrator.scala deleted file mode 100644 index 652b40111..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydrator.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.unified_user_actions.enricher.hydrator -import com.twitter.unified_user_actions.enricher.ImplementationException -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.util.Future - -/** - * This hydrator does nothing. If it's used by mistake for any reason, an exception will be thrown. - * Use this when you expect to have no hydration (for example, the planner shouldn't hydrate anything - * and only would perform the partitioning function). - */ -object NoopHydrator { - val OutputTopic: Option[String] = None -} - -class NoopHydrator extends Hydrator { - override def hydrate( - instruction: EnrichmentInstruction, - key: Option[EnrichmentKey], - envelop: EnrichmentEnvelop - ): Future[EnrichmentEnvelop] = { - throw new ImplementationException( - "NoopHydrator shouldn't be invoked when configure. Check your " + - "enrichment plan.") - } -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD deleted file mode 100644 index 7a6098f17..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - name = "default", - sources = ["DefaultPartitioner.scala"], - tags = ["bazel-compatible"], - dependencies = [ - ":base", - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - ], -) - -scala_library( - name = "base", - sources = ["Partitioner.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - ], -) diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD.docx new file mode 100644 index 000000000..430f7acba Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitioner.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitioner.docx new file mode 100644 index 000000000..c2f6a65b4 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitioner.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitioner.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitioner.scala deleted file mode 100644 index 06e88cc08..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitioner.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.unified_user_actions.enricher.partitioner -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentIdType -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction.NotificationTweetEnrichment -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction.TweetEnrichment -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.unified_user_actions.enricher.partitioner.DefaultPartitioner.NullKey -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.NotificationContent - -object DefaultPartitioner { - val NullKey: Option[EnrichmentKey] = None -} - -class DefaultPartitioner extends Partitioner { - override def repartition( - instruction: EnrichmentInstruction, - envelop: EnrichmentEnvelop - ): Option[EnrichmentKey] = { - (instruction, envelop.uua.item) match { - case (TweetEnrichment, Item.TweetInfo(info)) => - Some(EnrichmentKey(EnrichmentIdType.TweetId, info.actionTweetId)) - case (NotificationTweetEnrichment, Item.NotificationInfo(info)) => - info.content match { - case NotificationContent.TweetNotification(content) => - Some(EnrichmentKey(EnrichmentIdType.TweetId, content.tweetId)) - case NotificationContent.MultiTweetNotification(content) => - // we scarify on cache performance in this case since only a small % of - // notification content will be multi-tweet types. - Some(EnrichmentKey(EnrichmentIdType.TweetId, content.tweetIds.head)) - case _ => NullKey - } - case _ => NullKey - } - } -} diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/Partitioner.docx b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/Partitioner.docx new file mode 100644 index 000000000..766ff0496 Binary files /dev/null and b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/Partitioner.docx differ diff --git a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/Partitioner.scala b/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/Partitioner.scala deleted file mode 100644 index 0281c1a30..000000000 --- a/unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner/Partitioner.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.unified_user_actions.enricher.partitioner - -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey - -trait Partitioner { - def repartition( - instruction: EnrichmentInstruction, - envelop: EnrichmentEnvelop - ): Option[EnrichmentKey] -} diff --git a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/BUILD b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/BUILD deleted file mode 100644 index b2eb4873f..000000000 --- a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -create_thrift_libraries( - org = "com.twitter.unified_user_actions.enricher", - base_name = "internal", - sources = ["*.thrift"], - tags = ["bazel-compatible"], - dependency_roots = [ - "src/thrift/com/twitter/clientapp/gen:clientapp", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions", - ], - generate_languages = [ - "java", - "scala", - ], - provides_java_name = "enricher_internal-thrift-java", - provides_scala_name = "enricher_internal-thrift-scala", -) diff --git a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/BUILD.docx b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/BUILD.docx new file mode 100644 index 000000000..4a730d7b8 Binary files /dev/null and b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_envelop.docx b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_envelop.docx new file mode 100644 index 000000000..d4f6c0a83 Binary files /dev/null and b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_envelop.docx differ diff --git a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_envelop.thrift b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_envelop.thrift deleted file mode 100644 index 5f01dc039..000000000 --- a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_envelop.thrift +++ /dev/null @@ -1,26 +0,0 @@ -namespace java com.twitter.unified_user_actions.enricher.internal.thriftjava -#@namespace scala com.twitter.unified_user_actions.enricher.internal.thriftscala -#@namespace strato com.twitter.unified_user_actions.enricher.internal - -include "com/twitter/unified_user_actions/unified_user_actions.thrift" -include "enrichment_plan.thrift" - -struct EnrichmentEnvelop { - /** - * An internal ID that uniquely identifies this event created during the early stages of enrichment. - * It is useful for detecting debugging, tracing & profiling the events throughout the process. - **/ - 1: required i64 envelopId - - /** - * The UUA event to be enriched / currently being enriched / has been enriched depending on the - * stages of the enrichment process. - **/ - 2: unified_user_actions.UnifiedUserAction uua - - /** - * The current enrichment plan. It keeps track of what is currently being enriched, what still - * needs to be done so that we can bring the enrichment process to completion. - **/ - 3: enrichment_plan.EnrichmentPlan plan -}(persisted='true', hasPersonalData='true') diff --git a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_key.docx b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_key.docx new file mode 100644 index 000000000..17540d151 Binary files /dev/null and b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_key.docx differ diff --git a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_key.thrift b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_key.thrift deleted file mode 100644 index abb9ea33d..000000000 --- a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_key.thrift +++ /dev/null @@ -1,41 +0,0 @@ -namespace java com.twitter.unified_user_actions.enricher.internal.thriftjava -#@namespace scala com.twitter.unified_user_actions.enricher.internal.thriftscala -#@namespace strato com.twitter.unified_user_actions.enricher.internal - -/* - * Internal key used for controling UUA enrichment & caching process. It contains very minimal - * information to allow for efficient serde, fast data look-up and to drive the partioning logics. - * - * NOTE: Don't depend on it in your application. - * NOTE: This is used internally by UUA and may change at anytime. There's no guarantee for - * backward / forward-compatibility. - * NOTE: Don't add any other metadata unless it is needed for partitioning logic. Extra enrichment - * metdata can go into the envelop. - */ -struct EnrichmentKey { - /* - * The internal type of the primary ID used for partitioning UUA data. - * - * Each type should directly correspond to an entity-level ID in UUA. - * For example, TweetInfo.actionTweetId & TweetNotification.tweetId are all tweet-entity level - * and should correspond to the same primary ID type. - **/ - 1: required EnrichmentIdType keyType - - /** - * The primary ID. This is usually a long, for other incompatible data type such as string or - * a bytes array, they can be converted into a long using their native hashCode() function. - **/ - 2: required i64 id -}(persisted='true', hasPersonalData='true') - -/** -* The type of the primary ID. For example, tweetId on a tweet & tweetId on a notification are -* all TweetId type. Similarly, UserID of a viewer and AuthorID of a tweet are all UserID type. -* -* The type here ensures that we will partition UUA data correctly across different entity-type -* (user, tweets, notification, etc.) -**/ -enum EnrichmentIdType { - TweetId = 0 -} diff --git a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_plan.docx b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_plan.docx new file mode 100644 index 000000000..5fdac5215 Binary files /dev/null and b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_plan.docx differ diff --git a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_plan.thrift b/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_plan.thrift deleted file mode 100644 index e64170752..000000000 --- a/unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal/enrichment_plan.thrift +++ /dev/null @@ -1,52 +0,0 @@ -namespace java com.twitter.unified_user_actions.enricher.internal.thriftjava -#@namespace scala com.twitter.unified_user_actions.enricher.internal.thriftscala -#@namespace strato com.twitter.unified_user_actions.enricher.internal - -/** -* An enrichment plan. It has multiple stages for different purposes during the enrichment process. -**/ -struct EnrichmentPlan { - 1: required list stages -}(persisted='true', hasPersonalData='false') - -/** -* A stage in the enrichment process with respect to the current key. Currently it can be of 2 options: -* - re-partitioning on an id of type X -* - hydrating metadata on an id of type X -* -* A stage also moves through different statues from initialized, processing until completion. -* Each stage contains one or more instructions. -**/ -struct EnrichmentStage { - 1: required EnrichmentStageStatus status - 2: required EnrichmentStageType stageType - 3: required list instructions - - // The output topic for this stage. This information is not available when the stage was - // first setup, and it's only available after the driver has finished working on - // this stage. - 4: optional string outputTopic -}(persisted='true', hasPersonalData='false') - -/** -* The current processing status of a stage. It should either be done (completion) or not done (initialized). -* Transient statuses such as "processing" is dangerous since we can't exactly be sure that has been done. -**/ -enum EnrichmentStageStatus { - Initialized = 0 - Completion = 20 -} - -/** -* The type of processing in this stage. For example, repartioning the data or hydrating the data. -**/ -enum EnrichmentStageType { - Repartition = 0 - Hydration = 10 -} - -enum EnrichmentInstruction { - // all enrichment based on a tweet id in UUA goes here - TweetEnrichment = 0 - NotificationTweetEnrichment = 10 -} diff --git a/unified_user_actions/enricher/src/test/resources/BUILD.bazel b/unified_user_actions/enricher/src/test/resources/BUILD.bazel deleted file mode 100644 index ae9669f4f..000000000 --- a/unified_user_actions/enricher/src/test/resources/BUILD.bazel +++ /dev/null @@ -1,4 +0,0 @@ -resources( - sources = ["*.*"], - tags = ["bazel-compatible"], -) diff --git a/unified_user_actions/enricher/src/test/resources/BUILD.docx b/unified_user_actions/enricher/src/test/resources/BUILD.docx new file mode 100644 index 000000000..939f8f0bb Binary files /dev/null and b/unified_user_actions/enricher/src/test/resources/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/test/resources/logback.docx b/unified_user_actions/enricher/src/test/resources/logback.docx new file mode 100644 index 000000000..5d1073331 Binary files /dev/null and b/unified_user_actions/enricher/src/test/resources/logback.docx differ diff --git a/unified_user_actions/enricher/src/test/resources/logback.xml b/unified_user_actions/enricher/src/test/resources/logback.xml deleted file mode 100644 index 27f50b1dc..000000000 --- a/unified_user_actions/enricher/src/test/resources/logback.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/BUILD.bazel b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/BUILD.bazel deleted file mode 100644 index 9f6cd6248..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - name = "fixture", - sources = ["EnricherFixture.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "finatra/inject/inject-core/src/test/scala/com/twitter/inject", - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/BUILD.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/BUILD.docx new file mode 100644 index 000000000..b9c6177fc Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/EnricherFixture.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/EnricherFixture.docx new file mode 100644 index 000000000..22b883201 Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/EnricherFixture.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/EnricherFixture.scala b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/EnricherFixture.scala deleted file mode 100644 index 7e7827ab6..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/EnricherFixture.scala +++ /dev/null @@ -1,100 +0,0 @@ -package com.twitter.unified_user_actions.enricher - -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentPlan -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStage -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageStatus -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageType -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.AuthorInfo -import com.twitter.unified_user_actions.thriftscala.EventMetadata -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.MultiTweetNotification -import com.twitter.unified_user_actions.thriftscala.NotificationContent -import com.twitter.unified_user_actions.thriftscala.NotificationInfo -import com.twitter.unified_user_actions.thriftscala.ProfileInfo -import com.twitter.unified_user_actions.thriftscala.SourceLineage -import com.twitter.unified_user_actions.thriftscala.TweetInfo -import com.twitter.unified_user_actions.thriftscala.TweetNotification -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.unified_user_actions.thriftscala.UnknownNotification -import com.twitter.unified_user_actions.thriftscala.UserIdentifier - -trait EnricherFixture { - val partitionedTopic = "unified_user_actions_keyed_dev" - val tweetInfoEnrichmentPlan = EnrichmentPlan( - Seq( - // first stage: to repartition on tweet id -> done - EnrichmentStage( - EnrichmentStageStatus.Completion, - EnrichmentStageType.Repartition, - Seq(EnrichmentInstruction.TweetEnrichment), - Some(partitionedTopic) - ), - // next stage: to hydrate more metadata based on tweet id -> initialized - EnrichmentStage( - EnrichmentStageStatus.Initialized, - EnrichmentStageType.Hydration, - Seq(EnrichmentInstruction.TweetEnrichment) - ) - )) - - val tweetNotificationEnrichmentPlan = EnrichmentPlan( - Seq( - // first stage: to repartition on tweet id -> done - EnrichmentStage( - EnrichmentStageStatus.Completion, - EnrichmentStageType.Repartition, - Seq(EnrichmentInstruction.NotificationTweetEnrichment), - Some(partitionedTopic) - ), - // next stage: to hydrate more metadata based on tweet id -> initialized - EnrichmentStage( - EnrichmentStageStatus.Initialized, - EnrichmentStageType.Hydration, - Seq(EnrichmentInstruction.NotificationTweetEnrichment), - ) - )) - - def mkUUATweetEvent(tweetId: Long, author: Option[AuthorInfo] = None): UnifiedUserAction = { - UnifiedUserAction( - UserIdentifier(userId = Some(1L)), - item = Item.TweetInfo(TweetInfo(actionTweetId = tweetId, actionTweetAuthorInfo = author)), - actionType = ActionType.ClientTweetReport, - eventMetadata = EventMetadata(1234L, 2345L, SourceLineage.ServerTweetypieEvents) - ) - } - - def mkUUATweetNotificationEvent(tweetId: Long): UnifiedUserAction = { - mkUUATweetEvent(-1L).copy( - item = Item.NotificationInfo( - NotificationInfo( - actionNotificationId = "123456", - content = NotificationContent.TweetNotification(TweetNotification(tweetId = tweetId)))) - ) - } - - def mkUUAMultiTweetNotificationEvent(tweetIds: Long*): UnifiedUserAction = { - mkUUATweetEvent(-1L).copy( - item = Item.NotificationInfo( - NotificationInfo( - actionNotificationId = "123456", - content = NotificationContent.MultiTweetNotification( - MultiTweetNotification(tweetIds = tweetIds)))) - ) - } - - def mkUUATweetNotificationUnknownEvent(): UnifiedUserAction = { - mkUUATweetEvent(-1L).copy( - item = Item.NotificationInfo( - NotificationInfo( - actionNotificationId = "123456", - content = NotificationContent.UnknownNotification(UnknownNotification()))) - ) - } - - def mkUUAProfileEvent(userId: Long): UnifiedUserAction = { - val event = mkUUATweetEvent(1L) - event.copy(item = Item.ProfileInfo(ProfileInfo(userId))) - } -} diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/BUILD.bazel b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/BUILD.bazel deleted file mode 100644 index a6109e868..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -junit_tests( - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "finatra/inject/inject-core/src/test/scala/com/twitter/inject", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator:base", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner:base", - "unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher:fixture", - "util/util-core:scala", - ], -) diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/BUILD.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/BUILD.docx new file mode 100644 index 000000000..07ba46f4c Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/DriverTest.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/DriverTest.docx new file mode 100644 index 000000000..f9a1f2bc6 Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/DriverTest.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/DriverTest.scala b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/DriverTest.scala deleted file mode 100644 index 434760d5c..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/driver/DriverTest.scala +++ /dev/null @@ -1,284 +0,0 @@ -package com.twitter.unified_user_actions.enricher.driver - -import com.twitter.inject.Test -import com.twitter.unified_user_actions.enricher.EnricherFixture -import com.twitter.unified_user_actions.enricher.hydrator.Hydrator -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentIdType -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentPlan -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStage -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageStatus -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageType -import com.twitter.unified_user_actions.enricher.partitioner.Partitioner -import com.twitter.util.Await -import com.twitter.util.Future -import org.scalatest.BeforeAndAfter -import org.scalatest.matchers.should.Matchers -import scala.collection.mutable - -class DriverTest extends Test with Matchers with BeforeAndAfter { - object ExecutionContext { - var executionCount = 0 - } - - before { - ExecutionContext.executionCount = 0 - } - - trait Fixtures extends EnricherFixture { - val repartitionTweet = mkStage() - val repartitionNotiTweet = - mkStage(instructions = Seq(EnrichmentInstruction.NotificationTweetEnrichment)) - val hydrateTweet = mkStage(stageType = EnrichmentStageType.Hydration) - val hydrateTweetMultiInstructions = mkStage( - stageType = EnrichmentStageType.Hydration, - instructions = Seq( - EnrichmentInstruction.NotificationTweetEnrichment, - EnrichmentInstruction.TweetEnrichment, - EnrichmentInstruction.NotificationTweetEnrichment, - EnrichmentInstruction.TweetEnrichment - ) - ) - val hydrateNotiTweet = mkStage( - stageType = EnrichmentStageType.Hydration, - instructions = Seq(EnrichmentInstruction.NotificationTweetEnrichment)) - val key1 = EnrichmentKey(EnrichmentIdType.TweetId, 123L) - val tweet1 = mkUUATweetEvent(981L) - val hydrator = new MockHydrator - val partitioner = new MockPartitioner - val outputTopic = "output" - val partitionTopic = "partition" - - def complete( - enrichmentStage: EnrichmentStage, - outputTopic: Option[String] = None - ): EnrichmentStage = { - enrichmentStage.copy(status = EnrichmentStageStatus.Completion, outputTopic = outputTopic) - } - - def mkPlan(enrichmentStages: EnrichmentStage*): EnrichmentPlan = { - EnrichmentPlan(enrichmentStages) - } - - def mkStage( - status: EnrichmentStageStatus = EnrichmentStageStatus.Initialized, - stageType: EnrichmentStageType = EnrichmentStageType.Repartition, - instructions: Seq[EnrichmentInstruction] = Seq(EnrichmentInstruction.TweetEnrichment) - ): EnrichmentStage = { - EnrichmentStage(status, stageType, instructions) - } - - trait ExecutionCount { - val callMap: mutable.Map[Int, (EnrichmentInstruction, EnrichmentEnvelop)] = - mutable.Map[Int, (EnrichmentInstruction, EnrichmentEnvelop)]() - - def recordExecution(instruction: EnrichmentInstruction, envelop: EnrichmentEnvelop): Unit = { - ExecutionContext.executionCount = ExecutionContext.executionCount + 1 - callMap.put(ExecutionContext.executionCount, (instruction, envelop)) - } - } - - class MockHydrator extends Hydrator with ExecutionCount { - def hydrate( - instruction: EnrichmentInstruction, - key: Option[EnrichmentKey], - envelop: EnrichmentEnvelop - ): Future[EnrichmentEnvelop] = { - recordExecution(instruction, envelop) - Future(envelop.copy(envelopId = ExecutionContext.executionCount)) - } - } - - class MockPartitioner extends Partitioner with ExecutionCount { - def repartition( - instruction: EnrichmentInstruction, - envelop: EnrichmentEnvelop - ): Option[EnrichmentKey] = { - recordExecution(instruction, envelop) - Some(EnrichmentKey(EnrichmentIdType.TweetId, ExecutionContext.executionCount)) - } - } - } - - test("single partitioning plan works") { - new Fixtures { - val driver = new EnrichmentDriver(Some(outputTopic), partitionTopic, hydrator, partitioner) - // given a simple plan that only repartition the input and nothing else - val plan = mkPlan(repartitionTweet) - - (1L to 10).foreach(id => { - val envelop = EnrichmentEnvelop(id, tweet1, plan) - - // when - val actual = Await.result(driver.execute(Some(key1), Future(envelop))) - - val expectedKey = Some(key1.copy(id = id)) - val expectedValue = - envelop.copy(plan = mkPlan(complete(repartitionTweet, Some(partitionTopic)))) - - // then the result should have a new partitioned key, with the envelop unchanged except the plan is complete - // however, the output topic is the partitionTopic (since this is only a partitioning stage) - assert((expectedKey, expectedValue) == actual) - }) - } - } - - test("multi-stage partitioning plan works") { - new Fixtures { - val driver = new EnrichmentDriver(Some(outputTopic), partitionTopic, hydrator, partitioner) - // given a plan that chain multiple repartition stages together - val plan = mkPlan(repartitionTweet, repartitionNotiTweet) - val envelop1 = EnrichmentEnvelop(1L, tweet1, plan) - - // when 1st partitioning trip - val actual1 = Await.result(driver.execute(Some(key1), Future(envelop1))) - - // then the result should have a new partitioned key, with the envelop unchanged except the - // 1st stage of the plan is complete - val expectedKey1 = key1.copy(id = 1L) - val expectedValue1 = - envelop1.copy(plan = - mkPlan(complete(repartitionTweet, Some(partitionTopic)), repartitionNotiTweet)) - - assert((Some(expectedKey1), expectedValue1) == actual1) - - // then, we reuse the last result to exercise the logics on the driver again for the 2st trip - val actual2 = Await.result(driver.execute(Some(expectedKey1), Future(expectedValue1))) - val expectedKey2 = key1.copy(id = 2L) - val expectedValue2 = - envelop1.copy(plan = mkPlan( - complete(repartitionTweet, Some(partitionTopic)), - complete(repartitionNotiTweet, Some(partitionTopic)))) - - assert((Some(expectedKey2), expectedValue2) == actual2) - } - } - - test("single hydration plan works") { - new Fixtures { - val driver = new EnrichmentDriver(Some(outputTopic), partitionTopic, hydrator, partitioner) - // given a simple plan that only hydrate the input and nothing else - val plan = mkPlan(hydrateTweet) - - (1L to 10).foreach(id => { - val envelop = EnrichmentEnvelop(id, tweet1, plan) - - // when - val actual = Await.result(driver.execute(Some(key1), Future(envelop))) - - val expectedValue = - envelop.copy(envelopId = id, plan = mkPlan(complete(hydrateTweet, Some(outputTopic)))) - - // then the result should have the same key, with the envelop hydrated & the plan is complete - // the output topic should be the final topic since this is a hydration stage and the plan is complete - assert((Some(key1), expectedValue) == actual) - }) - } - } - - test("single hydration with multiple instructions plan works") { - new Fixtures { - val driver = new EnrichmentDriver(Some(outputTopic), partitionTopic, hydrator, partitioner) - // given a simple plan that only hydrate the input and nothing else - val plan = mkPlan(hydrateTweetMultiInstructions) - val envelop = EnrichmentEnvelop(0L, tweet1, plan) - - // when - val actual = Await.result(driver.execute(Some(key1), Future(envelop))) - val expectedValue = envelop.copy( - envelopId = 4L, // hydrate is called 4 times for 4 instructions in 1 stage - plan = mkPlan(complete(hydrateTweetMultiInstructions, Some(outputTopic)))) - - // then the result should have the same key, with the envelop hydrated & the plan is complete - // the output topic should be the final topic since this is a hydration stage and the plan is complete - assert((Some(key1), expectedValue) == actual) - } - } - - test("multi-stage hydration plan works") { - new Fixtures { - val driver = new EnrichmentDriver(Some(outputTopic), partitionTopic, hydrator, partitioner) - // given a plan that only hydrate twice - val plan = mkPlan(hydrateTweet, hydrateNotiTweet) - val envelop = EnrichmentEnvelop(1L, tweet1, plan) - - // when - val actual = Await.result(driver.execute(Some(key1), Future(envelop))) - - // then the result should have the same key, with the envelop hydrated. since there's no - // partitioning stages, the driver will just recurse until all the hydration is done, - // then output to the final topic - val expectedValue = - envelop.copy( - envelopId = 2L, - plan = mkPlan( - complete(hydrateTweet), - complete( - hydrateNotiTweet, - Some(outputTopic) - ) // only the last stage has the output topic - )) - - assert((Some(key1), expectedValue) == actual) - } - } - - test("multi-stage partition+hydration plan works") { - new Fixtures { - val driver = new EnrichmentDriver(Some(outputTopic), partitionTopic, hydrator, partitioner) - - // given a plan that repartition then hydrate twice - val plan = mkPlan(repartitionTweet, hydrateTweet, repartitionNotiTweet, hydrateNotiTweet) - var curEnvelop = EnrichmentEnvelop(1L, tweet1, plan) - var curKey = key1 - - // stage 1, partitioning on tweet should be correct - var actual = Await.result(driver.execute(Some(curKey), Future(curEnvelop))) - var expectedKey = curKey.copy(id = 1L) - var expectedValue = curEnvelop.copy( - plan = mkPlan( - complete(repartitionTweet, Some(partitionTopic)), - hydrateTweet, - repartitionNotiTweet, - hydrateNotiTweet)) - - assert((Some(expectedKey), expectedValue) == actual) - curEnvelop = actual._2 - curKey = actual._1.get - - // stage 2-3, hydrating on tweet should be correct - // and since the next stage after hydration is a repartition, it will does so correctly - actual = Await.result(driver.execute(Some(curKey), Future(curEnvelop))) - expectedKey = curKey.copy(id = 3) // repartition is done in stage 3 - expectedValue = curEnvelop.copy( - envelopId = 2L, // hydration is done in stage 2 - plan = mkPlan( - complete(repartitionTweet, Some(partitionTopic)), - complete(hydrateTweet), - complete(repartitionNotiTweet, Some(partitionTopic)), - hydrateNotiTweet) - ) - - assert((Some(expectedKey), expectedValue) == actual) - curEnvelop = actual._2 - curKey = actual._1.get - - // then finally, stage 4 would output to the final topic - actual = Await.result(driver.execute(Some(curKey), Future(curEnvelop))) - expectedKey = curKey // nothing's changed in the key - expectedValue = curEnvelop.copy( - envelopId = 4L, - plan = mkPlan( - complete(repartitionTweet, Some(partitionTopic)), - complete(hydrateTweet), - complete(repartitionNotiTweet, Some(partitionTopic)), - complete(hydrateNotiTweet, Some(outputTopic)) - ) - ) - - assert((Some(expectedKey), expectedValue) == actual) - } - } -} diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD.bazel b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD.bazel deleted file mode 100644 index 39ba06b0d..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -junit_tests( - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "featureswitches/dynmap/src/main/scala/com/twitter/dynmap:dynmap-core", - "featureswitches/dynmap/src/main/scala/com/twitter/dynmap/json:dynmap-json", - "finatra/inject/inject-core/src/test/scala/com/twitter/inject", - "graphql/thrift/src/main/thrift/com/twitter/graphql:graphql-scala", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql", - "util/util-core:scala", - ], -) diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD.docx new file mode 100644 index 000000000..a84859900 Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlSpecs.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlSpecs.docx new file mode 100644 index 000000000..fc7e14193 Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlSpecs.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlSpecs.scala b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlSpecs.scala deleted file mode 100644 index e7ebd27e6..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/graphql/GraphqlSpecs.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.unified_user_actions.enricher.graphql - -import com.twitter.dynmap.DynMap -import com.twitter.inject.Test -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import org.scalatest.matchers.should.Matchers - -class GraphqlSpecs extends Test with Matchers { - trait Fixtures { - val sampleError = """ - |{ - | "errors": [ - | { - | "message": "Some err msg!", - | "code": 366, - | "kind": "Validation", - | "name": "QueryViolationError", - | "source": "Client", - | "tracing": { - | "trace_id": "1234567890" - | } - | } - | ] - |}""".stripMargin - - val sampleValidRsp = - """ - |{ - | "data": { - | "tweet_result_by_rest_id": { - | "result": { - | "core": { - | "user": { - | "legacy": { - | "id_str": "12" - | } - | } - | } - | } - | } - | } - |} - |""".stripMargin - - val sampleValidRspExpected = Return( - Set(("data.tweet_result_by_rest_id.result.core.user.legacy.id_str", "12"))) - val sampleErrorExpected = Throw( - GraphqlRspErrors( - DynMap.from( - "errors" -> List( - Map( - "message" -> "Some err msg!", - "code" -> 366, - "kind" -> "Validation", - "name" -> "QueryViolationError", - "source" -> "Client", - "tracing" -> Map("trace_id" -> "1234567890") - ))))) - def toFlattened(testStr: String): Try[Set[(String, Any)]] = - GraphqlRspParser.toDynMap(testStr).map { dm => dm.valuesFlattened.toSet } - } - - test("Graphql Response Parser") { - new Fixtures { - toFlattened(sampleValidRsp) shouldBe sampleValidRspExpected - toFlattened(sampleError) shouldBe sampleErrorExpected - } - } -} diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD.bazel b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD.bazel deleted file mode 100644 index 607524d25..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -junit_tests( - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "finatra/inject/inject-core/src/test/scala:test-deps", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache", - "util/util-cache-guava/src/main/scala", - "util/util-cache/src/main/scala", - ], -) diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD.docx new file mode 100644 index 000000000..cb6e603cb Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCacheTest.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCacheTest.docx new file mode 100644 index 000000000..a1e98c9d0 Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCacheTest.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCacheTest.scala b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCacheTest.scala deleted file mode 100644 index bcf3d5fb6..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hcache/LocalCacheTest.scala +++ /dev/null @@ -1,153 +0,0 @@ -package com.twitter.unified_user_actions.enricher.hcache - -import com.google.common.cache.Cache -import com.google.common.cache.CacheBuilder -import com.twitter.finagle.stats.InMemoryStatsReceiver -import com.twitter.inject.Test -import com.twitter.util.Await -import com.twitter.util.Future -import com.twitter.util.Time -import java.util.concurrent.TimeUnit -import java.lang.{Integer => JInt} - -class LocalCacheTest extends Test { - - trait Fixture { - val time = Time.fromMilliseconds(123456L) - val ttl = 5 - val maxSize = 10 - - val underlying: Cache[JInt, Future[JInt]] = CacheBuilder - .newBuilder() - .expireAfterWrite(ttl, TimeUnit.SECONDS) - .maximumSize(maxSize) - .build[JInt, Future[JInt]]() - - val stats = new InMemoryStatsReceiver - - val cache = new LocalCache[JInt, JInt]( - underlying = underlying, - statsReceiver = stats - ) - - def getCounts(counterName: String*): Long = stats.counter(counterName: _*)() - } - - test("simple local cache works") { - new Fixture { - Time.withTimeAt(time) { _ => - assert(cache.size === 0) - - (1 to maxSize + 1).foreach { id => - cache.getOrElseUpdate(id)(Future.value(id)) - - val actual = Await.result(cache.get(id).get) - assert(actual === id) - } - assert(cache.size === maxSize) - - assert(getCounts("gets") === 2 * (maxSize + 1)) - assert(getCounts("hits") === maxSize + 1) - assert(getCounts("misses") === maxSize + 1) - assert(getCounts("sets", "evictions", "failed_futures") === 0) - - cache.reset() - assert(cache.size === 0) - } - } - } - - test("getOrElseUpdate successful futures") { - new Fixture { - Time.withTimeAt(time) { _ => - assert(cache.size === 0) - - (1 to maxSize + 1).foreach { _ => - cache.getOrElseUpdate(1) { - Future.value(1) - } - } - assert(cache.size === 1) - - assert(getCounts("gets") === maxSize + 1) - assert(getCounts("hits") === maxSize) - assert(getCounts("misses") === 1) - assert(getCounts("sets", "evictions", "failed_futures") === 0) - - cache.reset() - assert(cache.size === 0) - } - } - } - - test("getOrElseUpdate Failed Futures") { - new Fixture { - Time.withTimeAt(time) { _ => - assert(cache.size === 0) - - (1 to maxSize + 1).foreach { id => - cache.getOrElseUpdate(id)(Future.exception(new IllegalArgumentException(""))) - assert(cache.get(id).map { - Await.result(_) - } === None) - } - assert(cache.size === 0) - - assert(getCounts("gets") === 2 * (maxSize + 1)) - assert(getCounts("hits", "misses", "sets") === 0) - assert(getCounts("evictions") === maxSize + 1) - assert(getCounts("failed_futures") === maxSize + 1) - } - } - } - - test("Set successful Future") { - new Fixture { - Time.withTimeAt(time) { _ => - assert(cache.size === 0) - - cache.set(1, Future.value(2)) - assert(Await.result(cache.get(1).get) === 2) - assert(getCounts("gets") === 1) - assert(getCounts("hits") === 1) - assert(getCounts("misses") === 0) - assert(getCounts("sets") === 1) - assert(getCounts("evictions", "failed_futures") === 0) - } - } - } - - test("Evict") { - new Fixture { - Time.withTimeAt(time) { _ => - assert(cache.size === 0) - - // need to use reference here!!! - val f1 = Future.value(int2Integer(1)) - val f2 = Future.value(int2Integer(2)) - cache.set(1, f2) - cache.evict(1, f1) - cache.evict(1, f2) - assert(getCounts("gets", "hits", "misses") === 0) - assert(getCounts("sets") === 1) - assert(getCounts("evictions") === 1) // not 2 - assert(getCounts("failed_futures") === 0) - } - } - } - - test("Set Failed Futures") { - new Fixture { - Time.withTimeAt(time) { _ => - assert(cache.size === 0) - - cache.set(1, Future.exception(new IllegalArgumentException(""))) - assert(cache.size === 0) - - assert(getCounts("gets", "hits", "misses", "sets") === 0) - assert(getCounts("evictions") === 1) - assert(getCounts("failed_futures") === 1) - } - } - } -} diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD.bazel b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD.bazel deleted file mode 100644 index 1ff01e4c5..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -junit_tests( - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/mockito:mockito-core", - "3rdparty/jvm/org/mockito:mockito-scala", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "featureswitches/dynmap/src/main/scala/com/twitter/dynmap:dynmap-core", - "finatra/inject/inject-core/src/test/scala/com/twitter/inject", - "graphql/thrift/src/main/thrift/com/twitter/graphql:graphql-scala", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator:default", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator:noop", - "unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher:fixture", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD.docx new file mode 100644 index 000000000..d99d47311 Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydratorTest.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydratorTest.docx new file mode 100644 index 000000000..ee53a99bd Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydratorTest.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydratorTest.scala b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydratorTest.scala deleted file mode 100644 index 1e4477318..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/DefaultHydratorTest.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.unified_user_actions.enricher.hydrator - -import com.google.common.cache.CacheBuilder -import com.twitter.dynmap.DynMap -import com.twitter.graphql.thriftscala.GraphQlRequest -import com.twitter.graphql.thriftscala.GraphQlResponse -import com.twitter.graphql.thriftscala.GraphqlExecutionService -import com.twitter.inject.Test -import com.twitter.unified_user_actions.enricher.EnricherFixture -import com.twitter.unified_user_actions.enricher.FatalException -import com.twitter.unified_user_actions.enricher.hcache.LocalCache -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentIdType -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.unified_user_actions.thriftscala.AuthorInfo -import com.twitter.util.Await -import com.twitter.util.Future -import org.mockito.ArgumentMatchers -import org.mockito.MockitoSugar - -class DefaultHydratorTest extends Test with MockitoSugar { - - trait Fixtures extends EnricherFixture { - val cache = new LocalCache[EnrichmentKey, DynMap]( - underlying = CacheBuilder - .newBuilder() - .maximumSize(10) - .build[EnrichmentKey, Future[DynMap]]()) - - val client = mock[GraphqlExecutionService.FinagledClient] - val key = EnrichmentKey(EnrichmentIdType.TweetId, 1L) - val envelop = EnrichmentEnvelop(123L, mkUUATweetEvent(1L), tweetInfoEnrichmentPlan) - - def mkGraphQLResponse(authorId: Long): GraphQlResponse = - GraphQlResponse( - Some( - s""" - |{ - | "data": { - | "tweet_result_by_rest_id": { - | "result": { - | "core": { - | "user": { - | "legacy": { - | "id_str": "$authorId" - | } - | } - | } - | } - | } - | } - |} - |""".stripMargin - )) - } - - test("non-fatal errors should proceed as normal") { - new Fixtures { - val hydrator = new DefaultHydrator(cache, client) - - // when graphql client encounter any exception - when(client.graphql(ArgumentMatchers.any[GraphQlRequest])) - .thenReturn(Future.exception(new IllegalStateException("any exception"))) - - val actual = - Await.result(hydrator.hydrate(EnrichmentInstruction.TweetEnrichment, Some(key), envelop)) - - // then the original envelop is expected - assert(envelop == actual) - } - } - - test("fatal errors should return a future exception") { - new Fixtures { - val hydrator = new DefaultHydrator(cache, client) - - // when graphql client encounter a fatal exception - when(client.graphql(ArgumentMatchers.any[GraphQlRequest])) - .thenReturn(Future.exception(new FatalException("fatal exception") {})) - - val actual = hydrator.hydrate(EnrichmentInstruction.TweetEnrichment, Some(key), envelop) - - // then a failed future is expected - assertFailedFuture[FatalException](actual) - } - } - - test("author_id should be hydrated from graphql respond") { - new Fixtures { - val hydrator = new DefaultHydrator(cache, client) - - when(client.graphql(ArgumentMatchers.any[GraphQlRequest])) - .thenReturn(Future.value(mkGraphQLResponse(888L))) - - val actual = hydrator.hydrate(EnrichmentInstruction.TweetEnrichment, Some(key), envelop) - - assertFutureValue( - actual, - envelop.copy(uua = mkUUATweetEvent(1L, Some(AuthorInfo(Some(888L)))))) - } - } - - test("when AuthorInfo is populated, there should be no hydration") { - new Fixtures { - val hydrator = new DefaultHydrator(cache, client) - - when(client.graphql(ArgumentMatchers.any[GraphQlRequest])) - .thenReturn(Future.value(mkGraphQLResponse(333L))) - - val expected = envelop.copy(uua = - mkUUATweetEvent(tweetId = 3L, author = Some(AuthorInfo(authorId = Some(222))))) - val actual = hydrator.hydrate(EnrichmentInstruction.TweetEnrichment, Some(key), expected) - - assertFutureValue(actual, expected) - } - } -} diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydratorTest.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydratorTest.docx new file mode 100644 index 000000000..e7643c7ad Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydratorTest.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydratorTest.scala b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydratorTest.scala deleted file mode 100644 index 79c7af790..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/hydrator/NoopHydratorTest.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.unified_user_actions.enricher.hydrator - -import com.twitter.inject.Test -import com.twitter.unified_user_actions.enricher.ImplementationException - -class NoopHydratorTest extends Test { - test("noop hydrator should throw an error when used") { - assertThrows[ImplementationException] { - new NoopHydrator().hydrate(null, null, null) - } - } -} diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD.bazel b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD.bazel deleted file mode 100644 index ab9678af4..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -junit_tests( - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "finatra/inject/inject-core/src/test/scala/com/twitter/inject", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner:default", - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - "unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher:fixture", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD.docx new file mode 100644 index 000000000..1ac3043f3 Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/BUILD.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitionerTest.docx b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitionerTest.docx new file mode 100644 index 000000000..10dea4d2d Binary files /dev/null and b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitionerTest.docx differ diff --git a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitionerTest.scala b/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitionerTest.scala deleted file mode 100644 index 7b8f59cb4..000000000 --- a/unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher/partitioner/DefaultPartitionerTest.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.unified_user_actions.enricher.partitioner - -import com.twitter.inject.Test -import com.twitter.unified_user_actions.enricher.EnricherFixture -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentIdType -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction.NotificationTweetEnrichment -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction.TweetEnrichment -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.unified_user_actions.enricher.partitioner.DefaultPartitioner.NullKey -import org.scalatest.prop.TableDrivenPropertyChecks - -class DefaultPartitionerTest extends Test with TableDrivenPropertyChecks { - test("default partitioner should work") { - new EnricherFixture { - val partitioner = new DefaultPartitioner - - val instructions = Table( - ("instruction", "envelop", "expected"), - // tweet info - ( - TweetEnrichment, - EnrichmentEnvelop(1L, mkUUATweetEvent(123L), tweetInfoEnrichmentPlan), - Some(EnrichmentKey(EnrichmentIdType.TweetId, 123L))), - // notification tweet info - ( - NotificationTweetEnrichment, - EnrichmentEnvelop(2L, mkUUATweetNotificationEvent(234L), tweetNotificationEnrichmentPlan), - Some(EnrichmentKey(EnrichmentIdType.TweetId, 234L))), - // notification with multiple tweet info - ( - NotificationTweetEnrichment, - EnrichmentEnvelop( - 3L, - mkUUAMultiTweetNotificationEvent(22L, 33L), - tweetNotificationEnrichmentPlan), - Some(EnrichmentKey(EnrichmentIdType.TweetId, 22L)) - ) // only the first tweet id is partitioned - ) - - forEvery(instructions) { - ( - instruction: EnrichmentInstruction, - envelop: EnrichmentEnvelop, - expected: Some[EnrichmentKey] - ) => - val actual = partitioner.repartition(instruction, envelop) - assert(expected === actual) - } - } - } - - test("unsupported events shouldn't be partitioned") { - new EnricherFixture { - val partitioner = new DefaultPartitioner - - val instructions = Table( - ("instruction", "envelop", "expected"), - // profile uua event - ( - TweetEnrichment, - EnrichmentEnvelop(1L, mkUUAProfileEvent(111L), tweetInfoEnrichmentPlan), - NullKey), - // unknown notification (not a tweet) - ( - NotificationTweetEnrichment, - EnrichmentEnvelop(1L, mkUUATweetNotificationUnknownEvent(), tweetInfoEnrichmentPlan), - NullKey), - ) - - forEvery(instructions) { - ( - instruction: EnrichmentInstruction, - envelop: EnrichmentEnvelop, - expected: Option[EnrichmentKey] - ) => - val actual = partitioner.repartition(instruction, envelop) - assert(expected === actual) - } - } - } -} diff --git a/unified_user_actions/graphql/README.docx b/unified_user_actions/graphql/README.docx new file mode 100644 index 000000000..3c8e91383 Binary files /dev/null and b/unified_user_actions/graphql/README.docx differ diff --git a/unified_user_actions/graphql/README.md b/unified_user_actions/graphql/README.md deleted file mode 100644 index fbbf8006f..000000000 --- a/unified_user_actions/graphql/README.md +++ /dev/null @@ -1,15 +0,0 @@ -Documents -========= - -TweetHydration --------------- - -Upload ------- - -``` -$ graphql stored_document put unified_user_actions/graphql/TweetHydration.graphql -``` - -DocumentId: `M5sHxua-RDiRtTn48CAhng` -Test: https://graphql.twitter.com/snaptest/tests/1580340324727017472/ diff --git a/unified_user_actions/graphql/TweetHydration.docx b/unified_user_actions/graphql/TweetHydration.docx new file mode 100644 index 000000000..5a41c7e7d Binary files /dev/null and b/unified_user_actions/graphql/TweetHydration.docx differ diff --git a/unified_user_actions/graphql/TweetHydration.graphql b/unified_user_actions/graphql/TweetHydration.graphql deleted file mode 100644 index 604019d69..000000000 --- a/unified_user_actions/graphql/TweetHydration.graphql +++ /dev/null @@ -1,15 +0,0 @@ -query TweetHydration($rest_id: NumericString!) { - tweet_result_by_rest_id(rest_id: $rest_id, safety_level: ForDevelopmentOnly) { - result { - ... on Tweet { - core { - user { - legacy { - id_str - } - } - } - } - } - } -} diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/BUILD b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/BUILD deleted file mode 100644 index 1698385eb..000000000 --- a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["**/*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "3rdparty/jvm/org/apache/thrift:libthrift", - "kafka/finagle-kafka/finatra-kafka/src/main/java", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/headers", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "scrooge/scrooge-core/src/main/scala", - "scrooge/scrooge-serializer/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - "util/util-thrift", - ], -) diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/BUILD.docx b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/BUILD.docx new file mode 100644 index 000000000..d12a8c30a Binary files /dev/null and b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/BUILD.docx differ diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientConfigs.docx b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientConfigs.docx new file mode 100644 index 000000000..14f043025 Binary files /dev/null and b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientConfigs.docx differ diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientConfigs.scala b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientConfigs.scala deleted file mode 100644 index ffbe8f126..000000000 --- a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientConfigs.scala +++ /dev/null @@ -1,211 +0,0 @@ -package com.twitter.unified_user_actions.kafka - -import com.twitter.conversions.DurationOps._ -import com.twitter.conversions.StorageUnitOps._ -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import org.apache.kafka.common.record.CompressionType - -object ClientConfigs { - final val kafkaBootstrapServerConfig = "kafka.bootstrap.servers" - final val kafkaBootstrapServerHelp: String = - """Kafka servers list. It is usually a WilyNs name at Twitter - """.stripMargin - - final val kafkaBootstrapServerRemoteDestConfig = "kafka.bootstrap.servers.remote.dest" - final val kafkaBootstrapServerRemoteDestHelp: String = - """Destination Kafka servers, if the sink cluster is different from the source cluster, - |i.e., read from one cluster and output to another cluster - """.stripMargin - - final val kafkaApplicationIdConfig = "kafka.application.id" - final val kafkaApplicationIdHelp: String = - """An identifier for the Kafka application. Must be unique within the Kafka cluster - """.stripMargin - - // Processor in general - final val enableTrustStore = "kafka.trust.store.enable" - final val enableTrustStoreDefault = true - final val enableTrustStoreHelp = "Whether to enable trust store location" - - final val trustStoreLocationConfig = "kafka.trust.store.location" - final val trustStoreLocationDefault = "/etc/tw_truststore/messaging/kafka/client.truststore.jks" - final val trustStoreLocationHelp = "trust store location" - - final val kafkaMaxPendingRequestsConfig = "kafka.max.pending.requests" - final val kafkaMaxPendingRequestsHelp = "the maximum number of concurrent pending requests." - - final val kafkaWorkerThreadsConfig = "kafka.worker.threads" - final val kafkaWorkerThreadsHelp = - """This has meaning that is dependent on the value of {@link usePerPartitionThreadPool} - - | if that is false, this is the number of parallel worker threads that will execute the processor function. - | if that is true, this is the number of parallel worker threads for each partition. So the total number of - | threads will be {@link workerThreads} * number_of_partitions. - |""".stripMargin - - final val retriesConfig = "kafka.retries" - final val retriesDefault = 300 - final val retriesHelp: String = - """Setting a value greater than zero will cause the client to resend any request that fails - |with a potentially transient error - """.stripMargin - - final val retryBackoffConfig = "kafka.retry.backoff" - final val retryBackoffDefault: Duration = 1.seconds - final val retryBackoffHelp: String = - """The amount of time to wait before attempting to retry a failed request to a given topic - |partition. This avoids repeatedly sending requests in a tight loop under some failure - |scenarios - """.stripMargin - - // Kafka Producer - final val producerClientIdConfig = "kafka.producer.client.id" - final val producerClientIdHelp: String = - """The client id of the Kafka producer, required for producers. - """.stripMargin - - final val producerIdempotenceConfig = "kafka.producer.idempotence" - final val producerIdempotenceDefault: Boolean = false - final val producerIdempotenceHelp: String = - """"retries due to broker failures, etc., may write duplicates of the retried message in the - stream. Note that enabling idempotence requires - MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION to be less than or equal to 5, - RETRIES_CONFIG to be greater than 0 and ACKS_CONFIG - must be 'all'. If these values are not explicitly set by the user, suitable values will be - chosen. If incompatible values are set, a ConfigException will be thrown. - """.stripMargin - - final val producerBatchSizeConfig = "kafka.producer.batch.size" - final val producerBatchSizeDefault: StorageUnit = 512.kilobytes - final val producerBatchSizeHelp: String = - """The producer will attempt to batch records together into fewer requests whenever multiple - |records are being sent to the same partition. This helps performance on both the client and - |the server. This configuration controls the default batch size in bytes. - |No attempt will be made to batch records larger than this size. - |Requests sent to brokers will contain multiple batches, one for each partition with data - |available to be sent. A small batch size will make batching less common and may reduce - |throughput (a batch size of zero will disable batching entirely). - |A very large batch size may use memory a bit more wastefully as we will always allocate a - |buffer of the specified batch size in anticipation of additional records. - """.stripMargin - - final val producerBufferMemConfig = "kafka.producer.buffer.mem" - final val producerBufferMemDefault: StorageUnit = 256.megabytes - final val producerBufferMemHelp: String = - """The total bytes of memory the producer can use to buffer records waiting to be sent to the - |server. If records are sent faster than they can be delivered to the server the producer - |will block for MAX_BLOCK_MS_CONFIG after which it will throw an exception. - |This setting should correspond roughly to the total memory the producer will use, but is not - |a hard bound since not all memory the producer uses is used for buffering. - |Some additional memory will be used for compression (if compression is enabled) as well as - |for maintaining in-flight requests. - """.stripMargin - - final val producerLingerConfig = "kafka.producer.linger" - final val producerLingerDefault: Duration = 100.milliseconds - final val producerLingerHelp: String = - """The producer groups together any records that arrive in between request transmissions into - |a single batched request. "Normally this occurs only under load when records arrive faster - |than they can be sent out. However in some circumstances the client may want to reduce the - |number of requests even under moderate load. This setting accomplishes this by adding a - |small amount of artificial delay—that is, rather than immediately sending out a record - |the producer will wait for up to the given delay to allow other records to be sent so that - |the sends can be batched together. This can be thought of as analogous to Nagle's algorithm - |in TCP. This setting gives the upper bound on the delay for batching: once we get - |BATCH_SIZE_CONFIG worth of records for a partition it will be sent immediately regardless - |of this setting, however if we have fewer than this many bytes accumulated for this - |partition we will 'linger' for the specified time waiting for more records to show up. - |This setting defaults to 0 (i.e. no delay). Setting LINGER_MS_CONFIG=5, for example, - |would have the effect of reducing the number of requests sent but would add up to 5ms of - |latency to records sent in the absence of load. - """.stripMargin - - final val producerRequestTimeoutConfig = "kafka.producer.request.timeout" - final val producerRequestTimeoutDefault: Duration = 30.seconds - final val producerRequestTimeoutHelp: String = - """"The configuration controls the maximum amount of time the client will wait - |for the response of a request. If the response is not received before the timeout - |elapses the client will resend the request if necessary or fail the request if - |retries are exhausted. - """.stripMargin - - final val compressionConfig = "kafka.producer.compression.type" - final val compressionDefault: CompressionTypeFlag = CompressionTypeFlag(CompressionType.NONE) - final val compressionHelp = "Producer compression type" - - // Kafka Consumer - final val kafkaGroupIdConfig = "kafka.group.id" - final val kafkaGroupIdHelp: String = - """The group identifier for the Kafka consumer - """.stripMargin - - final val kafkaCommitIntervalConfig = "kafka.commit.interval" - final val kafkaCommitIntervalDefault: Duration = 10.seconds - final val kafkaCommitIntervalHelp: String = - """The frequency with which to save the position of the processor. - """.stripMargin - - final val consumerMaxPollRecordsConfig = "kafka.max.poll.records" - final val consumerMaxPollRecordsDefault: Int = 1000 - final val consumerMaxPollRecordsHelp: String = - """The maximum number of records returned in a single call to poll() - """.stripMargin - - final val consumerMaxPollIntervalConfig = "kafka.max.poll.interval" - final val consumerMaxPollIntervalDefault: Duration = 5.minutes - final val consumerMaxPollIntervalHelp: String = - """The maximum delay between invocations of poll() when using consumer group management. - This places an upper bound on the amount of time that the consumer can be idle before fetching more records. - If poll() is not called before expiration of this timeout, then the consumer is considered failed and the group - will rebalance in order to reassign the partitions to another member. - """.stripMargin - - final val consumerSessionTimeoutConfig = "kafka.session.timeout" - final val consumerSessionTimeoutDefault: Duration = 1.minute - final val consumerSessionTimeoutHelp: String = - """The timeout used to detect client failures when using Kafka's group management facility. - The client sends periodic heartbeats to indicate its liveness to the broker. - If no heartbeats are received by the broker before the expiration of this session timeout, then the broker - will remove this client from the group and initiate a rebalance. Note that the value must be in the allowable - range as configured in the broker configuration by group.min.session.timeout.ms and group.max.session.timeout.ms. - """.stripMargin - - final val consumerFetchMinConfig = "kafka.consumer.fetch.min" - final val consumerFetchMinDefault: StorageUnit = 1.kilobyte - final val consumerFetchMinHelp: String = - """The minimum amount of data the server should return for a fetch request. If insufficient - |data is available the request will wait for that much data to accumulate before answering - |the request. The default setting of 1 byte means that fetch requests are answered as soon - |as a single byte of data is available or the fetch request times out waiting for data to - |arrive. Setting this to something greater than 1 will cause the server to wait for larger - |amounts of data to accumulate which can improve server throughput a bit at the cost of - |some additional latency. - """.stripMargin - - final val consumerFetchMaxConfig = "kafka.consumer.fetch.max" - final val consumerFetchMaxDefault: StorageUnit = 1.megabytes - final val consumerFetchMaxHelp: String = - """The maximum amount of data the server should return for a fetch request. Records are - |fetched in batches by the consumer, and if the first record batch in the first non-empty - |partition of the fetch is larger than this value, the record batch will still be returned - |to ensure that the consumer can make progress. As such, this is not a absolute maximum. - |The maximum record batch size accepted by the broker is defined via message.max.bytes - |(broker config) or max.message.bytes (topic config). - |Note that the consumer performs multiple fetches in parallel. - """.stripMargin - - final val consumerReceiveBufferSizeConfig = "kafka.consumer.receive.buffer.size" - final val consumerReceiveBufferSizeDefault: StorageUnit = 1.megabytes - final val consumerReceiveBufferSizeHelp: String = - """The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. - |If the value is -1, the OS default will be used. - """.stripMargin - - final val consumerApiTimeoutConfig = "kafka.consumer.api.timeout" - final val consumerApiTimeoutDefault: Duration = 120.seconds - final val consumerApiTimeoutHelp: String = - """Specifies the timeout (in milliseconds) for consumer APIs that could block. - |This configuration is used as the default timeout for all consumer operations that do - |not explicitly accept a timeout parameter."; - """.stripMargin -} diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientProviders.docx b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientProviders.docx new file mode 100644 index 000000000..b6fff62fb Binary files /dev/null and b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientProviders.docx differ diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientProviders.scala b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientProviders.scala deleted file mode 100644 index b7fcac3c7..000000000 --- a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/ClientProviders.scala +++ /dev/null @@ -1,141 +0,0 @@ -package com.twitter.unified_user_actions.kafka - -import com.twitter.conversions.StorageUnitOps._ -import com.twitter.finatra.kafka.consumers.FinagleKafkaConsumerBuilder -import com.twitter.finatra.kafka.domain.AckMode -import com.twitter.finatra.kafka.domain.KafkaGroupId -import com.twitter.finatra.kafka.producers.BlockingFinagleKafkaProducer -import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder -import com.twitter.kafka.client.processor.ThreadSafeKafkaConsumerClient -import com.twitter.util.logging.Logging -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import org.apache.kafka.clients.CommonClientConfigs -import org.apache.kafka.clients.producer.ProducerConfig -import org.apache.kafka.common.config.SaslConfigs -import org.apache.kafka.common.config.SslConfigs -import org.apache.kafka.common.record.CompressionType -import org.apache.kafka.common.security.auth.SecurityProtocol -import org.apache.kafka.common.serialization.Deserializer -import org.apache.kafka.common.serialization.Serializer - -/** - * A Utility class mainly provides raw Kafka producer/consumer supports - */ -object ClientProviders extends Logging { - - /** - * Provide a Finagle-thread-safe-and-compatible Kafka consumer. - * For the params and their significance, please see [[ClientConfigs]] - */ - def mkConsumer[CK, CV]( - bootstrapServer: String, - keySerde: Deserializer[CK], - valueSerde: Deserializer[CV], - groupId: String, - autoCommit: Boolean = false, - maxPollRecords: Int = ClientConfigs.consumerMaxPollRecordsDefault, - maxPollInterval: Duration = ClientConfigs.consumerMaxPollIntervalDefault, - autoCommitInterval: Duration = ClientConfigs.kafkaCommitIntervalDefault, - sessionTimeout: Duration = ClientConfigs.consumerSessionTimeoutDefault, - fetchMax: StorageUnit = ClientConfigs.consumerFetchMaxDefault, - fetchMin: StorageUnit = ClientConfigs.consumerFetchMinDefault, - receiveBuffer: StorageUnit = ClientConfigs.consumerReceiveBufferSizeDefault, - trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault) - ): ThreadSafeKafkaConsumerClient[CK, CV] = { - val baseBuilder = - FinagleKafkaConsumerBuilder[CK, CV]() - .keyDeserializer(keySerde) - .valueDeserializer(valueSerde) - .dest(bootstrapServer) - .groupId(KafkaGroupId(groupId)) - .enableAutoCommit(autoCommit) - .maxPollRecords(maxPollRecords) - .maxPollInterval(maxPollInterval) - .autoCommitInterval(autoCommitInterval) - .receiveBuffer(receiveBuffer) - .sessionTimeout(sessionTimeout) - .fetchMax(fetchMax) - .fetchMin(fetchMin) - .withConfig( - CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, - SecurityProtocol.PLAINTEXT.toString) - - trustStoreLocationOpt - .map { trustStoreLocation => - new ThreadSafeKafkaConsumerClient[CK, CV]( - baseBuilder - .withConfig( - CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, - SecurityProtocol.SASL_SSL.toString) - .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, trustStoreLocation) - .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) - .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") - .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") - .config) - }.getOrElse { - new ThreadSafeKafkaConsumerClient[CK, CV]( - baseBuilder - .withConfig( - CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, - SecurityProtocol.PLAINTEXT.toString) - .config) - } - } - - /** - * Provide a Finagle-compatible Kafka producer. - * For the params and their significance, please see [[ClientConfigs]] - */ - def mkProducer[PK, PV]( - bootstrapServer: String, - keySerde: Serializer[PK], - valueSerde: Serializer[PV], - clientId: String, - idempotence: Boolean = ClientConfigs.producerIdempotenceDefault, - batchSize: StorageUnit = ClientConfigs.producerBatchSizeDefault, - linger: Duration = ClientConfigs.producerLingerDefault, - bufferMem: StorageUnit = ClientConfigs.producerBufferMemDefault, - compressionType: CompressionType = ClientConfigs.compressionDefault.compressionType, - retries: Int = ClientConfigs.retriesDefault, - retryBackoff: Duration = ClientConfigs.retryBackoffDefault, - requestTimeout: Duration = ClientConfigs.producerRequestTimeoutDefault, - trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault) - ): BlockingFinagleKafkaProducer[PK, PV] = { - val baseBuilder = FinagleKafkaProducerBuilder[PK, PV]() - .keySerializer(keySerde) - .valueSerializer(valueSerde) - .dest(bootstrapServer) - .clientId(clientId) - .batchSize(batchSize) - .linger(linger) - .bufferMemorySize(bufferMem) - .maxRequestSize(4.megabytes) - .compressionType(compressionType) - .enableIdempotence(idempotence) - .ackMode(AckMode.ALL) - .maxInFlightRequestsPerConnection(5) - .retries(retries) - .retryBackoff(retryBackoff) - .requestTimeout(requestTimeout) - .withConfig(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, requestTimeout + linger) - trustStoreLocationOpt - .map { trustStoreLocation => - baseBuilder - .withConfig( - CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, - SecurityProtocol.SASL_SSL.toString) - .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, trustStoreLocation) - .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) - .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") - .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") - .build() - }.getOrElse { - baseBuilder - .withConfig( - CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, - SecurityProtocol.PLAINTEXT.toString) - .build() - } - } -} diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/CompressionTypeFlag.docx b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/CompressionTypeFlag.docx new file mode 100644 index 000000000..034d49793 Binary files /dev/null and b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/CompressionTypeFlag.docx differ diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/CompressionTypeFlag.scala b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/CompressionTypeFlag.scala deleted file mode 100644 index 43f2bdb57..000000000 --- a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/CompressionTypeFlag.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.unified_user_actions.kafka - -import com.twitter.app.Flaggable -import org.apache.kafka.common.record.CompressionType - -case class CompressionTypeFlag(compressionType: CompressionType) - -object CompressionTypeFlag { - - def fromString(s: String): CompressionType = s.toLowerCase match { - case "lz4" => CompressionType.LZ4 - case "snappy" => CompressionType.SNAPPY - case "gzip" => CompressionType.GZIP - case "zstd" => CompressionType.ZSTD - case _ => CompressionType.NONE - } - - implicit val flaggable: Flaggable[CompressionTypeFlag] = - Flaggable.mandatory(s => CompressionTypeFlag(fromString(s))) -} diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdes.docx b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdes.docx new file mode 100644 index 000000000..6abf9dfdf Binary files /dev/null and b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdes.docx differ diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdes.scala b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdes.scala deleted file mode 100644 index 511a30386..000000000 --- a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdes.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2016 Fred Cecilia, Valentin Kasas, Olivier Girardot - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -//Derived from: https://github.com/aseigneurin/kafka-streams-scala -package com.twitter.unified_user_actions.kafka.serde - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finatra.kafka.serde.internal._ - -import com.twitter.unified_user_actions.kafka.serde.internal._ -import com.twitter.scrooge.ThriftStruct - -/** - * NullableScalaSerdes is pretty much the same as com.twitter.finatra.kafka.serde.ScalaSerdes - * The only difference is that for the deserializer it returns null instead of throwing exceptions. - * The caller can also provide a counter so that the number of corrupt/bad records can be counted. - */ -object NullableScalaSerdes { - - def Thrift[T <: ThriftStruct: Manifest]( - nullCounter: Counter = NullStatsReceiver.NullCounter - ): ThriftSerDe[T] = new ThriftSerDe[T](nullCounter = nullCounter) - - def CompactThrift[T <: ThriftStruct: Manifest]( - nullCounter: Counter = NullStatsReceiver.NullCounter - ): CompactThriftSerDe[T] = new CompactThriftSerDe[T](nullCounter = nullCounter) - - val Int = IntSerde - - val Long = LongSerde - - val Double = DoubleSerde -} diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/internal/thrift.docx b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/internal/thrift.docx new file mode 100644 index 000000000..9fef63af2 Binary files /dev/null and b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/internal/thrift.docx differ diff --git a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/internal/thrift.scala b/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/internal/thrift.scala deleted file mode 100644 index 167b2c8f0..000000000 --- a/unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka/serde/internal/thrift.scala +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright 2021 Twitter, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ -package com.twitter.unified_user_actions.kafka.serde.internal - -import com.google.common.util.concurrent.RateLimiter -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.NullStatsReceiver -import java.util -import com.twitter.scrooge.CompactThriftSerializer -import com.twitter.scrooge.ThriftStruct -import com.twitter.scrooge.ThriftStructCodec -import com.twitter.scrooge.ThriftStructSerializer -import org.apache.kafka.common.serialization.Deserializer -import org.apache.kafka.common.serialization.Serde -import org.apache.kafka.common.serialization.Serializer -import com.twitter.util.logging.Logging -import org.apache.thrift.protocol.TBinaryProtocol - -abstract class AbstractScroogeSerDe[T <: ThriftStruct: Manifest](nullCounter: Counter) - extends Serde[T] - with Logging { - - private val rateLimiter = RateLimiter.create(1.0) // at most 1 log message per second - - private def rateLimitedLogError(e: Exception): Unit = - if (rateLimiter.tryAcquire()) { - logger.error(e.getMessage, e) - } - - private[kafka] val thriftStructSerializer: ThriftStructSerializer[T] = { - val clazz = manifest.runtimeClass.asInstanceOf[Class[T]] - val codec = ThriftStructCodec.forStructClass(clazz) - - constructThriftStructSerializer(clazz, codec) - } - - private val _deserializer = new Deserializer[T] { - override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {} - - override def close(): Unit = {} - - override def deserialize(topic: String, data: Array[Byte]): T = { - if (data == null) { - null.asInstanceOf[T] - } else { - try { - thriftStructSerializer.fromBytes(data) - } catch { - case e: Exception => - nullCounter.incr() - rateLimitedLogError(e) - null.asInstanceOf[T] - } - } - } - } - - private val _serializer = new Serializer[T] { - override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {} - - override def serialize(topic: String, data: T): Array[Byte] = { - if (data == null) { - null - } else { - thriftStructSerializer.toBytes(data) - } - } - - override def close(): Unit = {} - } - - /* Public */ - - override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {} - - override def close(): Unit = {} - - override def deserializer: Deserializer[T] = { - _deserializer - } - - override def serializer: Serializer[T] = { - _serializer - } - - /** - * Subclasses should implement this method and provide a concrete ThriftStructSerializer - */ - protected[this] def constructThriftStructSerializer( - thriftStructClass: Class[T], - thriftStructCodec: ThriftStructCodec[T] - ): ThriftStructSerializer[T] -} - -class ThriftSerDe[T <: ThriftStruct: Manifest](nullCounter: Counter = NullStatsReceiver.NullCounter) - extends AbstractScroogeSerDe[T](nullCounter = nullCounter) { - protected[this] override def constructThriftStructSerializer( - thriftStructClass: Class[T], - thriftStructCodec: ThriftStructCodec[T] - ): ThriftStructSerializer[T] = { - new ThriftStructSerializer[T] { - override val protocolFactory = new TBinaryProtocol.Factory - override def codec: ThriftStructCodec[T] = thriftStructCodec - } - } -} - -class CompactThriftSerDe[T <: ThriftStruct: Manifest]( - nullCounter: Counter = NullStatsReceiver.NullCounter) - extends AbstractScroogeSerDe[T](nullCounter = nullCounter) { - override protected[this] def constructThriftStructSerializer( - thriftStructClass: Class[T], - thriftStructCodec: ThriftStructCodec[T] - ): ThriftStructSerializer[T] = { - new CompactThriftSerializer[T] { - override def codec: ThriftStructCodec[T] = thriftStructCodec - } - } -} diff --git a/unified_user_actions/kafka/src/test/resources/BUILD.bazel b/unified_user_actions/kafka/src/test/resources/BUILD.bazel deleted file mode 100644 index 515a45887..000000000 --- a/unified_user_actions/kafka/src/test/resources/BUILD.bazel +++ /dev/null @@ -1,4 +0,0 @@ -resources( - sources = ["*.xml"], - tags = ["bazel-compatible"], -) diff --git a/unified_user_actions/kafka/src/test/resources/BUILD.docx b/unified_user_actions/kafka/src/test/resources/BUILD.docx new file mode 100644 index 000000000..0fa3e7ebd Binary files /dev/null and b/unified_user_actions/kafka/src/test/resources/BUILD.docx differ diff --git a/unified_user_actions/kafka/src/test/resources/logback-test.docx b/unified_user_actions/kafka/src/test/resources/logback-test.docx new file mode 100644 index 000000000..41215ad71 Binary files /dev/null and b/unified_user_actions/kafka/src/test/resources/logback-test.docx differ diff --git a/unified_user_actions/kafka/src/test/resources/logback-test.xml b/unified_user_actions/kafka/src/test/resources/logback-test.xml deleted file mode 100644 index 3544a0909..000000000 --- a/unified_user_actions/kafka/src/test/resources/logback-test.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - - - - - - - - - - - - - - - - - - - - - - diff --git a/unified_user_actions/kafka/src/test/scala/BUILD.bazel b/unified_user_actions/kafka/src/test/scala/BUILD.bazel deleted file mode 100644 index 3ae26e5a9..000000000 --- a/unified_user_actions/kafka/src/test/scala/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -junit_tests( - sources = ["**/*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "finatra/inject/inject-core/src/test/scala:test-deps", - "kafka/finagle-kafka/finatra-kafka/src/test/scala:test-deps", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/kafka/src/test/resources", - "unified_user_actions/thrift/src/test/thrift/com/twitter/unified_user_actions:unified_user_actions_spec-scala", - ], -) diff --git a/unified_user_actions/kafka/src/test/scala/BUILD.docx b/unified_user_actions/kafka/src/test/scala/BUILD.docx new file mode 100644 index 000000000..8e1ed9a29 Binary files /dev/null and b/unified_user_actions/kafka/src/test/scala/BUILD.docx differ diff --git a/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdesSpec.docx b/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdesSpec.docx new file mode 100644 index 000000000..16f9cc40e Binary files /dev/null and b/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdesSpec.docx differ diff --git a/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdesSpec.scala b/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdesSpec.scala deleted file mode 100644 index 3c8d9d793..000000000 --- a/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/NullableScalaSerdesSpec.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.unified_user_actions.kafka.serde - -import com.twitter.finagle.stats.InMemoryStatsReceiver -import com.twitter.inject.Test -import com.twitter.unified_user_actions.thriftscala._ - -class NullableScalaSerdesSpec extends Test { - val counter = (new InMemoryStatsReceiver).counter("nullCounts") - val nullableDeserializer = NullableScalaSerdes.Thrift[UnifiedUserActionSpec](counter).deserializer - val serializer = NullableScalaSerdes.Thrift[UnifiedUserActionSpec]().serializer - val uua = UnifiedUserActionSpec( - userId = 1L, - payload = Some("test"), - ) - - test("serde") { - nullableDeserializer.deserialize("", serializer.serialize("", uua)) should be(uua) - nullableDeserializer.deserialize("", "Whatever".getBytes) should be( - null.asInstanceOf[UnifiedUserActionSpec]) - counter.apply() should equal(1) - } - - test("rate limited logger when there's an exception") { - for (_ <- 1 to 10) { - nullableDeserializer.deserialize("", "Whatever".getBytes) should be( - null.asInstanceOf[UnifiedUserActionSpec]) - } - - TestLogAppender.events.size should (be(1) or be(2)) - counter.apply() should equal(11) - } -} diff --git a/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/TestLogAppender.docx b/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/TestLogAppender.docx new file mode 100644 index 000000000..1eaef629f Binary files /dev/null and b/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/TestLogAppender.docx differ diff --git a/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/TestLogAppender.scala b/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/TestLogAppender.scala deleted file mode 100644 index 454c6c14e..000000000 --- a/unified_user_actions/kafka/src/test/scala/com/twitter/unified_user_actions/kafka/serde/TestLogAppender.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.unified_user_actions.kafka.serde - -import ch.qos.logback.classic.spi.ILoggingEvent -import ch.qos.logback.core.AppenderBase -import scala.collection.mutable.ArrayBuffer - -class TestLogAppender extends AppenderBase[ILoggingEvent] { - import TestLogAppender._ - - override def append(eventObject: ILoggingEvent): Unit = - recordLog(eventObject) -} - -object TestLogAppender { - val events: ArrayBuffer[ILoggingEvent] = ArrayBuffer() - - def recordLog(event: ILoggingEvent): Unit = - events += event -} diff --git a/unified_user_actions/scripts/kill_staging.docx b/unified_user_actions/scripts/kill_staging.docx new file mode 100644 index 000000000..2d53e3bc9 Binary files /dev/null and b/unified_user_actions/scripts/kill_staging.docx differ diff --git a/unified_user_actions/scripts/kill_staging.sh b/unified_user_actions/scripts/kill_staging.sh deleted file mode 100755 index d1376fefc..000000000 --- a/unified_user_actions/scripts/kill_staging.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -ex - -service_account="discode" -env="staging" -dcs=("pdxa") -services=("uua-tls-favs" "uua-client-event" "uua-bce" "uua-tweetypie-event" "uua-social-graph" "uua-email-notification-event" "uua-user-modification" "uua-ads-callback-engagements" "uua-favorite-archival-events" "uua-retweet-archival-events" "rekey-uua" "rekey-uua-iesource") -for dc in "${dcs[@]}"; do - for service in "${services[@]}"; do - aurora job killall --no-batch "$dc/$service_account/$env/$service" - done -done diff --git a/unified_user_actions/service/deploy/kill-staging-services.docx b/unified_user_actions/service/deploy/kill-staging-services.docx new file mode 100644 index 000000000..ecfb129d8 Binary files /dev/null and b/unified_user_actions/service/deploy/kill-staging-services.docx differ diff --git a/unified_user_actions/service/deploy/kill-staging-services.workflow b/unified_user_actions/service/deploy/kill-staging-services.workflow deleted file mode 100644 index 389db1bf3..000000000 --- a/unified_user_actions/service/deploy/kill-staging-services.workflow +++ /dev/null @@ -1,46 +0,0 @@ -{ - "role": "discode", - "name": "uua-kill-staging-services", - "config-files": [], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 1" - }, - "dependencies": [], - "steps": [] - }, - "targets": [ - { - "type": "script", - "name": "uua-kill-staging-services", - "keytab": "/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab", - "repository": "source", - "command": "bash unified_user_actions/scripts/kill_staging.sh", - "dependencies": [{ - "version": "latest", - "role": "aurora", - "name": "aurora" - }], - "timeout": "10.minutes" - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "unified_user_actions_dev" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "unified_user_actions_dev" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/rekey-uua-iesource-prod.docx b/unified_user_actions/service/deploy/rekey-uua-iesource-prod.docx new file mode 100644 index 000000000..0c9771089 Binary files /dev/null and b/unified_user_actions/service/deploy/rekey-uua-iesource-prod.docx differ diff --git a/unified_user_actions/service/deploy/rekey-uua-iesource-prod.workflow b/unified_user_actions/service/deploy/rekey-uua-iesource-prod.workflow deleted file mode 100644 index 71961c118..000000000 --- a/unified_user_actions/service/deploy/rekey-uua-iesource-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "rekey-uua-iesource-prod", - "config-files": [ - "rekey-uua-iesource.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:rekey-uua-iesource" - }, - { - "type": "packer", - "name": "rekey-uua-iesource", - "artifact": "./dist/rekey-uua-iesource.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "rekey-uua-iesource-prod-atla", - "key": "atla/discode/prod/rekey-uua-iesource" - }, - { - "name": "rekey-uua-iesource-prod-pdxa", - "key": "pdxa/discode/prod/rekey-uua-iesource" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/rekey-uua-iesource-staging.docx b/unified_user_actions/service/deploy/rekey-uua-iesource-staging.docx new file mode 100644 index 000000000..7c2003707 Binary files /dev/null and b/unified_user_actions/service/deploy/rekey-uua-iesource-staging.docx differ diff --git a/unified_user_actions/service/deploy/rekey-uua-iesource-staging.workflow b/unified_user_actions/service/deploy/rekey-uua-iesource-staging.workflow deleted file mode 100644 index 8de5cc73a..000000000 --- a/unified_user_actions/service/deploy/rekey-uua-iesource-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "rekey-uua-iesource-staging", - "config-files": [ - "rekey-uua-iesource.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:rekey-uua-iesource" - }, - { - "type": "packer", - "name": "rekey-uua-iesource-staging", - "artifact": "./dist/rekey-uua-iesource.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "rekey-uua-iesource-staging-pdxa", - "key": "pdxa/discode/staging/rekey-uua-iesource" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/rekey-uua-iesource.aurora b/unified_user_actions/service/deploy/rekey-uua-iesource.aurora deleted file mode 100644 index fcfd4cfd6..000000000 --- a/unified_user_actions/service/deploy/rekey-uua-iesource.aurora +++ /dev/null @@ -1,204 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'rekey-uua-iesource' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 250) - kafka_bootstrap_servers = Default(String, '/s/kafka/cdm-1:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'interaction_events') - sink_topics = Default(String, 'uua_keyed') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = DISK_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}-{{cluster}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=128.megabytes' - ' -kafka.producer.linger=50.milliseconds' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 500, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/srv#/devel/local/kafka/ingestion-1:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'DEBUG', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) - -### pdxa right now doesn't have InteractionEvents topic -PRODUCTION_PDXA = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml', - kafka_bootstrap_servers = '/srv#/prod/atla/kafka/cdm-1:kafka-tls' -) - -STAGING_PDXA = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers = '/srv#/prod/atla/kafka/cdm-1:kafka-tls', - kafka_bootstrap_servers_remote_dest = '/srv#/devel/local/kafka/ingestion-1:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL_PDXA = STAGING( - log_level = 'DEBUG', - kafka_bootstrap_servers = '/srv#/prod/atla/kafka/cdm-1:kafka-tls' -) - -prod_job_pdxa = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION_PDXA) - -staging_job_pdxa = job_template( - environment = 'staging' -).bind(profile = STAGING_PDXA) - -devel_job_pdxa = job_template( - environment = 'devel' -).bind(profile = DEVEL_PDXA) - -jobs.append(prod_job_pdxa(cluster = 'pdxa')) -jobs.append(staging_job_pdxa(cluster = 'pdxa')) -jobs.append(devel_job_pdxa(cluster = 'pdxa')) diff --git a/unified_user_actions/service/deploy/rekey-uua-iesource.docx b/unified_user_actions/service/deploy/rekey-uua-iesource.docx new file mode 100644 index 000000000..ad9cbfc66 Binary files /dev/null and b/unified_user_actions/service/deploy/rekey-uua-iesource.docx differ diff --git a/unified_user_actions/service/deploy/rekey-uua-prod.docx b/unified_user_actions/service/deploy/rekey-uua-prod.docx new file mode 100644 index 000000000..7294592d8 Binary files /dev/null and b/unified_user_actions/service/deploy/rekey-uua-prod.docx differ diff --git a/unified_user_actions/service/deploy/rekey-uua-prod.workflow b/unified_user_actions/service/deploy/rekey-uua-prod.workflow deleted file mode 100644 index b0a881e68..000000000 --- a/unified_user_actions/service/deploy/rekey-uua-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "rekey-uua-prod", - "config-files": [ - "rekey-uua.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:rekey-uua" - }, - { - "type": "packer", - "name": "rekey-uua", - "artifact": "./dist/rekey-uua.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "rekey-uua-prod-atla", - "key": "atla/discode/prod/rekey-uua" - }, - { - "name": "rekey-uua-prod-pdxa", - "key": "pdxa/discode/prod/rekey-uua" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/rekey-uua-staging.docx b/unified_user_actions/service/deploy/rekey-uua-staging.docx new file mode 100644 index 000000000..9df633758 Binary files /dev/null and b/unified_user_actions/service/deploy/rekey-uua-staging.docx differ diff --git a/unified_user_actions/service/deploy/rekey-uua-staging.workflow b/unified_user_actions/service/deploy/rekey-uua-staging.workflow deleted file mode 100644 index 70fe64489..000000000 --- a/unified_user_actions/service/deploy/rekey-uua-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "rekey-uua-staging", - "config-files": [ - "rekey-uua.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:rekey-uua" - }, - { - "type": "packer", - "name": "rekey-uua-staging", - "artifact": "./dist/rekey-uua.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "rekey-uua-staging-pdxa", - "key": "pdxa/discode/staging/rekey-uua" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/rekey-uua.aurora b/unified_user_actions/service/deploy/rekey-uua.aurora deleted file mode 100644 index bd64be350..000000000 --- a/unified_user_actions/service/deploy/rekey-uua.aurora +++ /dev/null @@ -1,167 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'rekey-uua' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 100) - kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'unified_user_actions') - sink_topics = Default(String, 'uua_keyed') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = DISK_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=128.megabytes' - ' -kafka.producer.linger=50.milliseconds' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 100, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/srv#/devel/local/kafka/ingestion-1:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'DEBUG', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/rekey-uua.docx b/unified_user_actions/service/deploy/rekey-uua.docx new file mode 100644 index 000000000..1ee131c16 Binary files /dev/null and b/unified_user_actions/service/deploy/rekey-uua.docx differ diff --git a/unified_user_actions/service/deploy/uua-ads-callback-engagements-prod.docx b/unified_user_actions/service/deploy/uua-ads-callback-engagements-prod.docx new file mode 100644 index 000000000..14df5fcbd Binary files /dev/null and b/unified_user_actions/service/deploy/uua-ads-callback-engagements-prod.docx differ diff --git a/unified_user_actions/service/deploy/uua-ads-callback-engagements-prod.workflow b/unified_user_actions/service/deploy/uua-ads-callback-engagements-prod.workflow deleted file mode 100644 index 857e42e6e..000000000 --- a/unified_user_actions/service/deploy/uua-ads-callback-engagements-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "uua-ads-callback-engagements-prod", - "config-files": [ - "uua-ads-callback-engagements.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-ads-callback-engagements" - }, - { - "type": "packer", - "name": "uua-ads-callback-engagements", - "artifact": "./dist/uua-ads-callback-engagements.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "uua-ads-callback-engagements-prod-atla", - "key": "atla/discode/prod/uua-ads-callback-engagements" - }, - { - "name": "uua-ads-callback-engagements-prod-pdxa", - "key": "pdxa/discode/prod/uua-ads-callback-engagements" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-ads-callback-engagements-staging.docx b/unified_user_actions/service/deploy/uua-ads-callback-engagements-staging.docx new file mode 100644 index 000000000..617071a1e Binary files /dev/null and b/unified_user_actions/service/deploy/uua-ads-callback-engagements-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-ads-callback-engagements-staging.workflow b/unified_user_actions/service/deploy/uua-ads-callback-engagements-staging.workflow deleted file mode 100644 index 3f7949bf5..000000000 --- a/unified_user_actions/service/deploy/uua-ads-callback-engagements-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-ads-callback-engagements-staging", - "config-files": [ - "uua-ads-callback-engagements.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-ads-callback-engagements" - }, - { - "type": "packer", - "name": "uua-ads-callback-engagements-staging", - "artifact": "./dist/uua-ads-callback-engagements.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-ads-callback-engagements-staging-pdxa", - "key": "pdxa/discode/staging/uua-ads-callback-engagements" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-ads-callback-engagements.aurora b/unified_user_actions/service/deploy/uua-ads-callback-engagements.aurora deleted file mode 100644 index 5b11177c3..000000000 --- a/unified_user_actions/service/deploy/uua-ads-callback-engagements.aurora +++ /dev/null @@ -1,167 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-ads-callback-engagements' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 50) - kafka_bootstrap_servers = Default(String, '/s/kafka/ads-callback-1:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'ads_spend_prod') - sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = DISK_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=128.megabytes' - ' -kafka.producer.linger=50.milliseconds' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'DEBUG', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-ads-callback-engagements.docx b/unified_user_actions/service/deploy/uua-ads-callback-engagements.docx new file mode 100644 index 000000000..39f2122bb Binary files /dev/null and b/unified_user_actions/service/deploy/uua-ads-callback-engagements.docx differ diff --git a/unified_user_actions/service/deploy/uua-client-event-prod.docx b/unified_user_actions/service/deploy/uua-client-event-prod.docx new file mode 100644 index 000000000..b42174754 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-client-event-prod.docx differ diff --git a/unified_user_actions/service/deploy/uua-client-event-prod.workflow b/unified_user_actions/service/deploy/uua-client-event-prod.workflow deleted file mode 100644 index 33a7a3983..000000000 --- a/unified_user_actions/service/deploy/uua-client-event-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "uua-client-event-prod", - "config-files": [ - "uua-client-event.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-client-event" - }, - { - "type": "packer", - "name": "uua-client-event", - "artifact": "./dist/uua-client-event.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "uua-client-event-prod-atla", - "key": "atla/discode/prod/uua-client-event" - }, - { - "name": "uua-client-event-prod-pdxa", - "key": "pdxa/discode/prod/uua-client-event" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-client-event-staging.docx b/unified_user_actions/service/deploy/uua-client-event-staging.docx new file mode 100644 index 000000000..e893592f8 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-client-event-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-client-event-staging.workflow b/unified_user_actions/service/deploy/uua-client-event-staging.workflow deleted file mode 100644 index 375c10341..000000000 --- a/unified_user_actions/service/deploy/uua-client-event-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-client-event-staging", - "config-files": [ - "uua-client-event.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-client-event" - }, - { - "type": "packer", - "name": "uua-client-event-staging", - "artifact": "./dist/uua-client-event.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-client-event-staging-pdxa", - "key": "pdxa/discode/staging/uua-client-event" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-client-event.aurora b/unified_user_actions/service/deploy/uua-client-event.aurora deleted file mode 100644 index 50bc06d48..000000000 --- a/unified_user_actions/service/deploy/uua-client-event.aurora +++ /dev/null @@ -1,174 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-client-event' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 1000) - kafka_bootstrap_servers = Default(String, '/s/kafka/client-events:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'client_event') - sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = DISK_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:MaxMetaspaceSize=536870912' - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - # CE events is about 0.4-0.6kb per message on the consumer side. A fetch size of 6~18 MB get us - # about 10k ~ 20k of messages per batch. This fits the size of our pending requests queue and - # within the limit of the max poll records. - ' -kafka.consumer.fetch.max=9.megabytes' - ' -kafka.consumer.fetch.min=3.megabytes' - ' -kafka.max.poll.records=40000' - ' -kafka.commit.interval=20.seconds' - ' -kafka.producer.batch.size=4.megabytes' - ' -kafka.producer.buffer.mem=64.megabytes' - ' -kafka.producer.linger=100.millisecond' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=4' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 1000, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'INFO', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-client-event.docx b/unified_user_actions/service/deploy/uua-client-event.docx new file mode 100644 index 000000000..7265b66f9 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-client-event.docx differ diff --git a/unified_user_actions/service/deploy/uua-email-notification-event-prod.docx b/unified_user_actions/service/deploy/uua-email-notification-event-prod.docx new file mode 100644 index 000000000..a00e490a2 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-email-notification-event-prod.docx differ diff --git a/unified_user_actions/service/deploy/uua-email-notification-event-prod.workflow b/unified_user_actions/service/deploy/uua-email-notification-event-prod.workflow deleted file mode 100644 index c25d8b0df..000000000 --- a/unified_user_actions/service/deploy/uua-email-notification-event-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "uua-email-notification-event-prod", - "config-files": [ - "uua-email-notification-event.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-email-notification-event" - }, - { - "type": "packer", - "name": "uua-email-notification-event", - "artifact": "./dist/uua-email-notification-event.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "uua-email-notification-event-prod-atla", - "key": "atla/discode/prod/uua-email-notification-event" - }, - { - "name": "uua-email-notification-event-prod-pdxa", - "key": "pdxa/discode/prod/uua-email-notification-event" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-email-notification-event-staging.docx b/unified_user_actions/service/deploy/uua-email-notification-event-staging.docx new file mode 100644 index 000000000..5d3bf894e Binary files /dev/null and b/unified_user_actions/service/deploy/uua-email-notification-event-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-email-notification-event-staging.workflow b/unified_user_actions/service/deploy/uua-email-notification-event-staging.workflow deleted file mode 100644 index 73e62dd3a..000000000 --- a/unified_user_actions/service/deploy/uua-email-notification-event-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-email-notification-event-staging", - "config-files": [ - "uua-email-notification-event.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-email-notification-event" - }, - { - "type": "packer", - "name": "uua-email-notification-event-staging", - "artifact": "./dist/uua-email-notification-event.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-email-notification-event-staging-pdxa", - "key": "pdxa/discode/staging/uua-email-notification-event" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-email-notification-event.aurora b/unified_user_actions/service/deploy/uua-email-notification-event.aurora deleted file mode 100644 index 83dcced60..000000000 --- a/unified_user_actions/service/deploy/uua-email-notification-event.aurora +++ /dev/null @@ -1,169 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-email-notification-event' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 20) - kafka_bootstrap_servers = Default(String, '/s/kafka/main-2:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'notifications') - sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = RAM_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.max.poll.records=20000' - ' -kafka.commit.interval=10.seconds' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=64.megabytes' - ' -kafka.producer.linger=0.milliseconds' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'INFO', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-email-notification-event.docx b/unified_user_actions/service/deploy/uua-email-notification-event.docx new file mode 100644 index 000000000..9aef6e858 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-email-notification-event.docx differ diff --git a/unified_user_actions/service/deploy/uua-enricher-staging.docx b/unified_user_actions/service/deploy/uua-enricher-staging.docx new file mode 100644 index 000000000..247b3fd37 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-enricher-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-enricher-staging.workflow b/unified_user_actions/service/deploy/uua-enricher-staging.workflow deleted file mode 100644 index 814708b47..000000000 --- a/unified_user_actions/service/deploy/uua-enricher-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-enricher-staging", - "config-files": [ - "uua-enricher.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-enricher" - }, - { - "type": "packer", - "name": "uua-enricher-staging", - "artifact": "./dist/uua-enricher.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-enricher-staging-pdxa", - "key": "pdxa/discode/staging/uua-enricher" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-enricher.aurora b/unified_user_actions/service/deploy/uua-enricher.aurora deleted file mode 100644 index e962f6885..000000000 --- a/unified_user_actions/service/deploy/uua-enricher.aurora +++ /dev/null @@ -1,151 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-enricher' - -CPU_NUM = 3 -HEAP_SIZE = 6 * GB -RAM_SIZE = 8 * GB -DISK_SIZE = 3 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 10) - kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = DISK_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.application.id={{name}}.{{environment}}' - ' -kafka.application.num.instances={{instances}}' # Used for static partitioning - ' -kafka.application.server={{mesos.instance}}.{{name}}.{{environment}}.{{role}}.service.{{cluster}}.twitter.com:80' - ' -com.twitter.finatra.kafkastreams.config.principal={{role}}' - ' -thrift.client.id={{name}}.{{environment}}' - ' -service.identifier="{{role}}:{{name}}:{{environment}}:{{cluster}}"' - ' -local.cache.ttl.seconds=86400' - ' -local.cache.max.size=400000000' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers = '/s/kafka/custdevel:kafka-tls' -) - -DEVEL = STAGING( - log_level = 'DEBUG', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-enricher.docx b/unified_user_actions/service/deploy/uua-enricher.docx new file mode 100644 index 000000000..ca2f10cef Binary files /dev/null and b/unified_user_actions/service/deploy/uua-enricher.docx differ diff --git a/unified_user_actions/service/deploy/uua-enrichment-planner-staging.docx b/unified_user_actions/service/deploy/uua-enrichment-planner-staging.docx new file mode 100644 index 000000000..57f56e132 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-enrichment-planner-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-enrichment-planner-staging.workflow b/unified_user_actions/service/deploy/uua-enrichment-planner-staging.workflow deleted file mode 100644 index c3ae6bcab..000000000 --- a/unified_user_actions/service/deploy/uua-enrichment-planner-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-enrichment-planner-staging", - "config-files": [ - "uua-enrichment-planner.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-enrichment-planner" - }, - { - "type": "packer", - "name": "uua-enrichment-planner-staging", - "artifact": "./dist/uua-enrichment-planner.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-enricher-enrichment-planner-pdxa", - "key": "pdxa/discode/staging/uua-enrichment-planner" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-enrichment-planner.aurora b/unified_user_actions/service/deploy/uua-enrichment-planner.aurora deleted file mode 100644 index c93d6f344..000000000 --- a/unified_user_actions/service/deploy/uua-enrichment-planner.aurora +++ /dev/null @@ -1,156 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-enrichment-planner' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 50) - kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls') - kafka_output_server = Default(String, '/s/kafka/bluebird-1:kafka-tls') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = DISK_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version(default_version='live') -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.output.server={{profile.kafka_output_server}}' - ' -kafka.application.id=uua-enrichment-planner' - ' -com.twitter.finatra.kafkastreams.config.principal={{role}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_output_server = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'DEBUG', - instances = 2, - kafka_output_server = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-enrichment-planner.docx b/unified_user_actions/service/deploy/uua-enrichment-planner.docx new file mode 100644 index 000000000..eff053358 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-enrichment-planner.docx differ diff --git a/unified_user_actions/service/deploy/uua-favorite-archival-events-prod.docx b/unified_user_actions/service/deploy/uua-favorite-archival-events-prod.docx new file mode 100644 index 000000000..b7db31372 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-favorite-archival-events-prod.docx differ diff --git a/unified_user_actions/service/deploy/uua-favorite-archival-events-prod.workflow b/unified_user_actions/service/deploy/uua-favorite-archival-events-prod.workflow deleted file mode 100644 index 75484576d..000000000 --- a/unified_user_actions/service/deploy/uua-favorite-archival-events-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "uua-favorite-archival-events-prod", - "config-files": [ - "uua-favorite-archival-events.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-favorite-archival-events" - }, - { - "type": "packer", - "name": "uua-favorite-archival-events", - "artifact": "./dist/uua-favorite-archival-events.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "uua-favorite-archival-events-prod-atla", - "key": "atla/discode/prod/uua-favorite-archival-events" - }, - { - "name": "uua-favorite-archival-events-prod-pdxa", - "key": "pdxa/discode/prod/uua-favorite-archival-events" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-favorite-archival-events-staging.docx b/unified_user_actions/service/deploy/uua-favorite-archival-events-staging.docx new file mode 100644 index 000000000..b4dd934a4 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-favorite-archival-events-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-favorite-archival-events-staging.workflow b/unified_user_actions/service/deploy/uua-favorite-archival-events-staging.workflow deleted file mode 100644 index 5954dd152..000000000 --- a/unified_user_actions/service/deploy/uua-favorite-archival-events-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-favorite-archival-events-staging", - "config-files": [ - "uua-favorite-archival-events.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-favorite-archival-events" - }, - { - "type": "packer", - "name": "uua-favorite-archival-events-staging", - "artifact": "./dist/uua-favorite-archival-events.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-favorite-archival-events-staging-pdxa", - "key": "pdxa/discode/staging/uua-favorite-archival-events" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-favorite-archival-events.aurora b/unified_user_actions/service/deploy/uua-favorite-archival-events.aurora deleted file mode 100644 index f37ad3d89..000000000 --- a/unified_user_actions/service/deploy/uua-favorite-archival-events.aurora +++ /dev/null @@ -1,167 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-favorite-archival-events' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 10) - kafka_bootstrap_servers = Default(String, '/s/kafka/main-2:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'favorite_archival_events') - sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = RAM_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=128.megabytes' - ' -kafka.producer.linger=0.milliseconds' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'INFO', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-favorite-archival-events.docx b/unified_user_actions/service/deploy/uua-favorite-archival-events.docx new file mode 100644 index 000000000..9a53e5804 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-favorite-archival-events.docx differ diff --git a/unified_user_actions/service/deploy/uua-retweet-archival-events-prod.docx b/unified_user_actions/service/deploy/uua-retweet-archival-events-prod.docx new file mode 100644 index 000000000..192abf6ae Binary files /dev/null and b/unified_user_actions/service/deploy/uua-retweet-archival-events-prod.docx differ diff --git a/unified_user_actions/service/deploy/uua-retweet-archival-events-prod.workflow b/unified_user_actions/service/deploy/uua-retweet-archival-events-prod.workflow deleted file mode 100644 index 519b8c958..000000000 --- a/unified_user_actions/service/deploy/uua-retweet-archival-events-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "uua-retweet-archival-events-prod", - "config-files": [ - "uua-retweet-archival-events.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-retweet-archival-events" - }, - { - "type": "packer", - "name": "uua-retweet-archival-events", - "artifact": "./dist/uua-retweet-archival-events.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "uua-retweet-archival-events-prod-atla", - "key": "atla/discode/prod/uua-retweet-archival-events" - }, - { - "name": "uua-retweet-archival-events-prod-pdxa", - "key": "pdxa/discode/prod/uua-retweet-archival-events" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-retweet-archival-events-staging.docx b/unified_user_actions/service/deploy/uua-retweet-archival-events-staging.docx new file mode 100644 index 000000000..c57e1d847 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-retweet-archival-events-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-retweet-archival-events-staging.workflow b/unified_user_actions/service/deploy/uua-retweet-archival-events-staging.workflow deleted file mode 100644 index 2cece5161..000000000 --- a/unified_user_actions/service/deploy/uua-retweet-archival-events-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-retweet-archival-events-staging", - "config-files": [ - "uua-retweet-archival-events.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-retweet-archival-events" - }, - { - "type": "packer", - "name": "uua-retweet-archival-events-staging", - "artifact": "./dist/uua-retweet-archival-events.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-retweet-archival-events-staging-pdxa", - "key": "pdxa/discode/staging/uua-retweet-archival-events" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-retweet-archival-events.aurora b/unified_user_actions/service/deploy/uua-retweet-archival-events.aurora deleted file mode 100644 index 12c4dedae..000000000 --- a/unified_user_actions/service/deploy/uua-retweet-archival-events.aurora +++ /dev/null @@ -1,167 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-retweet-archival-events' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 10) - kafka_bootstrap_servers = Default(String, '/s/kafka/main-2:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'retweet_archival_events') - sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = RAM_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=128.megabytes' - ' -kafka.producer.linger=0.milliseconds' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'INFO', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-retweet-archival-events.docx b/unified_user_actions/service/deploy/uua-retweet-archival-events.docx new file mode 100644 index 000000000..b07b3b305 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-retweet-archival-events.docx differ diff --git a/unified_user_actions/service/deploy/uua-social-graph-prod.docx b/unified_user_actions/service/deploy/uua-social-graph-prod.docx new file mode 100644 index 000000000..f484b3501 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-social-graph-prod.docx differ diff --git a/unified_user_actions/service/deploy/uua-social-graph-prod.workflow b/unified_user_actions/service/deploy/uua-social-graph-prod.workflow deleted file mode 100644 index bc9debfc5..000000000 --- a/unified_user_actions/service/deploy/uua-social-graph-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "uua-social-graph-prod", - "config-files": [ - "uua-social-graph.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-social-graph" - }, - { - "type": "packer", - "name": "uua-social-graph", - "artifact": "./dist/uua-social-graph.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "uua-social-graph-prod-atla", - "key": "atla/discode/prod/uua-social-graph" - }, - { - "name": "uua-social-graph-prod-pdxa", - "key": "pdxa/discode/prod/uua-social-graph" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-social-graph-staging.docx b/unified_user_actions/service/deploy/uua-social-graph-staging.docx new file mode 100644 index 000000000..519efd081 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-social-graph-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-social-graph-staging.workflow b/unified_user_actions/service/deploy/uua-social-graph-staging.workflow deleted file mode 100644 index 9d022b4eb..000000000 --- a/unified_user_actions/service/deploy/uua-social-graph-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-social-graph-staging", - "config-files": [ - "uua-social-graph.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-social-graph" - }, - { - "type": "packer", - "name": "uua-social-graph-staging", - "artifact": "./dist/uua-social-graph.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-social-graph-staging-pdxa", - "key": "pdxa/discode/staging/uua-social-graph" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-social-graph.aurora b/unified_user_actions/service/deploy/uua-social-graph.aurora deleted file mode 100644 index 79dbb4262..000000000 --- a/unified_user_actions/service/deploy/uua-social-graph.aurora +++ /dev/null @@ -1,167 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-social-graph' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 20) - kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'social_write_event') - sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = RAM_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=128.megabytes' - ' -kafka.producer.linger=0.second' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'INFO', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-social-graph.docx b/unified_user_actions/service/deploy/uua-social-graph.docx new file mode 100644 index 000000000..f72687646 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-social-graph.docx differ diff --git a/unified_user_actions/service/deploy/uua-tls-favs-prod.docx b/unified_user_actions/service/deploy/uua-tls-favs-prod.docx new file mode 100644 index 000000000..a6f1ae8a6 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-tls-favs-prod.docx differ diff --git a/unified_user_actions/service/deploy/uua-tls-favs-prod.workflow b/unified_user_actions/service/deploy/uua-tls-favs-prod.workflow deleted file mode 100644 index 1ca30b3dc..000000000 --- a/unified_user_actions/service/deploy/uua-tls-favs-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "uua-tls-favs-prod", - "config-files": [ - "uua-tls-favs.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-tls-favs" - }, - { - "type": "packer", - "name": "uua-tls-favs", - "artifact": "./dist/uua-tls-favs.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "uua-tls-favs-prod-atla", - "key": "atla/discode/prod/uua-tls-favs" - }, - { - "name": "uua-tls-favs-prod-pdxa", - "key": "pdxa/discode/prod/uua-tls-favs" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-tls-favs-staging.docx b/unified_user_actions/service/deploy/uua-tls-favs-staging.docx new file mode 100644 index 000000000..5c75c5f25 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-tls-favs-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-tls-favs-staging.workflow b/unified_user_actions/service/deploy/uua-tls-favs-staging.workflow deleted file mode 100644 index a2be55c29..000000000 --- a/unified_user_actions/service/deploy/uua-tls-favs-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-tls-favs-staging", - "config-files": [ - "uua-tls-favs.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-tls-favs" - }, - { - "type": "packer", - "name": "uua-tls-favs-staging", - "artifact": "./dist/uua-tls-favs.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-tls-favs-staging-pdxa", - "key": "pdxa/discode/staging/uua-tls-favs" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-tls-favs.aurora b/unified_user_actions/service/deploy/uua-tls-favs.aurora deleted file mode 100644 index 4f3c2a720..000000000 --- a/unified_user_actions/service/deploy/uua-tls-favs.aurora +++ /dev/null @@ -1,167 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-tls-favs' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 20) - kafka_bootstrap_servers = Default(String, '/s/kafka/main-1:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'timeline_service_favorites') - sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = RAM_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=128.megabytes' - ' -kafka.producer.linger=50.milliseconds' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'INFO', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-tls-favs.docx b/unified_user_actions/service/deploy/uua-tls-favs.docx new file mode 100644 index 000000000..361a86a44 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-tls-favs.docx differ diff --git a/unified_user_actions/service/deploy/uua-tweetypie-event-prod.docx b/unified_user_actions/service/deploy/uua-tweetypie-event-prod.docx new file mode 100644 index 000000000..91b494ec4 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-tweetypie-event-prod.docx differ diff --git a/unified_user_actions/service/deploy/uua-tweetypie-event-prod.workflow b/unified_user_actions/service/deploy/uua-tweetypie-event-prod.workflow deleted file mode 100644 index ee1cfede2..000000000 --- a/unified_user_actions/service/deploy/uua-tweetypie-event-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "uua-tweetypie-event-prod", - "config-files": [ - "uua-tweetypie-event.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-tweetypie-event" - }, - { - "type": "packer", - "name": "uua-tweetypie-event", - "artifact": "./dist/uua-tweetypie-event.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "uua-tweetypie-event-prod-atla", - "key": "atla/discode/prod/uua-tweetypie-event" - }, - { - "name": "uua-tweetypie-event-prod-pdxa", - "key": "pdxa/discode/prod/uua-tweetypie-event" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-tweetypie-event-staging.docx b/unified_user_actions/service/deploy/uua-tweetypie-event-staging.docx new file mode 100644 index 000000000..35bf5f294 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-tweetypie-event-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-tweetypie-event-staging.workflow b/unified_user_actions/service/deploy/uua-tweetypie-event-staging.workflow deleted file mode 100644 index be41907d6..000000000 --- a/unified_user_actions/service/deploy/uua-tweetypie-event-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-tweetypie-event-staging", - "config-files": [ - "uua-tweetypie-event.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-tweetypie-event" - }, - { - "type": "packer", - "name": "uua-tweetypie-event-staging", - "artifact": "./dist/uua-tweetypie-event.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-tweetypie-event-staging-pdxa", - "key": "pdxa/discode/staging/uua-tweetypie-event" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-tweetypie-event.aurora b/unified_user_actions/service/deploy/uua-tweetypie-event.aurora deleted file mode 100644 index 6adf59351..000000000 --- a/unified_user_actions/service/deploy/uua-tweetypie-event.aurora +++ /dev/null @@ -1,167 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-tweetypie-event' - -CPU_NUM = 2 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 20) - kafka_bootstrap_servers = Default(String, '/s/kafka/tweet-events:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'tweet_events') - sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = DISK_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=64.megabytes' - ' -kafka.producer.linger=0.milliseconds' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'INFO', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-tweetypie-event.docx b/unified_user_actions/service/deploy/uua-tweetypie-event.docx new file mode 100644 index 000000000..fee51f02b Binary files /dev/null and b/unified_user_actions/service/deploy/uua-tweetypie-event.docx differ diff --git a/unified_user_actions/service/deploy/uua-user-modification-prod.docx b/unified_user_actions/service/deploy/uua-user-modification-prod.docx new file mode 100644 index 000000000..228b1ae2e Binary files /dev/null and b/unified_user_actions/service/deploy/uua-user-modification-prod.docx differ diff --git a/unified_user_actions/service/deploy/uua-user-modification-prod.workflow b/unified_user_actions/service/deploy/uua-user-modification-prod.workflow deleted file mode 100644 index abb6397de..000000000 --- a/unified_user_actions/service/deploy/uua-user-modification-prod.workflow +++ /dev/null @@ -1,66 +0,0 @@ -{ - "role": "discode", - "name": "uua-user-modification-prod", - "config-files": [ - "uua-user-modification.aurora" - ], - "build": { - "play": true, - "trigger": { - "cron-schedule": "0 17 * * 2" - }, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-user-modification" - }, - { - "type": "packer", - "name": "uua-user-modification", - "artifact": "./dist/uua-user-modification.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "prod", - "targets": [ - { - "name": "uua-user-modification-prod-atla", - "key": "atla/discode/prod/uua-user-modification" - }, - { - "name": "uua-user-modification-prod-pdxa", - "key": "pdxa/discode/prod/uua-user-modification" - } - ] - } - ], - "subscriptions": [ - { - "type": "SLACK", - "recipients": [ - { - "to": "discode-oncall" - } - ], - "events": ["WORKFLOW_SUCCESS"] - }, - { - "type": "SLACK", - "recipients": [{ - "to": "discode-oncall" - }], - "events": ["*FAILED"] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-user-modification-staging.docx b/unified_user_actions/service/deploy/uua-user-modification-staging.docx new file mode 100644 index 000000000..44dc57994 Binary files /dev/null and b/unified_user_actions/service/deploy/uua-user-modification-staging.docx differ diff --git a/unified_user_actions/service/deploy/uua-user-modification-staging.workflow b/unified_user_actions/service/deploy/uua-user-modification-staging.workflow deleted file mode 100644 index 55f8f4ef7..000000000 --- a/unified_user_actions/service/deploy/uua-user-modification-staging.workflow +++ /dev/null @@ -1,41 +0,0 @@ -{ - "role": "discode", - "name": "uua-user-modification-staging", - "config-files": [ - "uua-user-modification.aurora" - ], - "build": { - "play": true, - "dependencies": [ - { - "role": "packer", - "name": "packer-client-no-pex", - "version": "latest" - } - ], - "steps": [ - { - "type": "bazel-bundle", - "name": "bundle", - "target": "unified_user_actions/service/src/main/scala:uua-user-modification" - }, - { - "type": "packer", - "name": "uua-user-modification-staging", - "artifact": "./dist/uua-user-modification.zip" - } - ] - }, - "targets": [ - { - "type": "group", - "name": "staging", - "targets": [ - { - "name": "uua-user-modification-staging-pdxa", - "key": "pdxa/discode/staging/uua-user-modification" - } - ] - } - ] -} diff --git a/unified_user_actions/service/deploy/uua-user-modification.aurora b/unified_user_actions/service/deploy/uua-user-modification.aurora deleted file mode 100644 index 82abd0483..000000000 --- a/unified_user_actions/service/deploy/uua-user-modification.aurora +++ /dev/null @@ -1,167 +0,0 @@ -import os -import itertools -import subprocess -import math - -SERVICE_NAME = 'uua-user-modification' - -CPU_NUM = 3 -HEAP_SIZE = 3 * GB -RAM_SIZE = HEAP_SIZE + 1 * GB -# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk. -DISK_SIZE = HEAP_SIZE + 2 * GB - -class Profile(Struct): - package = Default(String, SERVICE_NAME) - cmdline_flags = Default(String, '') - log_level = Default(String, 'INFO') - instances = Default(Integer, 10) - kafka_bootstrap_servers = Default(String, '/s/kafka/main-1:kafka-tls') - kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls') - source_topic = Default(String, 'user_modifications') - sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements') - decider_overlay = Default(String, '') - -resources = Resources( - cpu = CPU_NUM, - ram = RAM_SIZE, - disk = DISK_SIZE -) - -install = Packer.install( - name = '{{profile.package}}', - version = Workflows.package_version() -) - -async_profiler_install = Packer.install( - name = 'async-profiler', - role = 'csl-perf', - version = 'latest' -) - -setup_jaas_config = Process( - name = 'setup_jaas_config', - cmdline = ''' - mkdir -p jaas_config - echo "KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - principal=\\"discode@TWITTER.BIZ\\" - useKeyTab=true - storeKey=true - keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\" - doNotPrompt=true; - };" >> jaas_config/jaas.conf - ''' -) - -main = JVMProcess( - name = SERVICE_NAME, - jvm = Java11( - heap = HEAP_SIZE, - extra_jvm_flags = - '-Djava.net.preferIPv4Stack=true' - - ' -XX:+UseNUMA' - ' -XX:+AggressiveOpts' - ' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html - - ' -Dlog_level={{profile.log_level}}' - ' -Dlog.access.output=access.log' - ' -Dlog.service.output={{name}}.log' - ' -Djava.security.auth.login.config=jaas_config/jaas.conf' - ), - arguments = - '-jar {{name}}-bin.jar' - ' -admin.port=:{{thermos.ports[health]}}' - ' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}' - ' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}' - ' -kafka.group.id={{name}}-{{environment}}' - ' -kafka.producer.client.id={{name}}-{{environment}}' - ' -kafka.max.pending.requests=10000' - ' -kafka.consumer.fetch.max=1.megabytes' - ' -kafka.producer.batch.size=16.kilobytes' - ' -kafka.producer.buffer.mem=128.megabytes' - ' -kafka.producer.linger=50.milliseconds' - ' -kafka.producer.request.timeout=30.seconds' - ' -kafka.producer.compression.type=lz4' - ' -kafka.worker.threads=5' - ' -kafka.source.topic={{profile.source_topic}}' - ' -kafka.sink.topics={{profile.sink_topics}}' - ' -decider.base=decider.yml' - ' -decider.overlay={{profile.decider_overlay}}' - ' -cluster={{cluster}}' - ' {{profile.cmdline_flags}}', - resources = resources -) - -stats = Stats( - library = 'metrics', - port = 'admin' -) - -job_template = Service( - name = SERVICE_NAME, - role = 'discode', - instances = '{{profile.instances}}', - contact = 'disco-data-eng@twitter.com', - constraints = {'rack': 'limit:1', 'host': 'limit:1'}, - announce = Announcer( - primary_port = 'health', - portmap = {'aurora': 'health', 'admin': 'health'} - ), - task = Task( - resources = resources, - name = SERVICE_NAME, - processes = [async_profiler_install, install, setup_jaas_config, main, stats], - constraints = order(async_profiler_install, install, setup_jaas_config, main) - ), - health_check_config = HealthCheckConfig( - initial_interval_secs = 100, - interval_secs = 60, - timeout_secs = 60, - max_consecutive_failures = 4 - ), - update_config = UpdateConfig( - batch_size = 50, - watch_secs = 90, - max_per_shard_failures = 3, - max_total_failures = 0, - rollback_on_failure = False - ) -) - -PRODUCTION = Profile( - # go/uua-decider - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml' -) - -STAGING = Profile( - package = SERVICE_NAME+'-staging', - cmdline_flags = '', - kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls', - decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider -) - -DEVEL = STAGING( - log_level = 'DEBUG', -) - - -prod_job = job_template( - tier = 'preferred', - environment = 'prod', -).bind(profile = PRODUCTION) - -staging_job = job_template( - environment = 'staging' -).bind(profile = STAGING) - -devel_job = job_template( - environment = 'devel' -).bind(profile = DEVEL) - -jobs = [] -for cluster in ['atla', 'pdxa']: - jobs.append(prod_job(cluster = cluster)) - jobs.append(staging_job(cluster = cluster)) - jobs.append(devel_job(cluster = cluster)) diff --git a/unified_user_actions/service/deploy/uua-user-modification.docx b/unified_user_actions/service/deploy/uua-user-modification.docx new file mode 100644 index 000000000..b68223feb Binary files /dev/null and b/unified_user_actions/service/deploy/uua-user-modification.docx differ diff --git a/unified_user_actions/service/src/main/resources/BUILD b/unified_user_actions/service/src/main/resources/BUILD deleted file mode 100644 index 90cacb56c..000000000 --- a/unified_user_actions/service/src/main/resources/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -resources( - sources = ["*.*"], - tags = ["bazel-compatible"], -) - -files( - name = "files", - sources = [ - "!BUILD", - "**/*", - ], - tags = ["bazel-compatible"], -) diff --git a/unified_user_actions/service/src/main/resources/BUILD.docx b/unified_user_actions/service/src/main/resources/BUILD.docx new file mode 100644 index 000000000..b18bcc628 Binary files /dev/null and b/unified_user_actions/service/src/main/resources/BUILD.docx differ diff --git a/unified_user_actions/service/src/main/resources/decider.docx b/unified_user_actions/service/src/main/resources/decider.docx new file mode 100644 index 000000000..8ef6c5017 Binary files /dev/null and b/unified_user_actions/service/src/main/resources/decider.docx differ diff --git a/unified_user_actions/service/src/main/resources/decider.yml b/unified_user_actions/service/src/main/resources/decider.yml deleted file mode 100644 index 23aa40bc3..000000000 --- a/unified_user_actions/service/src/main/resources/decider.yml +++ /dev/null @@ -1,324 +0,0 @@ -# Naming convention: -# For publishing action types, use [Publish][ActionTypeInThrift]. Please see the Thrift definition at unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.thrift - -PublishServerTweetFav: - default_availability: 0 -PublishServerTweetUnfav: - default_availability: 0 -PublishServerTweetCreate: - default_availability: 0 -PublishServerTweetReply: - default_availability: 0 -PublishServerTweetQuote: - default_availability: 0 -PublishServerTweetRetweet: - default_availability: 0 -PublishServerTweetDelete: - default_availability: 0 -PublishServerTweetUnreply: - default_availability: 0 -PublishServerTweetUnquote: - default_availability: 0 -PublishServerTweetUnretweet: - default_availability: 0 -PublishServerTweetEdit: - default_availability: 0 -PublishServerTweetReport: - default_availability: 0 -PublishServerProfileFollow: - default_availability: 0 -PublishServerProfileUnfollow: - default_availability: 0 -PublishServerProfileBlock: - default_availability: 0 -PublishServerProfileUnblock: - default_availability: 0 -PublishServerProfileMute: - default_availability: 0 -PublishServerProfileUnmute: - default_availability: 0 -PublishServerProfileReport: - default_availability: 0 -PublishClientTweetFav: - default_availability: 0 -PublishClientTweetUnfav: - default_availability: 0 -PublishClientTweetLingerImpression: - default_availability: 0 -PublishClientTweetRenderImpression: - default_availability: 0 -PublishClientTweetReply: - default_availability: 0 -PublishClientTweetQuote: - default_availability: 0 -PublishClientTweetRetweet: - default_availability: 0 -PublishClientTweetClickReply: - default_availability: 0 -PublishClientTweetClickQuote: - default_availability: 0 -PublishClientTweetVideoPlayback25: - default_availability: 0 -PublishClientTweetVideoPlayback50: - default_availability: 0 -PublishClientTweetVideoPlayback75: - default_availability: 0 -PublishClientTweetVideoPlayback95: - default_availability: 0 -PublishClientTweetVideoPlayFromTap: - default_availability: 0 -PublishClientTweetVideoQualityView: - default_availability: 0 -PublishClientTweetVideoView: - default_availability: 0 -PublishClientTweetVideoMrcView: - default_availability: 0 -PublishClientTweetVideoViewThreshold: - default_availability: 0 -PublishClientTweetVideoCtaUrlClick: - default_availability: 0 -PublishClientTweetVideoCtaWatchClick: - default_availability: 0 -PublishClientTweetUnretweet: - default_availability: 0 -PublishClientTweetClickCaret: - default_availability: 0 -PublishClientTweetPhotoExpand: - default_availability: 0 -PublishClientTweetClickMentionScreenName: - default_availability: 0 -PublishClientCardClick: - default_availability: 0 -PublishClientCardOpenApp: - default_availability: 0 -PublishClientCardAppInstallAttempt: - default_availability: 0 -PublishClientPollCardVote: - default_availability: 0 -PublishClientTweetProfileMentionClick: - default_availability: 0 -PublishClientTweetClick: - default_availability: 0 -PublishClientTopicFollow: - default_availability: 0 -PublishClientTopicUnfollow: - default_availability: 0 -PublishClientTopicNotInterestedIn: - default_availability: 0 -PublishClientTopicUndoNotInterestedIn: - default_availability: 0 -PublishClientTweetNotHelpful: - default_availability: 0 -PublishClientTweetUndoNotHelpful: - default_availability: 0 -PublishClientTweetReport: - default_availability: 0 -PublishClientTweetNotInterestedIn: - default_availability: 0 -PublishClientTweetUndoNotInterestedIn: - default_availability: 0 -PublishClientTweetNotAboutTopic: - default_availability: 0 -PublishClientTweetUndoNotAboutTopic: - default_availability: 0 -PublishClientTweetNotRecent: - default_availability: 0 -PublishClientTweetUndoNotRecent: - default_availability: 0 -PublishClientTweetSeeFewer: - default_availability: 0 -PublishClientTweetUndoSeeFewer: - default_availability: 0 -PublishClientTweetNotRelevant: - default_availability: 0 -PublishClientTweetUndoNotRelevant: - default_availability: 0 -PublishClientProfileFollowAttempt: - default_availability: 0 -PublishClientTweetFavoriteAttempt: - default_availability: 0 -PublishClientTweetRetweetAttempt: - default_availability: 0 -PublishClientTweetReplyAttempt: - default_availability: 0 -PublishClientCTALoginClick: - default_availability: 0 -PublishClientCTALoginStart: - default_availability: 0 -PublishClientCTALoginSuccess: - default_availability: 0 -PublishClientCTASignupClick: - default_availability: 0 -PublishClientCTASignupSuccess: - default_availability: 0 -PublishClientProfileBlock: - default_availability: 0 -PublishClientProfileUnblock: - default_availability: 0 -PublishClientProfileMute: - default_availability: 0 -PublishClientProfileReport: - default_availability: 0 -PublishClientProfileFollow: - default_availability: 0 -PublishClientProfileClick: - default_availability: 0 -PublishClientTweetFollowAuthor: - default_availability: 0 -PublishClientTweetUnfollowAuthor: - default_availability: 0 -PublishClientTweetBlockAuthor: - default_availability: 0 -PublishClientTweetUnblockAuthor: - default_availability: 0 -PublishClientTweetMuteAuthor: - default_availability: 0 -PublishClientNotificationOpen: - default_availability: 0 -PublishClientNotificationClick: - default_availability: 0 -PublishClientNotificationSeeLessOften: - default_availability: 0 -PublishClientNotificationDismiss: - default_availability: 0 -PublishClientTypeaheadClick: - default_availability: 0 -PublishClientFeedbackPromptSubmit: - default_availability: 0 -PublishClientProfileShow: - default_availability: 0 -PublishClientTweetV2Impression: - default_availability: 0 -PublishClientTweetVideoFullscreenV2Impression: - default_availability: 0 -PublishClientTweetImageFullscreenV2Impression: - default_availability: 0 -PublishClientProfileV2Impression: - default_availability: 0 -PublishClientTweetClickProfile: - default_availability: 0 -PublishClientTweetClickShare: - default_availability: 0 -PublishClientTweetShareViaCopyLink: - default_availability: 0 -PublishClientTweetClickSendViaDirectMessage: - default_availability: 0 -PublishClientTweetShareViaBookmark: - default_availability: 0 -PublishClientTweetUnbookmark: - default_availability: 0 -PublishClientTweetClickHashtag: - default_availability: 0 -PublishClientTweetBookmark: - default_availability: 0 -PublishClientTweetOpenLink: - default_availability: 0 -PublishClientTweetTakeScreenshot: - default_availability: 0 -PublishClientTweetVideoPlaybackStart: - default_availability: 0 -PublishClientTweetVideoPlaybackComplete: - default_availability: 0 -PublishClientTweetEmailClick: - default_availability: 0 -PublishClientAppExit: - default_availability: 0 -PublishClientTweetGalleryImpression: - default_availability: 0 -PublishClientTweetDetailsImpression: - default_availability: 0 -PublishClientTweetMomentImpression: - default_availability: 0 -PublishServerUserCreate: - default_availability: 0 -PublishServerUserUpdate: - default_availability: 0 -PublishServerPromotedTweetFav: - default_availability: 0 -PublishServerPromotedTweetUnfav: - default_availability: 0 -PublishServerPromotedTweetReply: - default_availability: 0 -PublishServerPromotedTweetRetweet: - default_availability: 0 -PublishServerPromotedTweetComposeTweet: - default_availability: 0 -PublishServerPromotedTweetBlockAuthor: - default_availability: 0 -PublishServerPromotedTweetUnblockAuthor: - default_availability: 0 -PublishServerPromotedTweetClick: - default_availability: 0 -PublishServerPromotedTweetReport: - default_availability: 0 -PublishServerPromotedProfileFollow: - default_availability: 0 -PublishServerPromotedProfileUnfollow: - default_availability: 0 -PublishServerPromotedTweetMuteAuthor: - default_availability: 0 -PublishServerPromotedTweetClickProfile: - default_availability: 0 -PublishServerPromotedTweetClickHashtag: - default_availability: 0 -PublishServerPromotedTweetOpenLink: - default_availability: 0 -PublishServerPromotedTweetCarouselSwipeNext: - default_availability: 0 -PublishServerPromotedTweetCarouselSwipePrevious: - default_availability: 0 -PublishServerPromotedTweetLingerImpressionShort: - default_availability: 0 -PublishServerPromotedTweetLingerImpressionMedium: - default_availability: 0 -PublishServerPromotedTweetLingerImpressionLong: - default_availability: 0 -PublishServerPromotedTweetClickSpotlight: - default_availability: 0 -PublishServerPromotedTweetViewSpotlight: - default_availability: 0 -PublishServerPromotedTrendView: - default_availability: 0 -PublishServerPromotedTrendClick: - default_availability: 0 -PublishServerPromotedTweetVideoPlayback25: - default_availability: 0 -PublishServerPromotedTweetVideoPlayback50: - default_availability: 0 -PublishServerPromotedTweetVideoPlayback75: - default_availability: 0 -PublishServerPromotedTweetVideoAdPlayback25: - default_availability: 0 -PublishServerPromotedTweetVideoAdPlayback50: - default_availability: 0 -PublishServerPromotedTweetVideoAdPlayback75: - default_availability: 0 -PublishServerTweetVideoAdPlayback25: - default_availability: 0 -PublishServerTweetVideoAdPlayback50: - default_availability: 0 -PublishServerTweetVideoAdPlayback75: - default_availability: 0 -PublishServerPromotedTweetDismissWithoutReason: - default_availability: 0 -PublishServerPromotedTweetDismissUninteresting: - default_availability: 0 -PublishServerPromotedTweetDismissRepetitive: - default_availability: 0 -PublishServerPromotedTweetDismissSpam: - default_availability: 0 -PublishServerTweetArchiveFavorite: - default_availability: 0 -PublishServerTweetUnarchiveFavorite: - default_availability: 0 -PublishServerTweetArchiveRetweet: - default_availability: 0 -PublishServerTweetUnarchiveRetweet: - default_availability: 0 -RekeyUUAClientTweetRenderImpression: - default_availability: 0 -RekeyUUAIesourceClientTweetRenderImpression: - default_availability: 0 -EnrichmentPlannerSampling: - default_availability: 0 - diff --git a/unified_user_actions/service/src/main/resources/logback.docx b/unified_user_actions/service/src/main/resources/logback.docx new file mode 100644 index 000000000..61a808de0 Binary files /dev/null and b/unified_user_actions/service/src/main/resources/logback.docx differ diff --git a/unified_user_actions/service/src/main/resources/logback.xml b/unified_user_actions/service/src/main/resources/logback.xml deleted file mode 100644 index c23b0d6b6..000000000 --- a/unified_user_actions/service/src/main/resources/logback.xml +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - - - - true - - - - - - - - - - - - - - ${log.service.output} - - ${log.service.output}.%i - 1 - 10 - - - 50MB - - - %date %.-3level %logger ${DEFAULT_SERVICE_PATTERN}%n - - - - - - false - ${log.lens.index} - ${log.lens.tag}/service - - ${DEFAULT_SERVICE_PATTERN} - - - - - - - - - - - - - - WARN - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/unified_user_actions/service/src/main/scala/BUILD b/unified_user_actions/service/src/main/scala/BUILD deleted file mode 100644 index fe9ce7063..000000000 --- a/unified_user_actions/service/src/main/scala/BUILD +++ /dev/null @@ -1,390 +0,0 @@ -jvm_binary( - name = "uua-tls-favs-bin", - basename = "uua-tls-favs-bin", - main = "com.twitter.unified_user_actions.service.TlsFavsServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:tls-favs", - ], -) - -jvm_app( - name = "uua-tls-favs", - archive = "zip", - binary = ":uua-tls-favs-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "uua-client-event-bin", - basename = "uua-client-event-bin", - main = "com.twitter.unified_user_actions.service.ClientEventServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:client-event", - ], -) - -jvm_app( - name = "uua-client-event", - archive = "zip", - binary = ":uua-client-event-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - - -jvm_binary( - name = "uua-tweetypie-event-bin", - basename = "uua-tweetypie-event-bin", - main = "com.twitter.unified_user_actions.service.TweetypieEventServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:tweetypie-event", - ], -) - -jvm_app( - name = "uua-tweetypie-event", - archive = "zip", - binary = ":uua-tweetypie-event-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "uua-social-graph-bin", - basename = "uua-social-graph-bin", - main = "com.twitter.unified_user_actions.service.SocialGraphServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:social-graph-event", - ], -) - -jvm_app( - name = "uua-social-graph", - archive = "zip", - binary = ":uua-social-graph-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "uua-email-notification-event-bin", - basename = "uua-email-notification-event-bin", - main = "com.twitter.unified_user_actions.service.EmailNotificationEventServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:email-notification-event", - ], -) - -jvm_app( - name = "uua-email-notification-event", - archive = "zip", - binary = ":uua-email-notification-event-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "uua-user-modification-bin", - basename = "uua-user-modification-bin", - main = "com.twitter.unified_user_actions.service.UserModificationServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:user-modification-event", - ], -) - -jvm_app( - name = "uua-user-modification", - archive = "zip", - binary = ":uua-user-modification-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "uua-ads-callback-engagements-bin", - basename = "uua-ads-callback-engagements-bin", - main = "com.twitter.unified_user_actions.service.AdsCallbackEngagementsServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:ads-callback-engagements", - ], -) - -jvm_app( - name = "uua-ads-callback-engagements", - archive = "zip", - binary = ":uua-ads-callback-engagements-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "uua-favorite-archival-events-bin", - basename = "uua-favorite-archival-events-bin", - main = "com.twitter.unified_user_actions.service.FavoriteArchivalEventsServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:favorite-archival-events", - ], -) - -jvm_app( - name = "uua-favorite-archival-events", - archive = "zip", - binary = ":uua-favorite-archival-events-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "uua-retweet-archival-events-bin", - basename = "uua-retweet-archival-events-bin", - main = "com.twitter.unified_user_actions.service.RetweetArchivalEventsServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:retweet-archival-events", - ], -) - -jvm_app( - name = "uua-retweet-archival-events", - archive = "zip", - binary = ":uua-retweet-archival-events-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "rekey-uua-bin", - basename = "rekey-uua-bin", - main = "com.twitter.unified_user_actions.service.RekeyUuaServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:rekey-uua", - ], -) - -jvm_app( - name = "rekey-uua", - archive = "zip", - binary = ":rekey-uua-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "rekey-uua-iesource-bin", - basename = "rekey-uua-iesource-bin", - main = "com.twitter.unified_user_actions.service.RekeyUuaIesourceServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:rekey-uua-iesource", - ], -) - -jvm_app( - name = "rekey-uua-iesource", - archive = "zip", - binary = ":rekey-uua-iesource-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "uua-enrichment-planner-bin", - basename = "uua-enrichment-planner-bin", - main = "com.twitter.unified_user_actions.service.EnrichmentPlannerServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:enrichment-planner", - ], -) - -jvm_app( - name = "uua-enrichment-planner", - archive = "zip", - binary = ":uua-enrichment-planner-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) - -jvm_binary( - name = "uua-enricher-bin", - basename = "uua-enricher-bin", - main = "com.twitter.unified_user_actions.service.EnricherServiceMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - "unified_user_actions/service/src/main/resources", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:enricher", - ], -) - -jvm_app( - name = "uua-enricher", - archive = "zip", - binary = ":uua-enricher-bin", - bundles = [ - bundle( - fileset = ["**/*"], - owning_target = "unified_user_actions/service/src/main/resources:files", - rel_path = "unified_user_actions/service/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) diff --git a/unified_user_actions/service/src/main/scala/BUILD.docx b/unified_user_actions/service/src/main/scala/BUILD.docx new file mode 100644 index 000000000..1b988df3e Binary files /dev/null and b/unified_user_actions/service/src/main/scala/BUILD.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/AdsCallbackEngagementsService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/AdsCallbackEngagementsService.docx new file mode 100644 index 000000000..c2c5ac746 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/AdsCallbackEngagementsService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/AdsCallbackEngagementsService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/AdsCallbackEngagementsService.scala deleted file mode 100644 index 9e0a23aac..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/AdsCallbackEngagementsService.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.ads.spendserver.thriftscala.SpendServerEvent -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.service.module.KafkaProcessorAdsCallbackEngagementsModule - -object AdsCallbackEngagementsServiceMain extends AdsCallbackEngagementsService - -class AdsCallbackEngagementsService extends TwitterServer { - override val modules = Seq( - KafkaProcessorAdsCallbackEngagementsModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, SpendServerEvent]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/BUILD b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/BUILD deleted file mode 100644 index 2936e039d..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/BUILD +++ /dev/null @@ -1,270 +0,0 @@ -scala_library( - name = "tls-favs", - sources = ["TlsFavsService.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:tls-favs", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "client-event", - sources = ["ClientEventService.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twadoop_config/configuration/log_categories/group/scribelib:client_event-scala", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:client-event", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "tweetypie-event", - sources = ["TweetypieEventService.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twadoop_config/configuration/log_categories/group/scribelib:client_event-scala", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:tweetypie-event", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "social-graph-event", - sources = ["SocialGraphService.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:social-graph-event", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "email-notification-event", - sources = ["EmailNotificationEventService.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:email-notification-event", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "user-modification-event", - sources = ["UserModificationService.scala"], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:user-modification-event", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "ads-callback-engagements", - sources = ["AdsCallbackEngagementsService.scala"], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:ads-callback-engagements", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "favorite-archival-events", - sources = ["FavoriteArchivalEventsService.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:favorite-archival-events", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "retweet-archival-events", - sources = ["RetweetArchivalEventsService.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:retweet-archival-events", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "rekey-uua", - sources = ["RekeyUuaService.scala"], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:rekey-uua", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "rekey-uua-iesource", - sources = ["RekeyUuaIesourceService.scala"], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:rekey-uua-iesource", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "enrichment-planner", - sources = ["EnrichmentPlannerService.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "decider/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra-internal/kafka-streams/kafka-streams/src/main/scala", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/producers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra-internal/mtls/src/main/scala", - "kafka/finagle-kafka/finatra-kafka-streams/kafka-streams-static-partitioning/src/main/scala", - "kafka/finagle-kafka/finatra-kafka-streams/kafka-streams/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator:noop", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner:default", - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) - -scala_library( - name = "enricher", - sources = ["EnricherService.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "finatra-internal/kafka-streams/kafka-streams/src/main/scala", - "finatra-internal/mtls/src/main/scala", - "finatra/inject/inject-server/src/main/scala/com/twitter/inject/server", - "graphql/thrift/src/main/thrift/com/twitter/graphql:graphql-scala", - "kafka/finagle-kafka/finatra-kafka-streams/kafka-streams-static-partitioning/src/main/scala", - "kafka/finagle-kafka/finatra-kafka-streams/kafka-streams/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "twitter-server/server/src/main/scala", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator:default", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner:default", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:cache", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:graphql-client", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-app/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/BUILD.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/BUILD.docx new file mode 100644 index 000000000..6649c9b0f Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/BUILD.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/ClientEventService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/ClientEventService.docx new file mode 100644 index 000000000..126b8b5e2 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/ClientEventService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/ClientEventService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/ClientEventService.scala deleted file mode 100644 index 17584a2dc..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/ClientEventService.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.service.module.KafkaProcessorClientEventModule - -object ClientEventServiceMain extends ClientEventService - -class ClientEventService extends TwitterServer { - - override val modules = Seq(KafkaProcessorClientEventModule, DeciderModule) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, LogEvent]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EmailNotificationEventService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EmailNotificationEventService.docx new file mode 100644 index 000000000..2e82b64e3 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EmailNotificationEventService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EmailNotificationEventService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EmailNotificationEventService.scala deleted file mode 100644 index d5f2b6d9a..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EmailNotificationEventService.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.ibis.thriftscala.NotificationScribe -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.service.module.KafkaProcessorEmailNotificationEventModule - -object EmailNotificationEventServiceMain extends EmailNotificationEventService - -class EmailNotificationEventService extends TwitterServer { - - override val modules = Seq( - KafkaProcessorEmailNotificationEventModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, NotificationScribe]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnricherService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnricherService.docx new file mode 100644 index 000000000..2d88409b3 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnricherService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnricherService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnricherService.scala deleted file mode 100644 index 9459871ed..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnricherService.scala +++ /dev/null @@ -1,105 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.conversions.DurationOps._ -import com.twitter.conversions.StorageUnitOps._ -import com.twitter.dynmap.DynMap -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.domain.AckMode -import com.twitter.finatra.kafka.domain.KafkaGroupId -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafkastreams.config.KafkaStreamsConfig -import com.twitter.finatra.kafkastreams.config.SecureKafkaStreamsConfig -import com.twitter.finatra.kafkastreams.partitioning.StaticPartitioning -import com.twitter.finatra.mtls.modules.ServiceIdentifierModule -import com.twitter.finatra.kafkastreams.dsl.FinatraDslFlatMapAsync -import com.twitter.graphql.thriftscala.GraphqlExecutionService -import com.twitter.logging.Logging -import com.twitter.unified_user_actions.enricher.driver.EnrichmentDriver -import com.twitter.unified_user_actions.enricher.hcache.LocalCache -import com.twitter.unified_user_actions.enricher.hydrator.DefaultHydrator -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.unified_user_actions.enricher.partitioner.DefaultPartitioner -import com.twitter.unified_user_actions.service.module.CacheModule -import com.twitter.unified_user_actions.service.module.ClientIdModule -import com.twitter.unified_user_actions.service.module.GraphqlClientProviderModule -import com.twitter.util.Future -import org.apache.kafka.common.record.CompressionType -import org.apache.kafka.streams.StreamsBuilder -import org.apache.kafka.streams.processor.RecordContext -import org.apache.kafka.streams.processor.TopicNameExtractor -import org.apache.kafka.streams.scala.kstream.Consumed -import org.apache.kafka.streams.scala.kstream.Produced -import com.twitter.unified_user_actions.enricher.driver.EnrichmentPlanUtils._ - -object EnricherServiceMain extends EnricherService - -class EnricherService - extends FinatraDslFlatMapAsync - with StaticPartitioning - with SecureKafkaStreamsConfig - with Logging { - val InputTopic = "unified_user_actions_keyed_dev" - val OutputTopic = "unified_user_actions_enriched" - - override val modules = Seq( - CacheModule, - ClientIdModule, - GraphqlClientProviderModule, - ServiceIdentifierModule - ) - - override protected def configureKafkaStreams(builder: StreamsBuilder): Unit = { - val graphqlClient = injector.instance[GraphqlExecutionService.FinagledClient] - val localCache = injector.instance[LocalCache[EnrichmentKey, DynMap]] - val statsReceiver = injector.instance[StatsReceiver] - val driver = new EnrichmentDriver( - finalOutputTopic = Some(OutputTopic), - partitionedTopic = InputTopic, - hydrator = new DefaultHydrator( - cache = localCache, - graphqlClient = graphqlClient, - scopedStatsReceiver = statsReceiver.scope("DefaultHydrator")), - partitioner = new DefaultPartitioner - ) - - val kstream = builder.asScala - .stream(InputTopic)( - Consumed.`with`(ScalaSerdes.Thrift[EnrichmentKey], ScalaSerdes.Thrift[EnrichmentEnvelop])) - .flatMapAsync[EnrichmentKey, EnrichmentEnvelop]( - commitInterval = 5.seconds, - numWorkers = 10000 - ) { (enrichmentKey: EnrichmentKey, enrichmentEnvelop: EnrichmentEnvelop) => - driver - .execute(Some(enrichmentKey), Future.value(enrichmentEnvelop)) - .map(tuple => tuple._1.map(key => (key, tuple._2)).seq) - } - - val topicExtractor: TopicNameExtractor[EnrichmentKey, EnrichmentEnvelop] = - (_: EnrichmentKey, envelop: EnrichmentEnvelop, _: RecordContext) => - envelop.plan.getLastCompletedStage.outputTopic.getOrElse( - throw new IllegalStateException("Missing output topic in the last completed stage")) - - kstream.to(topicExtractor)( - Produced.`with`(ScalaSerdes.Thrift[EnrichmentKey], ScalaSerdes.Thrift[EnrichmentEnvelop])) - } - - override def streamsProperties(config: KafkaStreamsConfig): KafkaStreamsConfig = - super - .streamsProperties(config) - .consumer.groupId(KafkaGroupId(applicationId())) - .consumer.clientId(s"${applicationId()}-consumer") - .consumer.requestTimeout(30.seconds) - .consumer.sessionTimeout(30.seconds) - .consumer.fetchMin(1.megabyte) - .consumer.fetchMax(5.megabytes) - .consumer.receiveBuffer(32.megabytes) - .consumer.maxPollInterval(1.minute) - .consumer.maxPollRecords(50000) - .producer.clientId(s"${applicationId()}-producer") - .producer.batchSize(16.kilobytes) - .producer.bufferMemorySize(256.megabyte) - .producer.requestTimeout(30.seconds) - .producer.compressionType(CompressionType.LZ4) - .producer.ackMode(AckMode.ALL) -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerService.docx new file mode 100644 index 000000000..28e56d568 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerService.scala deleted file mode 100644 index fc8e8dbef..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerService.scala +++ /dev/null @@ -1,187 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.app.Flag -import com.twitter.conversions.DurationOps._ -import com.twitter.conversions.StorageUnitOps._ -import com.twitter.decider.Decider -import com.twitter.decider.SimpleRecipient -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.domain.AckMode -import com.twitter.finatra.kafka.domain.KafkaGroupId -import com.twitter.finatra.kafka.domain.KafkaTopic -import com.twitter.finatra.kafka.producers.FinagleKafkaProducerConfig -import com.twitter.finatra.kafka.producers.KafkaProducerConfig -import com.twitter.finatra.kafka.producers.TwitterKafkaProducerConfig -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.finatra.kafkastreams.config.KafkaStreamsConfig -import com.twitter.finatra.kafkastreams.config.SecureKafkaStreamsConfig -import com.twitter.finatra.kafkastreams.dsl.FinatraDslToCluster -import com.twitter.inject.TwitterModule -import com.twitter.unified_user_actions.enricher.driver.EnrichmentDriver -import com.twitter.unified_user_actions.enricher.hydrator.NoopHydrator -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction.NotificationTweetEnrichment -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction.TweetEnrichment -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentPlan -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStage -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageStatus -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageType -import com.twitter.unified_user_actions.enricher.partitioner.DefaultPartitioner -import com.twitter.unified_user_actions.enricher.partitioner.DefaultPartitioner.NullKey -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.util.Await -import com.twitter.util.Future -import org.apache.kafka.common.record.CompressionType -import org.apache.kafka.streams.StreamsBuilder -import org.apache.kafka.streams.scala.kstream.Consumed -import org.apache.kafka.streams.scala.kstream.KStream -import org.apache.kafka.streams.scala.kstream.Produced -object EnrichmentPlannerServiceMain extends EnrichmentPlannerService { - val ApplicationId = "uua-enrichment-planner" - val InputTopic = "unified_user_actions" - val OutputPartitionedTopic = "unified_user_actions_keyed_dev" - val SamplingDecider = "EnrichmentPlannerSampling" -} - -/** - * This service is the first step (planner) of the UUA Enrichment process. - * It does the following: - * 1. Read Prod UUA topic unified_user_actions from the Prod cluster and write to (see below) either Prod cluster (prod) or Dev cluster (dev/staging) - * 2. For the write, it optionally randomly downsample the events when publishing, controlled by a Decider - * 3. The output's key would be the first step of the repartitioning, most likely the EnrichmentKey of the Tweet type. - */ -class EnrichmentPlannerService extends FinatraDslToCluster with SecureKafkaStreamsConfig { - import EnrichmentPlannerServiceMain._ - - val kafkaOutputCluster: Flag[String] = flag( - name = "kafka.output.server", - default = "", - help = - """The output Kafka cluster. - |This is needed since we read from a cluster and potentially output to a different cluster. - |""".stripMargin - ) - - val kafkaOutputEnableTls: Flag[Boolean] = flag( - name = "kafka.output.enable.tls", - default = true, - help = "" - ) - - override val modules: Seq[TwitterModule] = Seq( - DeciderModule - ) - - override protected def configureKafkaStreams(builder: StreamsBuilder): Unit = { - val decider = injector.instance[Decider] - val driver = new EnrichmentDriver( - finalOutputTopic = NoopHydrator.OutputTopic, - partitionedTopic = OutputPartitionedTopic, - hydrator = new NoopHydrator, - partitioner = new DefaultPartitioner) - - val builderWithoutOutput = builder.asScala - .stream(InputTopic)(Consumed.`with`(UnKeyedSerde, ScalaSerdes.Thrift[UnifiedUserAction])) - // this maps and filters out the nil envelop before further processing - .flatMapValues { uua => - (uua.item match { - case Item.TweetInfo(_) => - Some(EnrichmentEnvelop( - envelopId = uua.hashCode.toLong, - uua = uua, - plan = EnrichmentPlan(Seq( - EnrichmentStage( - status = EnrichmentStageStatus.Initialized, - stageType = EnrichmentStageType.Repartition, - instructions = Seq(TweetEnrichment) - ), - EnrichmentStage( - status = EnrichmentStageStatus.Initialized, - stageType = EnrichmentStageType.Hydration, - instructions = Seq(TweetEnrichment) - ), - )) - )) - case Item.NotificationInfo(_) => - Some(EnrichmentEnvelop( - envelopId = uua.hashCode.toLong, - uua = uua, - plan = EnrichmentPlan(Seq( - EnrichmentStage( - status = EnrichmentStageStatus.Initialized, - stageType = EnrichmentStageType.Repartition, - instructions = Seq(NotificationTweetEnrichment) - ), - EnrichmentStage( - status = EnrichmentStageStatus.Initialized, - stageType = EnrichmentStageType.Hydration, - instructions = Seq(NotificationTweetEnrichment) - ), - )) - )) - case _ => None - }).seq - } - // execute our driver logics - .flatMap((_: UnKeyed, envelop: EnrichmentEnvelop) => { - // flatMap and Await.result is used here because our driver interface allows for - // both synchronous (repartition logic) and async operations (hydration logic), but in here - // we purely just need to repartition synchronously, and thus the flatMap + Await.result - // is used to simplify and make testing much easier. - val (keyOpt, value) = Await.result(driver.execute(NullKey, Future.value(envelop))) - keyOpt.map(key => (key, value)).seq - }) - // then finally we sample based on the output keys - .filter((key, _) => - decider.isAvailable(feature = SamplingDecider, Some(SimpleRecipient(key.id)))) - - configureOutput(builderWithoutOutput) - } - - private def configureOutput(kstream: KStream[EnrichmentKey, EnrichmentEnvelop]): Unit = { - if (kafkaOutputCluster().nonEmpty && kafkaOutputCluster() != bootstrapServer()) { - kstream.toCluster( - cluster = kafkaOutputCluster(), - topic = KafkaTopic(OutputPartitionedTopic), - clientId = s"$ApplicationId-output-producer", - kafkaProducerConfig = - if (kafkaOutputEnableTls()) - FinagleKafkaProducerConfig[EnrichmentKey, EnrichmentEnvelop](kafkaProducerConfig = - KafkaProducerConfig(TwitterKafkaProducerConfig().requestTimeout(1.minute).configMap)) - else - FinagleKafkaProducerConfig[EnrichmentKey, EnrichmentEnvelop]( - kafkaProducerConfig = KafkaProducerConfig() - .requestTimeout(1.minute)), - statsReceiver = statsReceiver, - commitInterval = 15.seconds - )(Produced.`with`(ScalaSerdes.Thrift[EnrichmentKey], ScalaSerdes.Thrift[EnrichmentEnvelop])) - } else { - kstream.to(OutputPartitionedTopic)( - Produced.`with`(ScalaSerdes.Thrift[EnrichmentKey], ScalaSerdes.Thrift[EnrichmentEnvelop])) - } - } - - override def streamsProperties(config: KafkaStreamsConfig): KafkaStreamsConfig = { - super - .streamsProperties(config) - .consumer.groupId(KafkaGroupId(ApplicationId)) - .consumer.clientId(s"$ApplicationId-consumer") - .consumer.requestTimeout(30.seconds) - .consumer.sessionTimeout(30.seconds) - .consumer.fetchMin(1.megabyte) - .consumer.fetchMax(5.megabyte) - .consumer.receiveBuffer(32.megabytes) - .consumer.maxPollInterval(1.minute) - .consumer.maxPollRecords(50000) - .producer.clientId(s"$ApplicationId-producer") - .producer.batchSize(16.kilobytes) - .producer.bufferMemorySize(256.megabyte) - .producer.requestTimeout(30.seconds) - .producer.compressionType(CompressionType.LZ4) - .producer.ackMode(AckMode.ALL) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/FavoriteArchivalEventsService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/FavoriteArchivalEventsService.docx new file mode 100644 index 000000000..c1ddada39 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/FavoriteArchivalEventsService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/FavoriteArchivalEventsService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/FavoriteArchivalEventsService.scala deleted file mode 100644 index b4014a13e..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/FavoriteArchivalEventsService.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.timelineservice.fanout.thriftscala.FavoriteArchivalEvent -import com.twitter.unified_user_actions.service.module.KafkaProcessorFavoriteArchivalEventsModule - -object FavoriteArchivalEventsServiceMain extends FavoriteArchivalEventsService - -class FavoriteArchivalEventsService extends TwitterServer { - - override val modules = Seq( - KafkaProcessorFavoriteArchivalEventsModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, FavoriteArchivalEvent]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceService.docx new file mode 100644 index 000000000..8a1c1e98b Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceService.scala deleted file mode 100644 index f0db8032b..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceService.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.iesource.thriftscala.InteractionEvent -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.service.module.KafkaProcessorRekeyUuaIesourceModule - -object RekeyUuaIesourceServiceMain extends RekeyUuaIesourceService - -class RekeyUuaIesourceService extends TwitterServer { - - override val modules = Seq( - KafkaProcessorRekeyUuaIesourceModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, InteractionEvent]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaService.docx new file mode 100644 index 000000000..54cdc99d7 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaService.scala deleted file mode 100644 index 6928df498..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RekeyUuaService.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.service.module.KafkaProcessorRekeyUuaModule -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction - -object RekeyUuaServiceMain extends RekeyUuaService - -class RekeyUuaService extends TwitterServer { - - override val modules = Seq( - KafkaProcessorRekeyUuaModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, UnifiedUserAction]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RetweetArchivalEventsService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RetweetArchivalEventsService.docx new file mode 100644 index 000000000..cffe4f3dd Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RetweetArchivalEventsService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RetweetArchivalEventsService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RetweetArchivalEventsService.scala deleted file mode 100644 index dcbbc8bd6..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/RetweetArchivalEventsService.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.tweetypie.thriftscala.RetweetArchivalEvent -import com.twitter.unified_user_actions.service.module.KafkaProcessorRetweetArchivalEventsModule - -object RetweetArchivalEventsServiceMain extends RetweetArchivalEventsService - -class RetweetArchivalEventsService extends TwitterServer { - - override val modules = Seq( - KafkaProcessorRetweetArchivalEventsModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, RetweetArchivalEvent]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/SocialGraphService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/SocialGraphService.docx new file mode 100644 index 000000000..8035f912a Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/SocialGraphService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/SocialGraphService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/SocialGraphService.scala deleted file mode 100644 index 89917d1ec..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/SocialGraphService.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.socialgraph.thriftscala.WriteEvent -import com.twitter.unified_user_actions.service.module.KafkaProcessorSocialGraphModule - -object SocialGraphServiceMain extends SocialGraphService - -class SocialGraphService extends TwitterServer { - override val modules = Seq( - KafkaProcessorSocialGraphModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, WriteEvent]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TlsFavsService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TlsFavsService.docx new file mode 100644 index 000000000..f6ba16e96 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TlsFavsService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TlsFavsService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TlsFavsService.scala deleted file mode 100644 index a96891c46..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TlsFavsService.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.timelineservice.thriftscala.ContextualizedFavoriteEvent -import com.twitter.unified_user_actions.service.module.KafkaProcessorTlsFavsModule - -object TlsFavsServiceMain extends TlsFavsService - -class TlsFavsService extends TwitterServer { - - override val modules = Seq( - KafkaProcessorTlsFavsModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, ContextualizedFavoriteEvent]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TweetypieEventService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TweetypieEventService.docx new file mode 100644 index 000000000..1c3d281df Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TweetypieEventService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TweetypieEventService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TweetypieEventService.scala deleted file mode 100644 index c8516492d..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/TweetypieEventService.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.tweetypie.thriftscala.TweetEvent -import com.twitter.unified_user_actions.service.module.KafkaProcessorTweetypieEventModule - -object TweetypieEventServiceMain extends TweetypieEventService - -class TweetypieEventService extends TwitterServer { - - override val modules = Seq( - KafkaProcessorTweetypieEventModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, TweetEvent]] - closeOnExit(processor) - processor.start() - } - -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/UserModificationService.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/UserModificationService.docx new file mode 100644 index 000000000..2906b3b14 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/UserModificationService.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/UserModificationService.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/UserModificationService.scala deleted file mode 100644 index ff16d6334..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/UserModificationService.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.gizmoduck.thriftscala.UserModification -import com.twitter.inject.server.TwitterServer -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.service.module.KafkaProcessorUserModificationModule - -object UserModificationServiceMain extends UserModificationService - -class UserModificationService extends TwitterServer { - override val modules = Seq( - KafkaProcessorUserModificationModule, - DeciderModule - ) - - override protected def setup(): Unit = {} - - override protected def start(): Unit = { - val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, UserModification]] - closeOnExit(processor) - processor.start() - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/BUILD b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/BUILD deleted file mode 100644 index 9586c637d..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/BUILD +++ /dev/null @@ -1,482 +0,0 @@ -scala_library( - name = "decider-utils", - sources = [ - "DeciderUtils.scala", - "TopicsMapping.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - "decider/src/main/scala", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) - -scala_library( - name = "base", - sources = [ - "FlagsModule.scala", - "KafkaProcessorProvider.scala", - "ZoneFiltering.scala", - ], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "tls-favs", - sources = [ - "FlagsModule.scala", - "KafkaProcessorProvider.scala", - "KafkaProcessorTlsFavsModule.scala", - "ZoneFiltering.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "client-event", - sources = [ - "FlagsModule.scala", - "KafkaProcessorClientEventModule.scala", - "KafkaProcessorProvider.scala", - "TopicsMapping.scala", - "ZoneFiltering.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - - -scala_library( - name = "tweetypie-event", - sources = [ - "FlagsModule.scala", - "KafkaProcessorProvider.scala", - "KafkaProcessorTweetypieEventModule.scala", - "ZoneFiltering.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "social-graph-event", - sources = [ - "FlagsModule.scala", - "KafkaProcessorProvider.scala", - "KafkaProcessorSocialGraphModule.scala", - "ZoneFiltering.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "email-notification-event", - sources = [ - "FlagsModule.scala", - "KafkaProcessorEmailNotificationEventModule.scala", - "KafkaProcessorProvider.scala", - "ZoneFiltering.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "user-modification-event", - sources = [ - "FlagsModule.scala", - "KafkaProcessorProvider.scala", - "KafkaProcessorUserModificationModule.scala", - "ZoneFiltering.scala", - ], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "ads-callback-engagements", - sources = [ - "FlagsModule.scala", - "KafkaProcessorAdsCallbackEngagementsModule.scala", - "KafkaProcessorProvider.scala", - "ZoneFiltering.scala", - ], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/ads_callback_engagements", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "favorite-archival-events", - sources = [ - "FlagsModule.scala", - "KafkaProcessorFavoriteArchivalEventsModule.scala", - "KafkaProcessorProvider.scala", - "ZoneFiltering.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "retweet-archival-events", - sources = [ - "FlagsModule.scala", - "KafkaProcessorProvider.scala", - "KafkaProcessorRetweetArchivalEventsModule.scala", - "ZoneFiltering.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "rekey-uua", - sources = [ - "FlagsModule.scala", - "KafkaProcessorProvider.scala", - "KafkaProcessorRekeyUuaModule.scala", - "ZoneFiltering.scala", - ], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "rekey-uua-iesource", - sources = [ - "FlagsModule.scala", - "KafkaProcessorProvider.scala", - "KafkaProcessorRekeyUuaIesourceModule.scala", - "ZoneFiltering.scala", - ], - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = [ - ":decider-utils", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/apache/kafka:kafka-clients", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "finatra/inject/inject-thrift-client/src/main/scala", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "twitter-server/server/src/main/scala", - "unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates", - "unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - "util/util-core:scala", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-slf4j-api/src/main/scala", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) - -scala_library( - name = "graphql-client", - sources = [ - "ClientIdModule.scala", - "GraphqlClientProviderModule.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client", - "finagle/finagle-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "graphql/thrift/src/main/thrift/com/twitter/graphql:graphql-scala", - "twitter-server/server/src/main/scala", - ], -) - -scala_library( - name = "cache", - sources = [ - "CacheModule.scala", - ], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "featureswitches/dynmap/src/main/scala/com/twitter/dynmap:dynmap-core", - "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules", - "graphql/thrift/src/main/thrift/com/twitter/graphql:graphql-scala", - "twitter-server/server/src/main/scala", - "unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache", - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - "util/util-cache-guava/src/main/scala", - "util/util-cache/src/main/scala", - ], -) diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/BUILD.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/BUILD.docx new file mode 100644 index 000000000..321e20dae Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/BUILD.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/CacheModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/CacheModule.docx new file mode 100644 index 000000000..76ad6b1ba Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/CacheModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/CacheModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/CacheModule.scala deleted file mode 100644 index 295c6ee39..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/CacheModule.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.common.cache.CacheBuilder -import com.google.inject.Provides -import com.twitter.dynmap.DynMap -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.unified_user_actions.enricher.hcache.LocalCache -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.util.Future -import java.util.concurrent.TimeUnit -import javax.inject.Singleton - -object CacheModule extends TwitterModule { - private final val localCacheTtlFlagName = "local.cache.ttl.seconds" - private final val localCacheMaxSizeFlagName = "local.cache.max.size" - - flag[Long]( - name = localCacheTtlFlagName, - default = 1800L, - help = "Local Cache's TTL in seconds" - ) - - flag[Long]( - name = localCacheMaxSizeFlagName, - default = 1000L, - help = "Local Cache's max size" - ) - - @Provides - @Singleton - def providesLocalCache( - @Flag(localCacheTtlFlagName) localCacheTtlFlag: Long, - @Flag(localCacheMaxSizeFlagName) localCacheMaxSizeFlag: Long, - statsReceiver: StatsReceiver - ): LocalCache[EnrichmentKey, DynMap] = { - val underlying = CacheBuilder - .newBuilder() - .expireAfterWrite(localCacheTtlFlag, TimeUnit.SECONDS) - .maximumSize(localCacheMaxSizeFlag) - .build[EnrichmentKey, Future[DynMap]]() - - new LocalCache[EnrichmentKey, DynMap]( - underlying = underlying, - statsReceiver = statsReceiver.scope("enricherLocalCache")) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ClientIdModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ClientIdModule.docx new file mode 100644 index 000000000..704944cc0 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ClientIdModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ClientIdModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ClientIdModule.scala deleted file mode 100644 index 9358834b9..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ClientIdModule.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.finagle.thrift.ClientId -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import javax.inject.Singleton - -object ClientIdModule extends TwitterModule { - private final val flagName = "thrift.client.id" - - flag[String]( - name = flagName, - help = "Thrift Client ID" - ) - - @Provides - @Singleton - def providesClientId( - @Flag(flagName) thriftClientId: String, - ): ClientId = ClientId( - name = thriftClientId - ) -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/DeciderUtils.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/DeciderUtils.docx new file mode 100644 index 000000000..2594a69fd Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/DeciderUtils.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/DeciderUtils.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/DeciderUtils.scala deleted file mode 100644 index f38a9ef92..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/DeciderUtils.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction - -sealed trait DeciderUtils { - def shouldPublish(decider: Decider, uua: UnifiedUserAction, sinkTopic: String): Boolean -} - -object DefaultDeciderUtils extends DeciderUtils { - override def shouldPublish(decider: Decider, uua: UnifiedUserAction, sinkTopic: String): Boolean = - decider.isAvailable(feature = s"Publish${uua.actionType}", Some(RandomRecipient)) -} - -object ClientEventDeciderUtils extends DeciderUtils { - override def shouldPublish(decider: Decider, uua: UnifiedUserAction, sinkTopic: String): Boolean = - decider.isAvailable( - feature = s"Publish${uua.actionType}", - Some(RandomRecipient)) && (uua.actionType match { - // for heavy impressions UUA only publishes to the "all" topic, not the engagementsOnly topic. - case ActionType.ClientTweetLingerImpression | ActionType.ClientTweetRenderImpression => - sinkTopic == TopicsMapping().all - case _ => true - }) -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/FlagsModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/FlagsModule.docx new file mode 100644 index 000000000..6d3fa6d9d Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/FlagsModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/FlagsModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/FlagsModule.scala deleted file mode 100644 index 62cb09825..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/FlagsModule.scala +++ /dev/null @@ -1,172 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.twitter.inject.TwitterModule -import com.twitter.unified_user_actions.kafka.ClientConfigs -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging - -object FlagsModule extends TwitterModule with Logging { - // Twitter - final val cluster = "cluster" - - // Required - final val kafkaSourceCluster = ClientConfigs.kafkaBootstrapServerConfig - final val kafkaDestCluster = ClientConfigs.kafkaBootstrapServerRemoteDestConfig - final val kafkaSourceTopic = "kafka.source.topic" - final val kafkaSinkTopics = "kafka.sink.topics" - final val kafkaGroupId = ClientConfigs.kafkaGroupIdConfig - final val kafkaProducerClientId = ClientConfigs.producerClientIdConfig - final val kafkaMaxPendingRequests = ClientConfigs.kafkaMaxPendingRequestsConfig - final val kafkaWorkerThreads = ClientConfigs.kafkaWorkerThreadsConfig - - // Optional - /// Authentication - final val enableTrustStore = ClientConfigs.enableTrustStore - final val trustStoreLocation = ClientConfigs.trustStoreLocationConfig - - /// Consumer - final val commitInterval = ClientConfigs.kafkaCommitIntervalConfig - final val maxPollRecords = ClientConfigs.consumerMaxPollRecordsConfig - final val maxPollInterval = ClientConfigs.consumerMaxPollIntervalConfig - final val sessionTimeout = ClientConfigs.consumerSessionTimeoutConfig - final val fetchMax = ClientConfigs.consumerFetchMaxConfig - final val fetchMin = ClientConfigs.consumerFetchMinConfig - final val receiveBuffer = ClientConfigs.consumerReceiveBufferSizeConfig - /// Producer - final val batchSize = ClientConfigs.producerBatchSizeConfig - final val linger = ClientConfigs.producerLingerConfig - final val bufferMem = ClientConfigs.producerBufferMemConfig - final val compressionType = ClientConfigs.compressionConfig - final val retries = ClientConfigs.retriesConfig - final val retryBackoff = ClientConfigs.retryBackoffConfig - final val requestTimeout = ClientConfigs.producerRequestTimeoutConfig - - // Twitter - flag[String]( - name = cluster, - help = "The zone (or DC) that this service runs, used to potentially filter events" - ) - - // Required - flag[String]( - name = kafkaSourceCluster, - help = ClientConfigs.kafkaBootstrapServerHelp - ) - flag[String]( - name = kafkaDestCluster, - help = ClientConfigs.kafkaBootstrapServerRemoteDestHelp - ) - flag[String]( - name = kafkaSourceTopic, - help = "Name of the source Kafka topic" - ) - flag[Seq[String]]( - name = kafkaSinkTopics, - help = "A list of sink Kafka topics, separated by comma (,)" - ) - flag[String]( - name = kafkaGroupId, - help = ClientConfigs.kafkaGroupIdHelp - ) - flag[String]( - name = kafkaProducerClientId, - help = ClientConfigs.producerClientIdHelp - ) - flag[Int]( - name = kafkaMaxPendingRequests, - help = ClientConfigs.kafkaMaxPendingRequestsHelp - ) - flag[Int]( - name = kafkaWorkerThreads, - help = ClientConfigs.kafkaWorkerThreadsHelp - ) - - // Optional - /// Authentication - flag[Boolean]( - name = enableTrustStore, - default = ClientConfigs.enableTrustStoreDefault, - help = ClientConfigs.enableTrustStoreHelp - ) - flag[String]( - name = trustStoreLocation, - default = ClientConfigs.trustStoreLocationDefault, - help = ClientConfigs.trustStoreLocationHelp - ) - - /// Consumer - flag[Duration]( - name = commitInterval, - default = ClientConfigs.kafkaCommitIntervalDefault, - help = ClientConfigs.kafkaCommitIntervalHelp - ) - flag[Int]( - name = maxPollRecords, - default = ClientConfigs.consumerMaxPollRecordsDefault, - help = ClientConfigs.consumerMaxPollRecordsHelp - ) - flag[Duration]( - name = maxPollInterval, - default = ClientConfigs.consumerMaxPollIntervalDefault, - help = ClientConfigs.consumerMaxPollIntervalHelp - ) - flag[Duration]( - name = sessionTimeout, - default = ClientConfigs.consumerSessionTimeoutDefault, - help = ClientConfigs.consumerSessionTimeoutHelp - ) - flag[StorageUnit]( - name = fetchMax, - default = ClientConfigs.consumerFetchMaxDefault, - help = ClientConfigs.consumerFetchMaxHelp - ) - flag[StorageUnit]( - name = fetchMin, - default = ClientConfigs.consumerFetchMinDefault, - help = ClientConfigs.consumerFetchMinHelp - ) - flag[StorageUnit]( - name = receiveBuffer, - default = ClientConfigs.consumerReceiveBufferSizeDefault, - help = ClientConfigs.consumerReceiveBufferSizeHelp - ) - - /// Producer - flag[StorageUnit]( - name = batchSize, - default = ClientConfigs.producerBatchSizeDefault, - help = ClientConfigs.producerBatchSizeHelp - ) - flag[Duration]( - name = linger, - default = ClientConfigs.producerLingerDefault, - help = ClientConfigs.producerLingerHelp - ) - flag[StorageUnit]( - name = bufferMem, - default = ClientConfigs.producerBufferMemDefault, - help = ClientConfigs.producerBufferMemHelp - ) - flag[CompressionTypeFlag]( - name = compressionType, - default = ClientConfigs.compressionDefault, - help = ClientConfigs.compressionHelp - ) - flag[Int]( - name = retries, - default = ClientConfigs.retriesDefault, - help = ClientConfigs.retriesHelp - ) - flag[Duration]( - name = retryBackoff, - default = ClientConfigs.retryBackoffDefault, - help = ClientConfigs.retryBackoffHelp - ) - flag[Duration]( - name = requestTimeout, - default = ClientConfigs.producerRequestTimeoutDefault, - help = ClientConfigs.producerRequestTimeoutHelp - ) -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/GraphqlClientProviderModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/GraphqlClientProviderModule.docx new file mode 100644 index 000000000..391c368c3 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/GraphqlClientProviderModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/GraphqlClientProviderModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/GraphqlClientProviderModule.scala deleted file mode 100644 index 6a9973655..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/GraphqlClientProviderModule.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax -import com.twitter.finagle.ssl.OpportunisticTls -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.RichClientParam -import com.twitter.graphql.thriftscala.GraphqlExecutionService -import com.twitter.inject.TwitterModule -import com.twitter.util.Duration -import javax.inject.Singleton - -object GraphqlClientProviderModule extends TwitterModule { - private def buildClient(serviceIdentifier: ServiceIdentifier, clientId: ClientId) = - ThriftMux.client - .withRequestTimeout(Duration.fromSeconds(5)) - .withMutualTls(serviceIdentifier) - .withOpportunisticTls(OpportunisticTls.Required) - .withClientId(clientId) - .newService("/s/graphql-service/graphql-api:thrift") - - def buildGraphQlClient( - serviceIdentifer: ServiceIdentifier, - clientId: ClientId - ): GraphqlExecutionService.FinagledClient = { - val client = buildClient(serviceIdentifer, clientId) - new GraphqlExecutionService.FinagledClient(client, RichClientParam()) - } - - @Provides - @Singleton - def providesGraphQlClient( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId - ): GraphqlExecutionService.FinagledClient = - buildGraphQlClient( - serviceIdentifier, - clientId - ) -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorAdsCallbackEngagementsModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorAdsCallbackEngagementsModule.docx new file mode 100644 index 000000000..22e5d7a49 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorAdsCallbackEngagementsModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorAdsCallbackEngagementsModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorAdsCallbackEngagementsModule.scala deleted file mode 100644 index 404eafa23..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorAdsCallbackEngagementsModule.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.ads.spendserver.thriftscala.SpendServerEvent -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.adapter.ads_callback_engagements.AdsCallbackEngagementsAdapter -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton - -object KafkaProcessorAdsCallbackEngagementsModule extends TwitterModule with Logging { - override def modules = Seq(FlagsModule) - - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, SpendServerEvent] = { - KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[SpendServerEvent](statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = new AdsCallbackEngagementsAdapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorClientEventModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorClientEventModule.docx new file mode 100644 index 000000000..4af4b3449 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorClientEventModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorClientEventModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorClientEventModule.scala deleted file mode 100644 index b6f36589c..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorClientEventModule.scala +++ /dev/null @@ -1,142 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.producers.BlockingFinagleKafkaProducer -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.kafka.client.headers.Zone -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.adapter.client_event.ClientEventAdapter -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.unified_user_actions.service.module.KafkaProcessorProvider.updateActionTypeCounters -import com.twitter.unified_user_actions.service.module.KafkaProcessorProvider.updateProcessingTimeStats -import com.twitter.unified_user_actions.service.module.KafkaProcessorProvider.updateProductSurfaceTypeCounters -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.header.Headers - -object KafkaProcessorClientEventModule extends TwitterModule with Logging { - override def modules: Seq[FlagsModule.type] = Seq(FlagsModule) - - private val clientEventAdapter = new ClientEventAdapter - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.fetchMin) fetchMin: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, LogEvent] = { - KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[LogEvent](statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - fetchMin = fetchMin, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = clientEventAdapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - produceOpt = Some(clientEventProducer), - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - ) - } - - /** - * ClientEvent producer is different from the defaultProducer. - * While the defaultProducer publishes every event to all sink topics, ClientEventProducer (this producer) requires - * exactly 2 sink topics: Topic with all events (impressions and engagements) and Topic with engagements only. - * And the publishing is based the action type. - */ - def clientEventProducer( - producer: BlockingFinagleKafkaProducer[UnKeyed, UnifiedUserAction], - k: UnKeyed, - v: UnifiedUserAction, - sinkTopic: String, - headers: Headers, - statsReceiver: StatsReceiver, - decider: Decider - ): Future[Unit] = - if (ClientEventDeciderUtils.shouldPublish(decider = decider, uua = v, sinkTopic = sinkTopic)) { - updateActionTypeCounters(statsReceiver, v, sinkTopic) - updateProductSurfaceTypeCounters(statsReceiver, v, sinkTopic) - updateProcessingTimeStats(statsReceiver, v) - - // If we were to enable xDC replicator, then we can safely remove the Zone header since xDC - // replicator works in the following way: - // - If the message does not have a header, the replicator will assume it is local and - // set the header, copy the message - // - If the message has a header that is the local zone, the replicator will copy the message - // - If the message has a header for a different zone, the replicator will drop the message - producer - .send( - new ProducerRecord[UnKeyed, UnifiedUserAction]( - sinkTopic, - null, - k, - v, - headers.remove(Zone.Key))) - .onSuccess { _ => statsReceiver.counter("publishSuccess", sinkTopic).incr() } - .onFailure { e: Throwable => - statsReceiver.counter("publishFailure", sinkTopic).incr() - error(s"Publish error to topic $sinkTopic: $e") - }.unit - } else Future.Unit -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorEmailNotificationEventModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorEmailNotificationEventModule.docx new file mode 100644 index 000000000..515e26e94 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorEmailNotificationEventModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorEmailNotificationEventModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorEmailNotificationEventModule.scala deleted file mode 100644 index 116792b7e..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorEmailNotificationEventModule.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.ibis.thriftscala.NotificationScribe -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.unified_user_actions.adapter.email_notification_event.EmailNotificationEventAdapter -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton - -object KafkaProcessorEmailNotificationEventModule extends TwitterModule with Logging { - override def modules = Seq(FlagsModule) - private val notificationEventAdapter = new EmailNotificationEventAdapter - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, NotificationScribe] = { - KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[NotificationScribe](statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = notificationEventAdapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - maybeProcess = ZoneFiltering.localDCFiltering - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorFavoriteArchivalEventsModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorFavoriteArchivalEventsModule.docx new file mode 100644 index 000000000..cd55c8077 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorFavoriteArchivalEventsModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorFavoriteArchivalEventsModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorFavoriteArchivalEventsModule.scala deleted file mode 100644 index 3e8f5592b..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorFavoriteArchivalEventsModule.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.adapter.favorite_archival_events.FavoriteArchivalEventsAdapter -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.timelineservice.fanout.thriftscala.FavoriteArchivalEvent -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton - -object KafkaProcessorFavoriteArchivalEventsModule extends TwitterModule with Logging { - override def modules = Seq(FlagsModule) - - private val adapter = new FavoriteArchivalEventsAdapter - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, FavoriteArchivalEvent] = { - KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[FavoriteArchivalEvent](statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = adapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorProvider.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorProvider.docx new file mode 100644 index 000000000..6c847a7c8 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorProvider.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorProvider.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorProvider.scala deleted file mode 100644 index da8ad39f9..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorProvider.scala +++ /dev/null @@ -1,271 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.twitter.decider.Decider -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.producers.BlockingFinagleKafkaProducer -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.kafka.client.headers.Implicits._ -import com.twitter.kafka.client.headers.Zone -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.kafka.ClientConfigs -import com.twitter.unified_user_actions.kafka.ClientProviders -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.header.Headers -import org.apache.kafka.common.record.CompressionType -import org.apache.kafka.common.serialization.Deserializer - -object KafkaProcessorProvider extends Logging { - lazy val actionTypeStatsCounterMap: collection.mutable.Map[String, Counter] = - collection.mutable.Map.empty - lazy val productSurfaceTypeStatsCounterMap: collection.mutable.Map[String, Counter] = - collection.mutable.Map.empty - - def updateActionTypeCounters( - statsReceiver: StatsReceiver, - v: UnifiedUserAction, - topic: String - ): Unit = { - val actionType = v.actionType.name - val actionTypeAndTopicKey = s"$actionType-$topic" - actionTypeStatsCounterMap.get(actionTypeAndTopicKey) match { - case Some(actionCounter) => actionCounter.incr() - case _ => - actionTypeStatsCounterMap(actionTypeAndTopicKey) = - statsReceiver.counter("uuaActionType", topic, actionType) - actionTypeStatsCounterMap(actionTypeAndTopicKey).incr() - } - } - - def updateProductSurfaceTypeCounters( - statsReceiver: StatsReceiver, - v: UnifiedUserAction, - topic: String - ): Unit = { - val productSurfaceType = v.productSurface.map(_.name).getOrElse("null") - val productSurfaceTypeAndTopicKey = s"$productSurfaceType-$topic" - productSurfaceTypeStatsCounterMap.get(productSurfaceTypeAndTopicKey) match { - case Some(productSurfaceCounter) => productSurfaceCounter.incr() - case _ => - productSurfaceTypeStatsCounterMap(productSurfaceTypeAndTopicKey) = - statsReceiver.counter("uuaProductSurfaceType", topic, productSurfaceType) - productSurfaceTypeStatsCounterMap(productSurfaceTypeAndTopicKey).incr() - } - } - - def updateProcessingTimeStats(statsReceiver: StatsReceiver, v: UnifiedUserAction): Unit = { - statsReceiver - .stat("uuaProcessingTimeDiff").add( - v.eventMetadata.receivedTimestampMs - v.eventMetadata.sourceTimestampMs) - } - - def defaultProducer( - producer: BlockingFinagleKafkaProducer[UnKeyed, UnifiedUserAction], - k: UnKeyed, - v: UnifiedUserAction, - sinkTopic: String, - headers: Headers, - statsReceiver: StatsReceiver, - decider: Decider, - ): Future[Unit] = - if (DefaultDeciderUtils.shouldPublish(decider = decider, uua = v, sinkTopic = sinkTopic)) { - updateActionTypeCounters(statsReceiver, v, sinkTopic) - updateProcessingTimeStats(statsReceiver, v) - - // If we were to enable xDC replicator, then we can safely remove the Zone header since xDC - // replicator works in the following way: - // - If the message does not have a header, the replicator will assume it is local and - // set the header, copy the message - // - If the message has a header that is the local zone, the replicator will copy the message - // - If the message has a header for a different zone, the replicator will drop the message - producer - .send( - new ProducerRecord[UnKeyed, UnifiedUserAction]( - sinkTopic, - null, - k, - v, - headers.remove(Zone.Key))) - .onSuccess { _ => statsReceiver.counter("publishSuccess", sinkTopic).incr() } - .onFailure { e: Throwable => - statsReceiver.counter("publishFailure", sinkTopic).incr() - error(s"Publish error to topic $sinkTopic: $e") - }.unit - } else Future.Unit - - /** - * The default AtLeastOnceProcessor mainly for consuming from a single Kafka topic -> process/adapt -> publish to - * the single sink Kafka topic. - * - * Important Note: Currently all sink topics share the same Kafka producer!!! If you need to create different - * producers for different topics, you would need to create a customized function like this one. - */ - def provideDefaultAtLeastOnceProcessor[K, V]( - name: String, - kafkaSourceCluster: String, - kafkaGroupId: String, - kafkaSourceTopic: String, - sourceKeyDeserializer: Deserializer[K], - sourceValueDeserializer: Deserializer[V], - commitInterval: Duration = ClientConfigs.kafkaCommitIntervalDefault, - maxPollRecords: Int = ClientConfigs.consumerMaxPollRecordsDefault, - maxPollInterval: Duration = ClientConfigs.consumerMaxPollIntervalDefault, - sessionTimeout: Duration = ClientConfigs.consumerSessionTimeoutDefault, - fetchMax: StorageUnit = ClientConfigs.consumerFetchMaxDefault, - fetchMin: StorageUnit = ClientConfigs.consumerFetchMinDefault, - receiveBuffer: StorageUnit = ClientConfigs.consumerReceiveBufferSizeDefault, - processorMaxPendingRequests: Int, - processorWorkerThreads: Int, - adapter: AbstractAdapter[V, UnKeyed, UnifiedUserAction], - kafkaSinkTopics: Seq[String], - kafkaDestCluster: String, - kafkaProducerClientId: String, - batchSize: StorageUnit = ClientConfigs.producerBatchSizeDefault, - linger: Duration = ClientConfigs.producerLingerDefault, - bufferMem: StorageUnit = ClientConfigs.producerBufferMemDefault, - compressionType: CompressionType = ClientConfigs.compressionDefault.compressionType, - retries: Int = ClientConfigs.retriesDefault, - retryBackoff: Duration = ClientConfigs.retryBackoffDefault, - requestTimeout: Duration = ClientConfigs.producerRequestTimeoutDefault, - produceOpt: Option[ - (BlockingFinagleKafkaProducer[UnKeyed, UnifiedUserAction], UnKeyed, UnifiedUserAction, String, - Headers, StatsReceiver, Decider) => Future[Unit] - ] = None, - trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault), - statsReceiver: StatsReceiver, - decider: Decider, - zone: Zone, - maybeProcess: (ConsumerRecord[K, V], Zone) => Boolean = ZoneFiltering.localDCFiltering[K, V] _, - ): AtLeastOnceProcessor[K, V] = { - - lazy val singletonProducer = ClientProviders.mkProducer[UnKeyed, UnifiedUserAction]( - bootstrapServer = kafkaDestCluster, - clientId = kafkaProducerClientId, - keySerde = UnKeyedSerde.serializer, - valueSerde = ScalaSerdes.Thrift[UnifiedUserAction].serializer, - idempotence = false, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - trustStoreLocationOpt = trustStoreLocationOpt, - ) - - mkAtLeastOnceProcessor[K, V, UnKeyed, UnifiedUserAction]( - name = name, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = sourceKeyDeserializer, - sourceValueDeserializer = sourceValueDeserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - fetchMin = fetchMin, - receiveBuffer = receiveBuffer, - processorMaxPendingRequests = processorMaxPendingRequests, - processorWorkerThreads = processorWorkerThreads, - adapter = adapter, - kafkaProducersAndSinkTopics = - kafkaSinkTopics.map(sinkTopic => (singletonProducer, sinkTopic)), - produce = produceOpt.getOrElse(defaultProducer), - trustStoreLocationOpt = trustStoreLocationOpt, - statsReceiver = statsReceiver, - decider = decider, - zone = zone, - maybeProcess = maybeProcess, - ) - } - - /** - * A common AtLeastOnceProcessor provider - */ - def mkAtLeastOnceProcessor[K, V, OUTK, OUTV]( - name: String, - kafkaSourceCluster: String, - kafkaGroupId: String, - kafkaSourceTopic: String, - sourceKeyDeserializer: Deserializer[K], - sourceValueDeserializer: Deserializer[V], - commitInterval: Duration = ClientConfigs.kafkaCommitIntervalDefault, - maxPollRecords: Int = ClientConfigs.consumerMaxPollRecordsDefault, - maxPollInterval: Duration = ClientConfigs.consumerMaxPollIntervalDefault, - sessionTimeout: Duration = ClientConfigs.consumerSessionTimeoutDefault, - fetchMax: StorageUnit = ClientConfigs.consumerFetchMaxDefault, - fetchMin: StorageUnit = ClientConfigs.consumerFetchMinDefault, - receiveBuffer: StorageUnit = ClientConfigs.consumerReceiveBufferSizeDefault, - processorMaxPendingRequests: Int, - processorWorkerThreads: Int, - adapter: AbstractAdapter[V, OUTK, OUTV], - kafkaProducersAndSinkTopics: Seq[(BlockingFinagleKafkaProducer[OUTK, OUTV], String)], - produce: (BlockingFinagleKafkaProducer[OUTK, OUTV], OUTK, OUTV, String, Headers, StatsReceiver, - Decider) => Future[Unit], - trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault), - statsReceiver: StatsReceiver, - decider: Decider, - zone: Zone, - maybeProcess: (ConsumerRecord[K, V], Zone) => Boolean = ZoneFiltering.localDCFiltering[K, V] _, - ): AtLeastOnceProcessor[K, V] = { - val threadSafeKafkaClient = - ClientProviders.mkConsumer[K, V]( - bootstrapServer = kafkaSourceCluster, - keySerde = sourceKeyDeserializer, - valueSerde = sourceValueDeserializer, - groupId = kafkaGroupId, - autoCommit = false, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - fetchMin = fetchMin, - receiveBuffer = receiveBuffer, - trustStoreLocationOpt = trustStoreLocationOpt - ) - - def publish( - event: ConsumerRecord[K, V] - ): Future[Unit] = { - statsReceiver.counter("consumedEvents").incr() - - if (maybeProcess(event, zone)) - Future - .collect( - adapter - .adaptOneToKeyedMany(event.value, statsReceiver) - .flatMap { - case (k, v) => - kafkaProducersAndSinkTopics.map { - case (producer, sinkTopic) => - produce(producer, k, v, sinkTopic, event.headers(), statsReceiver, decider) - } - }).unit - else - Future.Unit - } - - AtLeastOnceProcessor[K, V]( - name = name, - topic = kafkaSourceTopic, - consumer = threadSafeKafkaClient, - processor = publish, - maxPendingRequests = processorMaxPendingRequests, - workerThreads = processorWorkerThreads, - commitIntervalMs = commitInterval.inMilliseconds, - statsReceiver = statsReceiver - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaIesourceModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaIesourceModule.docx new file mode 100644 index 000000000..23f549ab6 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaIesourceModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaIesourceModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaIesourceModule.scala deleted file mode 100644 index 466fbec0c..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaIesourceModule.scala +++ /dev/null @@ -1,207 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.decider.SimpleRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.producers.BlockingFinagleKafkaProducer -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.iesource.thriftscala.InteractionEvent -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.kafka.client.headers.Zone -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.adapter.uua_aggregates.RekeyUuaFromInteractionEventsAdapter -import com.twitter.unified_user_actions.kafka.ClientConfigs -import com.twitter.unified_user_actions.kafka.ClientProviders -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.thriftscala.KeyedUuaTweet -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.header.Headers -import org.apache.kafka.common.record.CompressionType -import javax.inject.Singleton -import javax.inject.Inject - -object KafkaProcessorRekeyUuaIesourceModule extends TwitterModule with Logging { - override def modules = Seq(FlagsModule) - - private val adapter = new RekeyUuaFromInteractionEventsAdapter - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - @Inject - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.receiveBuffer) receiveBuffer: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, InteractionEvent] = { - provideAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - receiveBuffer = receiveBuffer, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = adapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - maybeProcess = ZoneFiltering.noFiltering - ) - } - - def producer( - producer: BlockingFinagleKafkaProducer[Long, KeyedUuaTweet], - k: Long, - v: KeyedUuaTweet, - sinkTopic: String, - headers: Headers, - statsReceiver: StatsReceiver, - decider: Decider, - ): Future[Unit] = - if (decider.isAvailable(feature = s"RekeyUUAIesource${v.actionType}", Some(SimpleRecipient(k)))) - // If we were to enable xDC replicator, then we can safely remove the Zone header since xDC - // replicator works in the following way: - // - If the message does not have a header, the replicator will assume it is local and - // set the header, copy the message - // - If the message has a header that is the local zone, the replicator will copy the message - // - If the message has a header for a different zone, the replicator will drop the message - producer - .send(new ProducerRecord[Long, KeyedUuaTweet](sinkTopic, null, k, v, headers)) - .onSuccess { _ => statsReceiver.counter("publishSuccess", sinkTopic).incr() } - .onFailure { e: Throwable => - statsReceiver.counter("publishFailure", sinkTopic).incr() - error(s"Publish error to topic $sinkTopic: $e") - }.unit - else Future.Unit - - def provideAtLeastOnceProcessor( - name: String, - kafkaSourceCluster: String, - kafkaGroupId: String, - kafkaSourceTopic: String, - commitInterval: Duration = ClientConfigs.kafkaCommitIntervalDefault, - maxPollRecords: Int = ClientConfigs.consumerMaxPollRecordsDefault, - maxPollInterval: Duration = ClientConfigs.consumerMaxPollIntervalDefault, - sessionTimeout: Duration = ClientConfigs.consumerSessionTimeoutDefault, - fetchMax: StorageUnit = ClientConfigs.consumerFetchMaxDefault, - fetchMin: StorageUnit = ClientConfigs.consumerFetchMinDefault, - receiveBuffer: StorageUnit = ClientConfigs.consumerReceiveBufferSizeDefault, - processorMaxPendingRequests: Int, - processorWorkerThreads: Int, - adapter: AbstractAdapter[InteractionEvent, Long, KeyedUuaTweet], - kafkaSinkTopics: Seq[String], - kafkaDestCluster: String, - kafkaProducerClientId: String, - batchSize: StorageUnit = ClientConfigs.producerBatchSizeDefault, - linger: Duration = ClientConfigs.producerLingerDefault, - bufferMem: StorageUnit = ClientConfigs.producerBufferMemDefault, - compressionType: CompressionType = ClientConfigs.compressionDefault.compressionType, - retries: Int = ClientConfigs.retriesDefault, - retryBackoff: Duration = ClientConfigs.retryBackoffDefault, - requestTimeout: Duration = ClientConfigs.producerRequestTimeoutDefault, - produceOpt: Option[ - (BlockingFinagleKafkaProducer[Long, KeyedUuaTweet], Long, KeyedUuaTweet, String, Headers, - StatsReceiver, Decider) => Future[Unit] - ] = Some(producer), - trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault), - statsReceiver: StatsReceiver, - decider: Decider, - zone: Zone, - maybeProcess: (ConsumerRecord[UnKeyed, InteractionEvent], Zone) => Boolean, - ): AtLeastOnceProcessor[UnKeyed, InteractionEvent] = { - - lazy val singletonProducer = ClientProviders.mkProducer[Long, KeyedUuaTweet]( - bootstrapServer = kafkaDestCluster, - clientId = kafkaProducerClientId, - keySerde = ScalaSerdes.Long.serializer, - valueSerde = ScalaSerdes.Thrift[KeyedUuaTweet].serializer, - idempotence = false, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - trustStoreLocationOpt = trustStoreLocationOpt, - ) - - KafkaProcessorProvider.mkAtLeastOnceProcessor[UnKeyed, InteractionEvent, Long, KeyedUuaTweet]( - name = name, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = ScalaSerdes.CompactThrift[InteractionEvent].deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - fetchMin = fetchMin, - receiveBuffer = receiveBuffer, - processorMaxPendingRequests = processorMaxPendingRequests, - processorWorkerThreads = processorWorkerThreads, - adapter = adapter, - kafkaProducersAndSinkTopics = - kafkaSinkTopics.map(sinkTopic => (singletonProducer, sinkTopic)), - produce = produceOpt.getOrElse(producer), - trustStoreLocationOpt = trustStoreLocationOpt, - statsReceiver = statsReceiver, - decider = decider, - zone = zone, - maybeProcess = maybeProcess, - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaModule.docx new file mode 100644 index 000000000..09fe99d88 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaModule.scala deleted file mode 100644 index 3b961fabb..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRekeyUuaModule.scala +++ /dev/null @@ -1,203 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.decider.SimpleRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.producers.BlockingFinagleKafkaProducer -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.header.Headers -import org.apache.kafka.common.record.CompressionType -import com.twitter.kafka.client.headers.Zone -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.adapter.AbstractAdapter -import com.twitter.unified_user_actions.adapter.uua_aggregates.RekeyUuaAdapter -import com.twitter.unified_user_actions.kafka.ClientConfigs -import com.twitter.unified_user_actions.kafka.ClientProviders -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.unified_user_actions.thriftscala.KeyedUuaTweet -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton - -object KafkaProcessorRekeyUuaModule extends TwitterModule with Logging { - override def modules = Seq(FlagsModule) - - private val adapter = new RekeyUuaAdapter - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, UnifiedUserAction] = { - provideAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = adapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - maybeProcess = ZoneFiltering.noFiltering - ) - } - - def producer( - producer: BlockingFinagleKafkaProducer[Long, KeyedUuaTweet], - k: Long, - v: KeyedUuaTweet, - sinkTopic: String, - headers: Headers, - statsReceiver: StatsReceiver, - decider: Decider, - ): Future[Unit] = - if (decider.isAvailable(feature = s"RekeyUUA${v.actionType}", Some(SimpleRecipient(k)))) - // If we were to enable xDC replicator, then we can safely remove the Zone header since xDC - // replicator works in the following way: - // - If the message does not have a header, the replicator will assume it is local and - // set the header, copy the message - // - If the message has a header that is the local zone, the replicator will copy the message - // - If the message has a header for a different zone, the replicator will drop the message - producer - .send(new ProducerRecord[Long, KeyedUuaTweet](sinkTopic, null, k, v, headers)) - .onSuccess { _ => statsReceiver.counter("publishSuccess", sinkTopic).incr() } - .onFailure { e: Throwable => - statsReceiver.counter("publishFailure", sinkTopic).incr() - error(s"Publish error to topic $sinkTopic: $e") - }.unit - else Future.Unit - - def provideAtLeastOnceProcessor[K, V]( - name: String, - kafkaSourceCluster: String, - kafkaGroupId: String, - kafkaSourceTopic: String, - commitInterval: Duration = ClientConfigs.kafkaCommitIntervalDefault, - maxPollRecords: Int = ClientConfigs.consumerMaxPollRecordsDefault, - maxPollInterval: Duration = ClientConfigs.consumerMaxPollIntervalDefault, - sessionTimeout: Duration = ClientConfigs.consumerSessionTimeoutDefault, - fetchMax: StorageUnit = ClientConfigs.consumerFetchMaxDefault, - fetchMin: StorageUnit = ClientConfigs.consumerFetchMinDefault, - processorMaxPendingRequests: Int, - processorWorkerThreads: Int, - adapter: AbstractAdapter[UnifiedUserAction, Long, KeyedUuaTweet], - kafkaSinkTopics: Seq[String], - kafkaDestCluster: String, - kafkaProducerClientId: String, - batchSize: StorageUnit = ClientConfigs.producerBatchSizeDefault, - linger: Duration = ClientConfigs.producerLingerDefault, - bufferMem: StorageUnit = ClientConfigs.producerBufferMemDefault, - compressionType: CompressionType = ClientConfigs.compressionDefault.compressionType, - retries: Int = ClientConfigs.retriesDefault, - retryBackoff: Duration = ClientConfigs.retryBackoffDefault, - requestTimeout: Duration = ClientConfigs.producerRequestTimeoutDefault, - produceOpt: Option[ - (BlockingFinagleKafkaProducer[Long, KeyedUuaTweet], Long, KeyedUuaTweet, String, Headers, - StatsReceiver, Decider) => Future[Unit] - ] = Some(producer), - trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault), - statsReceiver: StatsReceiver, - decider: Decider, - zone: Zone, - maybeProcess: (ConsumerRecord[UnKeyed, UnifiedUserAction], Zone) => Boolean, - ): AtLeastOnceProcessor[UnKeyed, UnifiedUserAction] = { - - lazy val singletonProducer = ClientProviders.mkProducer[Long, KeyedUuaTweet]( - bootstrapServer = kafkaDestCluster, - clientId = kafkaProducerClientId, - keySerde = ScalaSerdes.Long.serializer, - valueSerde = ScalaSerdes.Thrift[KeyedUuaTweet].serializer, - idempotence = false, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - trustStoreLocationOpt = trustStoreLocationOpt, - ) - - KafkaProcessorProvider.mkAtLeastOnceProcessor[UnKeyed, UnifiedUserAction, Long, KeyedUuaTweet]( - name = name, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[UnifiedUserAction](statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - fetchMin = fetchMin, - processorMaxPendingRequests = processorMaxPendingRequests, - processorWorkerThreads = processorWorkerThreads, - adapter = adapter, - kafkaProducersAndSinkTopics = - kafkaSinkTopics.map(sinkTopic => (singletonProducer, sinkTopic)), - produce = produceOpt.getOrElse(producer), - trustStoreLocationOpt = trustStoreLocationOpt, - statsReceiver = statsReceiver, - decider = decider, - zone = zone, - maybeProcess = maybeProcess, - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRetweetArchivalEventsModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRetweetArchivalEventsModule.docx new file mode 100644 index 000000000..41c5953d7 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRetweetArchivalEventsModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRetweetArchivalEventsModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRetweetArchivalEventsModule.scala deleted file mode 100644 index b3bdc2fda..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorRetweetArchivalEventsModule.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.tweetypie.thriftscala.RetweetArchivalEvent -import com.twitter.unified_user_actions.adapter.retweet_archival_events.RetweetArchivalEventsAdapter -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton - -object KafkaProcessorRetweetArchivalEventsModule extends TwitterModule with Logging { - override def modules = Seq(FlagsModule) - - private val adapter = new RetweetArchivalEventsAdapter - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, RetweetArchivalEvent] = { - KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[RetweetArchivalEvent](statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = adapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorSocialGraphModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorSocialGraphModule.docx new file mode 100644 index 000000000..96492e744 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorSocialGraphModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorSocialGraphModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorSocialGraphModule.scala deleted file mode 100644 index f9d734490..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorSocialGraphModule.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.socialgraph.thriftscala.WriteEvent -import com.twitter.unified_user_actions.adapter.social_graph_event.SocialGraphAdapter -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton - -class KafkaProcessorSocialGraphModule {} - -object KafkaProcessorSocialGraphModule extends TwitterModule with Logging { - override def modules = Seq(FlagsModule) - - private val socialGraphAdapter = new SocialGraphAdapter - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, WriteEvent] = { - KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[WriteEvent](statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = socialGraphAdapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTlsFavsModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTlsFavsModule.docx new file mode 100644 index 000000000..9cba6c6ae Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTlsFavsModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTlsFavsModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTlsFavsModule.scala deleted file mode 100644 index 65970d333..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTlsFavsModule.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.inject.annotations.Flag -import com.twitter.inject.TwitterModule -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.timelineservice.thriftscala.ContextualizedFavoriteEvent -import com.twitter.unified_user_actions.adapter.tls_favs_event.TlsFavsAdapter -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton - -object KafkaProcessorTlsFavsModule extends TwitterModule with Logging { - override def modules = Seq(FlagsModule) - - private val tlsFavsAdapter = new TlsFavsAdapter - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, ContextualizedFavoriteEvent] = { - KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[ContextualizedFavoriteEvent]( - statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = tlsFavsAdapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTweetypieEventModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTweetypieEventModule.docx new file mode 100644 index 000000000..26bf6148f Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTweetypieEventModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTweetypieEventModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTweetypieEventModule.scala deleted file mode 100644 index d4a9b7e58..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorTweetypieEventModule.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.tweetypie.thriftscala.TweetEvent -import com.twitter.unified_user_actions.adapter.tweetypie_event.TweetypieEventAdapter -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton - -object KafkaProcessorTweetypieEventModule extends TwitterModule with Logging { - override def modules: Seq[inject.Module] = Seq(FlagsModule) - - private val tweetypieEventAdapter = new TweetypieEventAdapter - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, TweetEvent] = { - KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[TweetEvent](statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = tweetypieEventAdapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - ) - } - -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorUserModificationModule.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorUserModificationModule.docx new file mode 100644 index 000000000..9715498ae Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorUserModificationModule.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorUserModificationModule.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorUserModificationModule.scala deleted file mode 100644 index 0e17fa9f2..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorUserModificationModule.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.gizmoduck.thriftscala.UserModification -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.unified_user_actions.adapter.user_modification.UserModificationAdapter -import com.twitter.unified_user_actions.kafka.CompressionTypeFlag -import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes -import com.twitter.util.Duration -import com.twitter.util.StorageUnit -import com.twitter.util.logging.Logging -import javax.inject.Singleton - -object KafkaProcessorUserModificationModule extends TwitterModule with Logging { - override def modules = Seq(FlagsModule) - - // NOTE: This is a shared processor name in order to simplify monviz stat computation. - private final val processorName = "uuaProcessor" - - @Provides - @Singleton - def providesKafkaProcessor( - decider: Decider, - @Flag(FlagsModule.cluster) cluster: String, - @Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String, - @Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String, - @Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String, - @Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String], - @Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String, - @Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String, - @Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int, - @Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int, - @Flag(FlagsModule.commitInterval) commitInterval: Duration, - @Flag(FlagsModule.maxPollRecords) maxPollRecords: Int, - @Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration, - @Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration, - @Flag(FlagsModule.fetchMax) fetchMax: StorageUnit, - @Flag(FlagsModule.batchSize) batchSize: StorageUnit, - @Flag(FlagsModule.linger) linger: Duration, - @Flag(FlagsModule.bufferMem) bufferMem: StorageUnit, - @Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag, - @Flag(FlagsModule.retries) retries: Int, - @Flag(FlagsModule.retryBackoff) retryBackoff: Duration, - @Flag(FlagsModule.requestTimeout) requestTimeout: Duration, - @Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean, - @Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String, - statsReceiver: StatsReceiver, - ): AtLeastOnceProcessor[UnKeyed, UserModification] = { - KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor( - name = processorName, - kafkaSourceCluster = kafkaSourceCluster, - kafkaGroupId = kafkaGroupId, - kafkaSourceTopic = kafkaSourceTopic, - sourceKeyDeserializer = UnKeyedSerde.deserializer, - sourceValueDeserializer = NullableScalaSerdes - .Thrift[UserModification](statsReceiver.counter("deserializerErrors")).deserializer, - commitInterval = commitInterval, - maxPollRecords = maxPollRecords, - maxPollInterval = maxPollInterval, - sessionTimeout = sessionTimeout, - fetchMax = fetchMax, - processorMaxPendingRequests = kafkaMaxPendingRequests, - processorWorkerThreads = kafkaWorkerThreads, - adapter = new UserModificationAdapter, - kafkaSinkTopics = kafkaSinkTopics, - kafkaDestCluster = kafkaDestCluster, - kafkaProducerClientId = kafkaProducerClientId, - batchSize = batchSize, - linger = linger, - bufferMem = bufferMem, - compressionType = compressionTypeFlag.compressionType, - retries = retries, - retryBackoff = retryBackoff, - requestTimeout = requestTimeout, - statsReceiver = statsReceiver, - trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None, - decider = decider, - zone = ZoneFiltering.zoneMapping(cluster), - ) - } -} diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/TopicsMapping.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/TopicsMapping.docx new file mode 100644 index 000000000..c05a3d257 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/TopicsMapping.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/TopicsMapping.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/TopicsMapping.scala deleted file mode 100644 index 8959118f9..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/TopicsMapping.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -case class TopicsMapping( - all: String = "unified_user_actions", - engagementsOnly: String = "unified_user_actions_engagements") diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ZoneFiltering.docx b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ZoneFiltering.docx new file mode 100644 index 000000000..d002f9ed9 Binary files /dev/null and b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ZoneFiltering.docx differ diff --git a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ZoneFiltering.scala b/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ZoneFiltering.scala deleted file mode 100644 index 3da3e80d6..000000000 --- a/unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/ZoneFiltering.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.unified_user_actions.service.module - -import com.twitter.kafka.client.headers.ATLA -import com.twitter.kafka.client.headers.Implicits._ -import com.twitter.kafka.client.headers.PDXA -import com.twitter.kafka.client.headers.Zone -import org.apache.kafka.clients.consumer.ConsumerRecord - -object ZoneFiltering { - def zoneMapping(zone: String): Zone = zone.toLowerCase match { - case "atla" => ATLA - case "pdxa" => PDXA - case _ => - throw new IllegalArgumentException( - s"zone must be provided and must be one of [atla,pdxa], provided $zone") - } - - def localDCFiltering[K, V](event: ConsumerRecord[K, V], localZone: Zone): Boolean = - event.headers().isLocalZone(localZone) - - def noFiltering[K, V](event: ConsumerRecord[K, V], localZone: Zone): Boolean = true -} diff --git a/unified_user_actions/service/src/test/resources/BUILD.bazel b/unified_user_actions/service/src/test/resources/BUILD.bazel deleted file mode 100644 index ae9669f4f..000000000 --- a/unified_user_actions/service/src/test/resources/BUILD.bazel +++ /dev/null @@ -1,4 +0,0 @@ -resources( - sources = ["*.*"], - tags = ["bazel-compatible"], -) diff --git a/unified_user_actions/service/src/test/resources/BUILD.docx b/unified_user_actions/service/src/test/resources/BUILD.docx new file mode 100644 index 000000000..6d4e837e3 Binary files /dev/null and b/unified_user_actions/service/src/test/resources/BUILD.docx differ diff --git a/unified_user_actions/service/src/test/resources/decider.docx b/unified_user_actions/service/src/test/resources/decider.docx new file mode 100644 index 000000000..542bd460b Binary files /dev/null and b/unified_user_actions/service/src/test/resources/decider.docx differ diff --git a/unified_user_actions/service/src/test/resources/decider.yml b/unified_user_actions/service/src/test/resources/decider.yml deleted file mode 100644 index 604217f37..000000000 --- a/unified_user_actions/service/src/test/resources/decider.yml +++ /dev/null @@ -1,6 +0,0 @@ -PublishServerTweetFav: - default_availability: 10000 -RekeyUUAIesourceClientTweetRenderImpression: - default_availability: 10000 -EnrichmentPlannerSampling: - default_availability: 10000 diff --git a/unified_user_actions/service/src/test/resources/logback.docx b/unified_user_actions/service/src/test/resources/logback.docx new file mode 100644 index 000000000..fdcb08c09 Binary files /dev/null and b/unified_user_actions/service/src/test/resources/logback.docx differ diff --git a/unified_user_actions/service/src/test/resources/logback.xml b/unified_user_actions/service/src/test/resources/logback.xml deleted file mode 100644 index 27f50b1dc..000000000 --- a/unified_user_actions/service/src/test/resources/logback.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/BUILD.bazel b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/BUILD.bazel deleted file mode 100644 index 7b42c4c0f..000000000 --- a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -junit_tests( - name = "tests", - sources = ["*.scala"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/ch/qos/logback:logback-classic", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "decider/src/main/scala", - "kafka/finagle-kafka/finatra-kafka-streams/kafka-streams/src/test/scala:test-deps", - "kafka/finagle-kafka/finatra-kafka/src/test/scala:test-deps", - "unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala", - "unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher:fixture", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:client-event", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:enrichment-planner", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:rekey-uua-iesource", - "unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:tls-favs", - "unified_user_actions/service/src/test/resources", - "util/util-mock/src/main/scala/com/twitter/util/mock", - ], -) diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/BUILD.docx b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/BUILD.docx new file mode 100644 index 000000000..737c8ca2e Binary files /dev/null and b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/BUILD.docx differ diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ClientEventServiceStartupTest.docx b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ClientEventServiceStartupTest.docx new file mode 100644 index 000000000..7a63338d5 Binary files /dev/null and b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ClientEventServiceStartupTest.docx differ diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ClientEventServiceStartupTest.scala b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ClientEventServiceStartupTest.scala deleted file mode 100644 index 0503d8782..000000000 --- a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/ClientEventServiceStartupTest.scala +++ /dev/null @@ -1,141 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.google.inject.Stage -import com.twitter.app.GlobalFlag -import com.twitter.clientapp.thriftscala.EventDetails -import com.twitter.clientapp.thriftscala.EventNamespace -import com.twitter.clientapp.thriftscala.Item -import com.twitter.clientapp.thriftscala.ItemType -import com.twitter.clientapp.thriftscala.LogEvent -import com.twitter.finatra.kafka.consumers.FinagleKafkaConsumerBuilder -import com.twitter.finatra.kafka.domain.AckMode -import com.twitter.finatra.kafka.domain.KafkaGroupId -import com.twitter.finatra.kafka.domain.KafkaTopic -import com.twitter.finatra.kafka.domain.SeekStrategy -import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.finatra.kafka.test.KafkaFeatureTest -import com.twitter.inject.server.EmbeddedTwitterServer -import com.twitter.kafka.client.processor.KafkaConsumerClient -import com.twitter.logbase.thriftscala.LogBase -import com.twitter.unified_user_actions.kafka.ClientConfigs -import com.twitter.unified_user_actions.service.module.KafkaProcessorClientEventModule -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.util.Duration -import com.twitter.util.StorageUnit - -class ClientEventServiceStartupTest extends KafkaFeatureTest { - private val inputTopic = - kafkaTopic(UnKeyedSerde, ScalaSerdes.Thrift[LogEvent], name = "source") - private val outputTopic = - kafkaTopic(UnKeyedSerde, ScalaSerdes.Thrift[UnifiedUserAction], name = "sink") - - val startupFlags = Map( - "kafka.group.id" -> "client-event", - "kafka.producer.client.id" -> "uua", - "kafka.source.topic" -> inputTopic.topic, - "kafka.sink.topics" -> outputTopic.topic, - "kafka.consumer.fetch.min" -> "6.megabytes", - "kafka.max.pending.requests" -> "100", - "kafka.worker.threads" -> "1", - "kafka.trust.store.enable" -> "false", - "kafka.producer.batch.size" -> "0.byte", - "cluster" -> "atla", - ) - - val deciderFlags = Map( - "decider.base" -> "/decider.yml" - ) - - override protected def kafkaBootstrapFlag: Map[String, String] = { - Map( - ClientConfigs.kafkaBootstrapServerConfig -> kafkaCluster.bootstrapServers(), - ClientConfigs.kafkaBootstrapServerRemoteDestConfig -> kafkaCluster.bootstrapServers(), - ) - } - - override val server: EmbeddedTwitterServer = new EmbeddedTwitterServer( - twitterServer = new ClientEventService() { - override def warmup(): Unit = { - // noop - } - - override val overrideModules = Seq( - KafkaProcessorClientEventModule - ) - }, - globalFlags = Map[GlobalFlag[_], String]( - com.twitter.finatra.kafka.consumers.enableTlsAndKerberos -> "false", - ), - flags = startupFlags ++ kafkaBootstrapFlag ++ deciderFlags, - stage = Stage.PRODUCTION - ) - - private def getConsumer( - seekStrategy: SeekStrategy = SeekStrategy.BEGINNING, - ) = { - val builder = FinagleKafkaConsumerBuilder() - .dest(brokers.map(_.brokerList()).mkString(",")) - .clientId("consumer") - .groupId(KafkaGroupId("validator")) - .keyDeserializer(UnKeyedSerde.deserializer) - .valueDeserializer(ScalaSerdes.Thrift[LogEvent].deserializer) - .requestTimeout(Duration.fromSeconds(1)) - .enableAutoCommit(false) - .seekStrategy(seekStrategy) - - new KafkaConsumerClient(builder.config) - } - - private def getProducer(clientId: String = "producer") = { - FinagleKafkaProducerBuilder() - .dest(brokers.map(_.brokerList()).mkString(",")) - .clientId(clientId) - .ackMode(AckMode.ALL) - .batchSize(StorageUnit.zero) - .keySerializer(UnKeyedSerde.serializer) - .valueSerializer(ScalaSerdes.Thrift[LogEvent].serializer) - .build() - } - - test("ClientEventService starts") { - server.assertHealthy() - } - - test("ClientEventService should process input events") { - val producer = getProducer() - val inputConsumer = getConsumer() - - val value: LogEvent = LogEvent( - eventName = "test_tweet_render_impression_event", - eventNamespace = - Some(EventNamespace(component = Some("stream"), element = None, action = Some("results"))), - eventDetails = Some( - EventDetails( - items = Some( - Seq[Item]( - Item(id = Some(1L), itemType = Some(ItemType.Tweet)) - )) - )), - logBase = Some(LogBase(timestamp = 10001L, transactionId = "", ipAddress = "")) - ) - - try { - server.assertHealthy() - - // before, should be empty - inputConsumer.subscribe(Set(KafkaTopic(inputTopic.topic))) - assert(inputConsumer.poll().count() == 0) - - // after, should contain at least a message - await(producer.send(inputTopic.topic, new UnKeyed, value, System.currentTimeMillis)) - producer.flush() - assert(inputConsumer.poll().count() >= 1) - } finally { - await(producer.close()) - inputConsumer.close() - } - } -} diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/DeciderUtilsTest.docx b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/DeciderUtilsTest.docx new file mode 100644 index 000000000..bb9a16306 Binary files /dev/null and b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/DeciderUtilsTest.docx differ diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/DeciderUtilsTest.scala b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/DeciderUtilsTest.scala deleted file mode 100644 index f5a0af48c..000000000 --- a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/DeciderUtilsTest.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.decider.MockDecider -import com.twitter.inject.Test -import com.twitter.unified_user_actions.service.module.ClientEventDeciderUtils -import com.twitter.unified_user_actions.service.module.DefaultDeciderUtils -import com.twitter.unified_user_actions.thriftscala._ -import com.twitter.util.Time -import com.twitter.util.mock.Mockito -import org.junit.runner.RunWith -import org.scalatestplus.junit.JUnitRunner - -@RunWith(classOf[JUnitRunner]) -class DeciderUtilsTest extends Test with Mockito { - trait Fixture { - val frozenTime = Time.fromMilliseconds(1658949273000L) - - val publishActionTypes = - Set[ActionType](ActionType.ServerTweetFav, ActionType.ClientTweetRenderImpression) - - def decider( - features: Set[String] = publishActionTypes.map { action => - s"Publish${action.name}" - } - ) = new MockDecider(features = features) - - def mkUUA(actionType: ActionType) = UnifiedUserAction( - userIdentifier = UserIdentifier(userId = Some(91L)), - item = Item.TweetInfo( - TweetInfo( - actionTweetId = 1L, - actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(101L))), - ) - ), - actionType = actionType, - eventMetadata = EventMetadata( - sourceTimestampMs = 1001L, - receivedTimestampMs = frozenTime.inMilliseconds, - sourceLineage = SourceLineage.ServerTlsFavs, - traceId = Some(31L) - ) - ) - - val uuaServerTweetFav = mkUUA(ActionType.ServerTweetFav) - val uuaClientTweetFav = mkUUA(ActionType.ClientTweetFav) - val uuaClientTweetRenderImpression = mkUUA(ActionType.ClientTweetRenderImpression) - } - - test("Decider Utils") { - new Fixture { - Time.withTimeAt(frozenTime) { _ => - DefaultDeciderUtils.shouldPublish( - decider = decider(), - uua = uuaServerTweetFav, - sinkTopic = "") shouldBe true - DefaultDeciderUtils.shouldPublish( - decider = decider(), - uua = uuaClientTweetFav, - sinkTopic = "") shouldBe false - ClientEventDeciderUtils.shouldPublish( - decider = decider(), - uua = uuaClientTweetRenderImpression, - sinkTopic = "unified_user_actions_engagements") shouldBe false - ClientEventDeciderUtils.shouldPublish( - decider = decider(), - uua = uuaClientTweetFav, - sinkTopic = "unified_user_actions_engagements") shouldBe false - ClientEventDeciderUtils.shouldPublish( - decider = decider(features = Set[String](s"Publish${ActionType.ClientTweetFav.name}")), - uua = uuaClientTweetFav, - sinkTopic = "unified_user_actions_engagements") shouldBe true - } - } - } -} diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerServiceTest.docx b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerServiceTest.docx new file mode 100644 index 000000000..05dd64a4e Binary files /dev/null and b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerServiceTest.docx differ diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerServiceTest.scala b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerServiceTest.scala deleted file mode 100644 index a1038e1b2..000000000 --- a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/EnrichmentPlannerServiceTest.scala +++ /dev/null @@ -1,141 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.finatra.kafka.test.EmbeddedKafka -import com.twitter.finatra.kafkastreams.test.FinatraTopologyTester -import com.twitter.finatra.kafkastreams.test.TopologyFeatureTest -import com.twitter.unified_user_actions.enricher.EnricherFixture -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentIdType -import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.joda.time.DateTime - -/** - * This is to test the logic where the service reads and outputs to the same Kafka cluster - */ -class EnrichmentPlannerServiceTest extends TopologyFeatureTest { - val startTime = new DateTime("2022-10-01T00:00:00Z") - - override protected lazy val topologyTester: FinatraTopologyTester = FinatraTopologyTester( - "enrichment-planner-tester", - new EnrichmentPlannerService, - startingWallClockTime = startTime, - flags = Map( - "decider.base" -> "/decider.yml", - "kafka.output.server" -> "" - ) - ) - - private val inputTopic = topologyTester.topic( - name = EnrichmentPlannerServiceMain.InputTopic, - keySerde = UnKeyedSerde, - valSerde = ScalaSerdes.Thrift[UnifiedUserAction] - ) - - private val outputTopic = topologyTester.topic( - name = EnrichmentPlannerServiceMain.OutputPartitionedTopic, - keySerde = ScalaSerdes.Thrift[EnrichmentKey], - valSerde = ScalaSerdes.Thrift[EnrichmentEnvelop] - ) - - test("can filter unsupported events") { - new EnricherFixture { - (1L to 10L).foreach(id => { - inputTopic.pipeInput(UnKeyed, mkUUAProfileEvent(id)) - }) - - assert(outputTopic.readAllOutput().size === 0) - } - } - - test("partition key serialization should be correct") { - val key = EnrichmentKey(EnrichmentIdType.TweetId, 9999L) - val serializer = ScalaSerdes.Thrift[EnrichmentKey].serializer - - val actual = serializer.serialize("test", key) - val expected = Array[Byte](8, 0, 1, 0, 0, 0, 0, 10, 0, 2, 0, 0, 0, 0, 0, 0, 39, 15, 0) - - assert(actual.deep === expected.deep) - } - - test("partitioned enrichment tweet event is constructed correctly") { - new EnricherFixture { - val expected = mkUUATweetEvent(888L) - inputTopic.pipeInput(UnKeyed, expected) - - val actual = outputTopic.readAllOutput().head - - assert(actual.key() === EnrichmentKey(EnrichmentIdType.TweetId, 888L)) - assert( - actual - .value() === EnrichmentEnvelop( - expected.hashCode, - expected, - plan = tweetInfoEnrichmentPlan - )) - } - } - - test("partitioned enrichment tweet notification event is constructed correctly") { - new EnricherFixture { - val expected = mkUUATweetNotificationEvent(8989L) - inputTopic.pipeInput(UnKeyed, expected) - - val actual = outputTopic.readAllOutput().head - - assert(actual.key() === EnrichmentKey(EnrichmentIdType.TweetId, 8989L)) - assert( - actual - .value() === EnrichmentEnvelop( - expected.hashCode, - expected, - plan = tweetNotificationEnrichmentPlan - )) - } - } -} - -/** - * This is tests the bootstrap server logic in prod. Don't add any new tests here since it is slow. - * Use the tests above which is much quicker to be executed and and test the majority of prod logic. - */ -class EnrichmentPlannerServiceEmbeddedKafkaTest extends TopologyFeatureTest with EmbeddedKafka { - val startTime = new DateTime("2022-10-01T00:00:00Z") - - override protected lazy val topologyTester: FinatraTopologyTester = FinatraTopologyTester( - "enrichment-planner-tester", - new EnrichmentPlannerService, - startingWallClockTime = startTime, - flags = Map( - "decider.base" -> "/decider.yml", - "kafka.output.server" -> kafkaCluster.bootstrapServers(), - "kafka.output.enable.tls" -> "false" - ) - ) - - private lazy val inputTopic = topologyTester.topic( - name = EnrichmentPlannerServiceMain.InputTopic, - keySerde = UnKeyedSerde, - valSerde = ScalaSerdes.Thrift[UnifiedUserAction] - ) - - private val outputTopic = kafkaTopic( - name = EnrichmentPlannerServiceMain.OutputPartitionedTopic, - keySerde = ScalaSerdes.Thrift[EnrichmentKey], - valSerde = ScalaSerdes.Thrift[EnrichmentEnvelop] - ) - - test("toCluster should output to expected topic & embeded cluster") { - new EnricherFixture { - inputTopic.pipeInput(UnKeyed, mkUUATweetEvent(tweetId = 1)) - val records: Seq[ConsumerRecord[Array[Byte], Array[Byte]]] = outputTopic.consumeRecords(1) - - assert(records.size === 1) - assert(records.head.topic() == EnrichmentPlannerServiceMain.OutputPartitionedTopic) - } - } -} diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceServiceStartupTest.docx b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceServiceStartupTest.docx new file mode 100644 index 000000000..cfc0f2e59 Binary files /dev/null and b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceServiceStartupTest.docx differ diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceServiceStartupTest.scala b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceServiceStartupTest.scala deleted file mode 100644 index 9609a2691..000000000 --- a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/RekeyUuaIesourceServiceStartupTest.scala +++ /dev/null @@ -1,173 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.google.inject.Stage -import com.twitter.adserver.thriftscala.DisplayLocation -import com.twitter.app.GlobalFlag -import com.twitter.finatra.kafka.consumers.FinagleKafkaConsumerBuilder -import com.twitter.finatra.kafka.domain.AckMode -import com.twitter.finatra.kafka.domain.KafkaGroupId -import com.twitter.finatra.kafka.domain.KafkaTopic -import com.twitter.finatra.kafka.domain.SeekStrategy -import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.finatra.kafka.test.KafkaFeatureTest -import com.twitter.iesource.thriftscala.ClientEventContext -import com.twitter.iesource.thriftscala.TweetImpression -import com.twitter.iesource.thriftscala.ClientType -import com.twitter.iesource.thriftscala.ContextualEventNamespace -import com.twitter.iesource.thriftscala.EngagingContext -import com.twitter.iesource.thriftscala.EventSource -import com.twitter.iesource.thriftscala.InteractionDetails -import com.twitter.iesource.thriftscala.InteractionEvent -import com.twitter.iesource.thriftscala.InteractionType -import com.twitter.iesource.thriftscala.InteractionTargetType -import com.twitter.iesource.thriftscala.UserIdentifier -import com.twitter.inject.server.EmbeddedTwitterServer -import com.twitter.kafka.client.processor.KafkaConsumerClient -import com.twitter.unified_user_actions.kafka.ClientConfigs -import com.twitter.unified_user_actions.service.module.KafkaProcessorRekeyUuaIesourceModule -import com.twitter.unified_user_actions.thriftscala.KeyedUuaTweet -import com.twitter.util.Duration -import com.twitter.util.StorageUnit - -class RekeyUuaIesourceServiceStartupTest extends KafkaFeatureTest { - private val inputTopic = - kafkaTopic(ScalaSerdes.Long, ScalaSerdes.CompactThrift[InteractionEvent], name = "source") - private val outputTopic = - kafkaTopic(ScalaSerdes.Long, ScalaSerdes.Thrift[KeyedUuaTweet], name = "sink") - - val startupFlags = Map( - "kafka.group.id" -> "client-event", - "kafka.producer.client.id" -> "uua", - "kafka.source.topic" -> inputTopic.topic, - "kafka.sink.topics" -> outputTopic.topic, - "kafka.consumer.fetch.min" -> "6.megabytes", - "kafka.max.pending.requests" -> "100", - "kafka.worker.threads" -> "1", - "kafka.trust.store.enable" -> "false", - "kafka.producer.batch.size" -> "0.byte", - "cluster" -> "atla", - ) - - val deciderFlags = Map( - "decider.base" -> "/decider.yml" - ) - - override protected def kafkaBootstrapFlag: Map[String, String] = { - Map( - ClientConfigs.kafkaBootstrapServerConfig -> kafkaCluster.bootstrapServers(), - ClientConfigs.kafkaBootstrapServerRemoteDestConfig -> kafkaCluster.bootstrapServers(), - ) - } - - override val server: EmbeddedTwitterServer = new EmbeddedTwitterServer( - twitterServer = new RekeyUuaIesourceService() { - override def warmup(): Unit = { - // noop - } - - override val overrideModules = Seq( - KafkaProcessorRekeyUuaIesourceModule - ) - }, - globalFlags = Map[GlobalFlag[_], String]( - com.twitter.finatra.kafka.consumers.enableTlsAndKerberos -> "false", - ), - flags = startupFlags ++ kafkaBootstrapFlag ++ deciderFlags, - stage = Stage.PRODUCTION - ) - - private def getConsumer( - seekStrategy: SeekStrategy = SeekStrategy.BEGINNING, - ) = { - val builder = FinagleKafkaConsumerBuilder() - .dest(brokers.map(_.brokerList()).mkString(",")) - .clientId("consumer") - .groupId(KafkaGroupId("validator")) - .keyDeserializer(ScalaSerdes.Long.deserializer) - .valueDeserializer(ScalaSerdes.CompactThrift[InteractionEvent].deserializer) - .requestTimeout(Duration.fromSeconds(1)) - .enableAutoCommit(false) - .seekStrategy(seekStrategy) - - new KafkaConsumerClient(builder.config) - } - - private def getUUAConsumer( - seekStrategy: SeekStrategy = SeekStrategy.BEGINNING, - ) = { - val builder = FinagleKafkaConsumerBuilder() - .dest(brokers.map(_.brokerList()).mkString(",")) - .clientId("consumer_uua") - .groupId(KafkaGroupId("validator_uua")) - .keyDeserializer(UnKeyedSerde.deserializer) - .valueDeserializer(ScalaSerdes.Thrift[KeyedUuaTweet].deserializer) - .requestTimeout(Duration.fromSeconds(1)) - .enableAutoCommit(false) - .seekStrategy(seekStrategy) - - new KafkaConsumerClient(builder.config) - } - - private def getProducer(clientId: String = "producer") = { - FinagleKafkaProducerBuilder() - .dest(brokers.map(_.brokerList()).mkString(",")) - .clientId(clientId) - .ackMode(AckMode.ALL) - .batchSize(StorageUnit.zero) - .keySerializer(ScalaSerdes.Long.serializer) - .valueSerializer(ScalaSerdes.CompactThrift[InteractionEvent].serializer) - .build() - } - - test("RekeyUuaIesourceService starts") { - server.assertHealthy() - } - - test("RekeyUuaIesourceService should process input events") { - val producer = getProducer() - val inputConsumer = getConsumer() - val uuaConsumer = getUUAConsumer() - - val value: InteractionEvent = InteractionEvent( - targetId = 1L, - targetType = InteractionTargetType.Tweet, - engagingUserId = 11L, - eventSource = EventSource.ClientEvent, - timestampMillis = 123456L, - interactionType = Some(InteractionType.TweetRenderImpression), - details = InteractionDetails.TweetRenderImpression(TweetImpression()), - additionalEngagingUserIdentifiers = UserIdentifier(), - engagingContext = EngagingContext.ClientEventContext( - ClientEventContext( - clientEventNamespace = ContextualEventNamespace(), - clientType = ClientType.Iphone, - displayLocation = DisplayLocation(1))) - ) - - try { - server.assertHealthy() - - // before, should be empty - inputConsumer.subscribe(Set(KafkaTopic(inputTopic.topic))) - assert(inputConsumer.poll().count() == 0) - - // after, should contain at least a message - await(producer.send(inputTopic.topic, value.targetId, value, System.currentTimeMillis)) - producer.flush() - assert(inputConsumer.poll().count() == 1) - - uuaConsumer.subscribe(Set(KafkaTopic(outputTopic.topic))) - // This is tricky: it is not guaranteed that the srvice can process and output the - // event to output topic faster than the below consumer. So we'd use a timer here which may - // not be the best practice. - // If someone finds the below test is flaky, please just remove the below test completely. - Thread.sleep(5000L) - assert(uuaConsumer.poll().count() == 1) - } finally { - await(producer.close()) - inputConsumer.close() - } - } -} diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/TlsFavServiceStartupTest.docx b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/TlsFavServiceStartupTest.docx new file mode 100644 index 000000000..f8f7320a0 Binary files /dev/null and b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/TlsFavServiceStartupTest.docx differ diff --git a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/TlsFavServiceStartupTest.scala b/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/TlsFavServiceStartupTest.scala deleted file mode 100644 index 8c0615ea8..000000000 --- a/unified_user_actions/service/src/test/scala/com/twitter/unified_user_actions/service/TlsFavServiceStartupTest.scala +++ /dev/null @@ -1,153 +0,0 @@ -package com.twitter.unified_user_actions.service - -import com.google.inject.Stage -import com.twitter.app.GlobalFlag -import com.twitter.finatra.kafka.consumers.FinagleKafkaConsumerBuilder -import com.twitter.finatra.kafka.domain.AckMode -import com.twitter.finatra.kafka.domain.KafkaGroupId -import com.twitter.finatra.kafka.domain.KafkaTopic -import com.twitter.finatra.kafka.domain.SeekStrategy -import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import com.twitter.finatra.kafka.test.KafkaFeatureTest -import com.twitter.inject.server.EmbeddedTwitterServer -import com.twitter.kafka.client.processor.KafkaConsumerClient -import com.twitter.timelineservice.thriftscala.ContextualizedFavoriteEvent -import com.twitter.timelineservice.thriftscala.FavoriteEvent -import com.twitter.timelineservice.thriftscala.FavoriteEventUnion -import com.twitter.timelineservice.thriftscala.LogEventContext -import com.twitter.unified_user_actions.kafka.ClientConfigs -import com.twitter.unified_user_actions.service.module.KafkaProcessorTlsFavsModule -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.util.Duration -import com.twitter.util.StorageUnit - -class TlsFavServiceStartupTest extends KafkaFeatureTest { - private val inputTopic = - kafkaTopic(UnKeyedSerde, ScalaSerdes.Thrift[ContextualizedFavoriteEvent], name = "source") - private val outputTopic = - kafkaTopic(UnKeyedSerde, ScalaSerdes.Thrift[UnifiedUserAction], name = "sink") - - val startupFlags = Map( - "kafka.group.id" -> "tls", - "kafka.producer.client.id" -> "uua", - "kafka.source.topic" -> inputTopic.topic, - "kafka.sink.topics" -> outputTopic.topic, - "kafka.max.pending.requests" -> "100", - "kafka.worker.threads" -> "1", - "kafka.trust.store.enable" -> "false", - "kafka.producer.batch.size" -> "0.byte", - "cluster" -> "atla", - ) - - val deciderFlags = Map( - "decider.base" -> "/decider.yml" - ) - - override protected def kafkaBootstrapFlag: Map[String, String] = { - Map( - ClientConfigs.kafkaBootstrapServerConfig -> kafkaCluster.bootstrapServers(), - ClientConfigs.kafkaBootstrapServerRemoteDestConfig -> kafkaCluster.bootstrapServers(), - ) - } - - override val server: EmbeddedTwitterServer = new EmbeddedTwitterServer( - twitterServer = new TlsFavsService() { - override def warmup(): Unit = { - // noop - } - - override val overrideModules = Seq( - KafkaProcessorTlsFavsModule - ) - }, - globalFlags = Map[GlobalFlag[_], String]( - com.twitter.finatra.kafka.consumers.enableTlsAndKerberos -> "false", - ), - flags = startupFlags ++ kafkaBootstrapFlag ++ deciderFlags, - stage = Stage.PRODUCTION - ) - - private def getConsumer( - seekStrategy: SeekStrategy = SeekStrategy.BEGINNING, - ) = { - val builder = FinagleKafkaConsumerBuilder() - .dest(brokers.map(_.brokerList()).mkString(",")) - .clientId("consumer") - .groupId(KafkaGroupId("validator")) - .keyDeserializer(UnKeyedSerde.deserializer) - .valueDeserializer(ScalaSerdes.Thrift[ContextualizedFavoriteEvent].deserializer) - .requestTimeout(Duration.fromSeconds(1)) - .enableAutoCommit(false) - .seekStrategy(seekStrategy) - - new KafkaConsumerClient(builder.config) - } - - private def getProducer(clientId: String = "producer") = { - FinagleKafkaProducerBuilder() - .dest(brokers.map(_.brokerList()).mkString(",")) - .clientId(clientId) - .ackMode(AckMode.ALL) - .batchSize(StorageUnit.zero) - .keySerializer(UnKeyedSerde.serializer) - .valueSerializer(ScalaSerdes.Thrift[ContextualizedFavoriteEvent].serializer) - .build() - } - - private def getUUAConsumer( - seekStrategy: SeekStrategy = SeekStrategy.BEGINNING, - ) = { - val builder = FinagleKafkaConsumerBuilder() - .dest(brokers.map(_.brokerList()).mkString(",")) - .clientId("consumer_uua") - .groupId(KafkaGroupId("validator_uua")) - .keyDeserializer(UnKeyedSerde.deserializer) - .valueDeserializer(ScalaSerdes.Thrift[UnifiedUserAction].deserializer) - .requestTimeout(Duration.fromSeconds(1)) - .enableAutoCommit(false) - .seekStrategy(seekStrategy) - - new KafkaConsumerClient(builder.config) - } - - test("TlsFavService starts") { - server.assertHealthy() - } - - test("TlsFavService should process input events") { - val producer = getProducer() - val inputConsumer = getConsumer() - val uuaConsumer = getUUAConsumer() - - val favoriteEvent = FavoriteEventUnion.Favorite(FavoriteEvent(123L, 123L, 123L, 123L)) - val value = - ContextualizedFavoriteEvent(favoriteEvent, LogEventContext("localhost", 123L)) - - try { - server.assertHealthy() - - // before, should be empty - inputConsumer.subscribe(Set(KafkaTopic(inputTopic.topic))) - assert(inputConsumer.poll().count() == 0) - - // after, should contain at least a message - await(producer.send(inputTopic.topic, new UnKeyed, value, System.currentTimeMillis)) - producer.flush() - assert(inputConsumer.poll().count() == 1) - - uuaConsumer.subscribe(Set(KafkaTopic(outputTopic.topic))) - // This is tricky: it is not guaranteed that the TlsFavsService can process and output the - // event to output topic faster than the below consumer. So we'd use a timer here which may - // not be the best practice. - // If someone finds the below test is flaky, please just remove the below test completely. - Thread.sleep(5000L) - assert(uuaConsumer.poll().count() == 1) - } finally { - await(producer.close()) - inputConsumer.close() - } - } -}