diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetDeletedTweetsHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetDeletedTweetsHandler.docx new file mode 100644 index 000000000..675b038f9 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetDeletedTweetsHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetDeletedTweetsHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetDeletedTweetsHandler.scala deleted file mode 100644 index b74acf94d..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetDeletedTweetsHandler.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.InternalServerError -import com.twitter.tweetypie.core.OverCapacity -import com.twitter.tweetypie.storage.Response.TweetResponseCode -import com.twitter.tweetypie.storage.TweetStorageClient.GetTweet -import com.twitter.tweetypie.storage.DeleteState -import com.twitter.tweetypie.storage.DeletedTweetResponse -import com.twitter.tweetypie.storage.RateLimited -import com.twitter.tweetypie.storage.TweetStorageClient -import com.twitter.tweetypie.thriftscala._ - -/** - * Allow access to raw, unhydrated deleted tweet fields from storage backends (currently Manhattan) - */ -object GetDeletedTweetsHandler { - - type Type = FutureArrow[GetDeletedTweetsRequest, Seq[GetDeletedTweetResult]] - type TweetsExist = Seq[TweetId] => Stitch[Set[TweetId]] - - def processTweetResponse(response: Try[GetTweet.Response]): Stitch[Option[Tweet]] = { - import GetTweet.Response._ - - response match { - case Return(Found(tweet)) => Stitch.value(Some(tweet)) - case Return(Deleted | NotFound | BounceDeleted(_)) => Stitch.None - case Throw(_: RateLimited) => Stitch.exception(OverCapacity("manhattan")) - case Throw(exception) => Stitch.exception(exception) - } - } - - def convertDeletedTweetResponse( - r: DeletedTweetResponse, - extantIds: Set[TweetId] - ): GetDeletedTweetResult = { - val id = r.tweetId - if (extantIds.contains(id) || r.deleteState == DeleteState.NotDeleted) { - GetDeletedTweetResult(id, DeletedTweetState.NotDeleted) - } else { - r.overallResponse match { - case TweetResponseCode.Success => - GetDeletedTweetResult(id, convertState(r.deleteState), r.tweet) - case TweetResponseCode.OverCapacity => throw OverCapacity("manhattan") - case _ => - throw InternalServerError( - s"Unhandled response ${r.overallResponse} from getDeletedTweets for tweet $id" - ) - } - } - } - - def convertState(d: DeleteState): DeletedTweetState = d match { - case DeleteState.NotFound => DeletedTweetState.NotFound - case DeleteState.NotDeleted => DeletedTweetState.NotDeleted - case DeleteState.SoftDeleted => DeletedTweetState.SoftDeleted - // Callers of this endpoint treat BounceDeleted tweets the same as SoftDeleted - case DeleteState.BounceDeleted => DeletedTweetState.SoftDeleted - case DeleteState.HardDeleted => DeletedTweetState.HardDeleted - } - - /** - * Converts [[TweetStorageClient.GetTweet]] into a FutureArrow that returns extant tweet ids from - * the original list. This method is used to check underlying storage againt cache, preferring - * cache if a tweet exists there. - */ - def tweetsExist(getTweet: TweetStorageClient.GetTweet): TweetsExist = - (tweetIds: Seq[TweetId]) => - for { - response <- Stitch.traverse(tweetIds) { tweetId => getTweet(tweetId).liftToTry } - tweets <- Stitch.collect(response.map(processTweetResponse)) - } yield tweets.flatten.map(_.id).toSet.filter(tweetIds.contains) - - def apply( - getDeletedTweets: TweetStorageClient.GetDeletedTweets, - tweetsExist: TweetsExist, - stats: StatsReceiver - ): Type = { - - val notFound = stats.counter("not_found") - val notDeleted = stats.counter("not_deleted") - val softDeleted = stats.counter("soft_deleted") - val hardDeleted = stats.counter("hard_deleted") - val unknown = stats.counter("unknown") - - def trackState(results: Seq[GetDeletedTweetResult]): Unit = - results.foreach { r => - r.state match { - case DeletedTweetState.NotFound => notFound.incr() - case DeletedTweetState.NotDeleted => notDeleted.incr() - case DeletedTweetState.SoftDeleted => softDeleted.incr() - case DeletedTweetState.HardDeleted => hardDeleted.incr() - case _ => unknown.incr() - } - } - - FutureArrow { request => - Stitch.run { - Stitch - .join( - getDeletedTweets(request.tweetIds), - tweetsExist(request.tweetIds) - ) - .map { - case (deletedTweetResponses, extantIds) => - val responseIds = deletedTweetResponses.map(_.tweetId) - assert( - responseIds == request.tweetIds, - s"getDeletedTweets response does not match order of request: Request ids " + - s"(${request.tweetIds.mkString(", ")}) != response ids (${responseIds - .mkString(", ")})" - ) - deletedTweetResponses.map { r => convertDeletedTweetResponse(r, extantIds) } - } - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsByUserHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsByUserHandler.docx new file mode 100644 index 000000000..fb37285cf Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsByUserHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsByUserHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsByUserHandler.scala deleted file mode 100644 index c9b096e0f..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsByUserHandler.scala +++ /dev/null @@ -1,188 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.flockdb.client.Cursor -import com.twitter.flockdb.client.PageResult -import com.twitter.flockdb.client.Select -import com.twitter.flockdb.client.StatusGraph -import com.twitter.flockdb.client.UserTimelineGraph -import com.twitter.flockdb.client.thriftscala.EdgeState -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.storage.TweetStorageClient -import com.twitter.tweetypie.storage.TweetStorageClient.GetStoredTweet -import com.twitter.tweetypie.thriftscala.GetStoredTweetsByUserOptions -import com.twitter.tweetypie.thriftscala.GetStoredTweetsByUserRequest -import com.twitter.tweetypie.thriftscala.GetStoredTweetsByUserResult -import com.twitter.tweetypie.thriftscala.GetStoredTweetsOptions -import com.twitter.tweetypie.thriftscala.GetStoredTweetsRequest - -object GetStoredTweetsByUserHandler { - type Type = FutureArrow[GetStoredTweetsByUserRequest, GetStoredTweetsByUserResult] - - def apply( - getStoredTweetsHandler: GetStoredTweetsHandler.Type, - getStoredTweet: TweetStorageClient.GetStoredTweet, - selectPage: FutureArrow[Select[StatusGraph], PageResult[Long]], - maxPages: Int - ): Type = { - FutureArrow { request => - val options = request.options.getOrElse(GetStoredTweetsByUserOptions()) - - val startTimeMsec: Long = options.startTimeMsec.getOrElse(0L) - val endTimeMsec: Long = options.endTimeMsec.getOrElse(Time.now.inMillis) - val cursor = options.cursor.map(Cursor(_)).getOrElse { - if (options.startFromOldest) Cursor.lowest else Cursor.highest - } - - getNextTweetIdsInTimeRange( - request.userId, - startTimeMsec, - endTimeMsec, - cursor, - selectPage, - getStoredTweet, - maxPages, - numTries = 0 - ).flatMap { - case (tweetIds, cursor) => - val getStoredTweetsRequest = toGetStoredTweetsRequest(tweetIds, request.userId, options) - - getStoredTweetsHandler(getStoredTweetsRequest) - .map { getStoredTweetsResults => - GetStoredTweetsByUserResult( - storedTweets = getStoredTweetsResults.map(_.storedTweet), - cursor = if (cursor.isEnd) None else Some(cursor.value) - ) - } - } - } - } - - private def toGetStoredTweetsRequest( - tweetIds: Seq[TweetId], - userId: UserId, - getStoredTweetsByUserOptions: GetStoredTweetsByUserOptions - ): GetStoredTweetsRequest = { - - val options: GetStoredTweetsOptions = GetStoredTweetsOptions( - bypassVisibilityFiltering = getStoredTweetsByUserOptions.bypassVisibilityFiltering, - forUserId = if (getStoredTweetsByUserOptions.setForUserId) Some(userId) else None, - additionalFieldIds = getStoredTweetsByUserOptions.additionalFieldIds - ) - - GetStoredTweetsRequest( - tweetIds = tweetIds, - options = Some(options) - ) - } - - private def getNextTweetIdsInTimeRange( - userId: UserId, - startTimeMsec: Long, - endTimeMsec: Long, - cursor: Cursor, - selectPage: FutureArrow[Select[StatusGraph], PageResult[Long]], - getStoredTweet: TweetStorageClient.GetStoredTweet, - maxPages: Int, - numTries: Int - ): Future[(Seq[TweetId], Cursor)] = { - val select = Select( - sourceId = userId, - graph = UserTimelineGraph, - stateIds = - Some(Seq(EdgeState.Archived.value, EdgeState.Positive.value, EdgeState.Removed.value)) - ).withCursor(cursor) - - def inTimeRange(timestamp: Long): Boolean = - timestamp >= startTimeMsec && timestamp <= endTimeMsec - def pastTimeRange(timestamps: Seq[Long]) = { - if (cursor.isAscending) { - timestamps.max > endTimeMsec - } else { - timestamps.min < startTimeMsec - } - } - - val pageResultFuture: Future[PageResult[Long]] = selectPage(select) - - pageResultFuture.flatMap { pageResult => - val groupedIds = pageResult.entries.groupBy(SnowflakeId.isSnowflakeId) - val nextCursor = if (cursor.isAscending) pageResult.previousCursor else pageResult.nextCursor - - // Timestamps for the creation of Tweets with snowflake IDs can be calculated from the IDs - // themselves. - val snowflakeIdsTimestamps: Seq[(Long, Long)] = groupedIds.getOrElse(true, Seq()).map { id => - val snowflakeTimeMillis = SnowflakeId.unixTimeMillisFromId(id) - (id, snowflakeTimeMillis) - } - - // For non-snowflake Tweets, we need to fetch the Tweet data from Manhattan to see when the - // Tweet was created. - val nonSnowflakeIdsTimestamps: Future[Seq[(Long, Long)]] = Stitch.run( - Stitch - .traverse(groupedIds.getOrElse(false, Seq()))(getStoredTweet) - .map { - _.flatMap { - case GetStoredTweet.Response.FoundAny(tweet, _, _, _, _) => { - if (tweet.coreData.exists(_.createdAtSecs > 0)) { - Some((tweet.id, tweet.coreData.get.createdAtSecs)) - } else None - } - case _ => None - } - }) - - nonSnowflakeIdsTimestamps.flatMap { nonSnowflakeList => - val allTweetIdsAndTimestamps = snowflakeIdsTimestamps ++ nonSnowflakeList - val filteredTweetIds = allTweetIdsAndTimestamps - .filter { - case (_, ts) => inTimeRange(ts) - } - .map(_._1) - - if (nextCursor.isEnd) { - // We've considered the last Tweet for this User. There are no more Tweets to return. - Future.value((filteredTweetIds, Cursor.end)) - } else if (allTweetIdsAndTimestamps.nonEmpty && - pastTimeRange(allTweetIdsAndTimestamps.map(_._2))) { - // At least one Tweet returned from Tflock has a timestamp past our time range, i.e. - // greater than the end time (if we're fetching in an ascending order) or lower than the - // start time (if we're fetching in a descending order). There is no point in looking at - // any more Tweets from this User as they'll all be outside the time range. - Future.value((filteredTweetIds, Cursor.end)) - } else if (filteredTweetIds.isEmpty) { - // We're here because one of two things happened: - // 1. allTweetIdsAndTimestamps is empty: Either Tflock has returned an empty page of Tweets - // or we weren't able to fetch timestamps for any of the Tweets Tflock returned. In this - // case, we fetch the next page of Tweets. - // 2. allTweetIdsAndTimestamps is non-empty but filteredTweetIds is empty: The current page - // has no Tweets inside the requested time range. We fetch the next page of Tweets and - // try again. - // If we hit the limit for the maximum number of pages from tflock to be requested, we - // return an empty list of Tweets with the cursor for the caller to try again. - - if (numTries == maxPages) { - Future.value((filteredTweetIds, nextCursor)) - } else { - getNextTweetIdsInTimeRange( - userId = userId, - startTimeMsec = startTimeMsec, - endTimeMsec = endTimeMsec, - cursor = nextCursor, - selectPage = selectPage, - getStoredTweet = getStoredTweet, - maxPages = maxPages, - numTries = numTries + 1 - ) - } - } else { - // filteredTweetIds is non-empty: There are some Tweets in this page that are within the - // requested time range, and we aren't out of the time range yet. We return the Tweets we - // have and set the cursor forward for the next request. - Future.value((filteredTweetIds, nextCursor)) - } - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsHandler.docx new file mode 100644 index 000000000..5212a1e4c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsHandler.scala deleted file mode 100644 index ab8bfb4ad..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetStoredTweetsHandler.scala +++ /dev/null @@ -1,161 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.StoredTweetResult._ -import com.twitter.tweetypie.core.StoredTweetResult -import com.twitter.tweetypie.core.TweetResult -import com.twitter.tweetypie.FieldId -import com.twitter.tweetypie.FutureArrow -import com.twitter.tweetypie.repository.CacheControl -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.repository.TweetResultRepository -import com.twitter.tweetypie.thriftscala.{BounceDeleted => BounceDeletedState} -import com.twitter.tweetypie.thriftscala.{ForceAdded => ForceAddedState} -import com.twitter.tweetypie.thriftscala.GetStoredTweetsRequest -import com.twitter.tweetypie.thriftscala.GetStoredTweetsOptions -import com.twitter.tweetypie.thriftscala.GetStoredTweetsResult -import com.twitter.tweetypie.thriftscala.{HardDeleted => HardDeletedState} -import com.twitter.tweetypie.thriftscala.{NotFound => NotFoundState} -import com.twitter.tweetypie.thriftscala.{SoftDeleted => SoftDeletedState} -import com.twitter.tweetypie.thriftscala.StatusCounts -import com.twitter.tweetypie.thriftscala.StoredTweetError -import com.twitter.tweetypie.thriftscala.StoredTweetInfo -import com.twitter.tweetypie.thriftscala.StoredTweetState -import com.twitter.tweetypie.thriftscala.{Undeleted => UndeletedState} - -object GetStoredTweetsHandler { - type Type = FutureArrow[GetStoredTweetsRequest, Seq[GetStoredTweetsResult]] - - def apply(tweetRepo: TweetResultRepository.Type): Type = { - FutureArrow[GetStoredTweetsRequest, Seq[GetStoredTweetsResult]] { request => - val requestOptions: GetStoredTweetsOptions = - request.options.getOrElse(GetStoredTweetsOptions()) - val queryOptions = toTweetQueryOptions(requestOptions) - - val result = Stitch - .traverse(request.tweetIds) { tweetId => - tweetRepo(tweetId, queryOptions) - .map(toStoredTweetInfo) - .map(GetStoredTweetsResult(_)) - .handle { - case _ => - GetStoredTweetsResult( - StoredTweetInfo( - tweetId = tweetId, - errors = Seq(StoredTweetError.FailedFetch) - ) - ) - } - } - - Stitch.run(result) - } - } - - private def toTweetQueryOptions(options: GetStoredTweetsOptions): TweetQuery.Options = { - val countsFields: Set[FieldId] = Set( - StatusCounts.FavoriteCountField.id, - StatusCounts.ReplyCountField.id, - StatusCounts.RetweetCountField.id, - StatusCounts.QuoteCountField.id - ) - - TweetQuery.Options( - include = GetTweetsHandler.BaseInclude.also( - tweetFields = Set(Tweet.CountsField.id) ++ options.additionalFieldIds, - countsFields = countsFields - ), - cacheControl = CacheControl.NoCache, - enforceVisibilityFiltering = !options.bypassVisibilityFiltering, - forUserId = options.forUserId, - requireSourceTweet = false, - fetchStoredTweets = true - ) - } - - private def toStoredTweetInfo(tweetResult: TweetResult): StoredTweetInfo = { - def translateErrors(errors: Seq[StoredTweetResult.Error]): Seq[StoredTweetError] = { - errors.map { - case StoredTweetResult.Error.Corrupt => StoredTweetError.Corrupt - case StoredTweetResult.Error.FieldsMissingOrInvalid => - StoredTweetError.FieldsMissingOrInvalid - case StoredTweetResult.Error.ScrubbedFieldsPresent => StoredTweetError.ScrubbedFieldsPresent - case StoredTweetResult.Error.ShouldBeHardDeleted => StoredTweetError.ShouldBeHardDeleted - } - } - - val tweetData = tweetResult.value - - tweetData.storedTweetResult match { - case Some(storedTweetResult) => { - val (tweet, storedTweetState, errors) = storedTweetResult match { - case Present(errors, _) => (Some(tweetData.tweet), None, translateErrors(errors)) - case HardDeleted(softDeletedAtMsec, hardDeletedAtMsec) => - ( - Some(tweetData.tweet), - Some( - StoredTweetState.HardDeleted( - HardDeletedState(softDeletedAtMsec, hardDeletedAtMsec))), - Seq() - ) - case SoftDeleted(softDeletedAtMsec, errors, _) => - ( - Some(tweetData.tweet), - Some(StoredTweetState.SoftDeleted(SoftDeletedState(softDeletedAtMsec))), - translateErrors(errors) - ) - case BounceDeleted(deletedAtMsec, errors, _) => - ( - Some(tweetData.tweet), - Some(StoredTweetState.BounceDeleted(BounceDeletedState(deletedAtMsec))), - translateErrors(errors) - ) - case Undeleted(undeletedAtMsec, errors, _) => - ( - Some(tweetData.tweet), - Some(StoredTweetState.Undeleted(UndeletedState(undeletedAtMsec))), - translateErrors(errors) - ) - case ForceAdded(addedAtMsec, errors, _) => - ( - Some(tweetData.tweet), - Some(StoredTweetState.ForceAdded(ForceAddedState(addedAtMsec))), - translateErrors(errors) - ) - case Failed(errors) => (None, None, translateErrors(errors)) - case NotFound => (None, Some(StoredTweetState.NotFound(NotFoundState())), Seq()) - } - - StoredTweetInfo( - tweetId = tweetData.tweet.id, - tweet = tweet.map(sanitizeNullMediaFields), - storedTweetState = storedTweetState, - errors = errors - ) - } - - case None => - StoredTweetInfo( - tweetId = tweetData.tweet.id, - tweet = Some(sanitizeNullMediaFields(tweetData.tweet)) - ) - } - } - - private def sanitizeNullMediaFields(tweet: Tweet): Tweet = { - // Some media fields are initialized as `null` at the storage layer. - // If the Tweet is meant to be hard deleted, or is not hydrated for - // some other reason but the media entities still exist, we sanitize - // these fields to allow serialization. - tweet.copy(media = tweet.media.map(_.map { mediaEntity => - mediaEntity.copy( - url = Option(mediaEntity.url).getOrElse(""), - mediaUrl = Option(mediaEntity.mediaUrl).getOrElse(""), - mediaUrlHttps = Option(mediaEntity.mediaUrlHttps).getOrElse(""), - displayUrl = Option(mediaEntity.displayUrl).getOrElse(""), - expandedUrl = Option(mediaEntity.expandedUrl).getOrElse(""), - ) - })) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetCountsHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetCountsHandler.docx new file mode 100644 index 000000000..9c2ecfabe Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetCountsHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetCountsHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetCountsHandler.scala deleted file mode 100644 index 4100a76dc..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetCountsHandler.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.servo.util.FutureArrow -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -/** - * Handler for the `getTweetCounts` endpoint. - */ -object GetTweetCountsHandler { - type Type = FutureArrow[GetTweetCountsRequest, Seq[GetTweetCountsResult]] - - def apply(repo: TweetCountsRepository.Type): Type = { - - def idToResult(id: TweetId, req: GetTweetCountsRequest): Stitch[GetTweetCountsResult] = - Stitch - .join( - // .liftToOption() converts any failures to None result - if (req.includeRetweetCount) repo(RetweetsKey(id)).liftToOption() else Stitch.None, - if (req.includeReplyCount) repo(RepliesKey(id)).liftToOption() else Stitch.None, - if (req.includeFavoriteCount) repo(FavsKey(id)).liftToOption() else Stitch.None, - if (req.includeQuoteCount) repo(QuotesKey(id)).liftToOption() else Stitch.None, - if (req.includeBookmarkCount) repo(BookmarksKey(id)).liftToOption() else Stitch.None - ).map { - case (retweetCount, replyCount, favoriteCount, quoteCount, bookmarkCount) => - GetTweetCountsResult( - tweetId = id, - retweetCount = retweetCount, - replyCount = replyCount, - favoriteCount = favoriteCount, - quoteCount = quoteCount, - bookmarkCount = bookmarkCount - ) - } - - FutureArrow[GetTweetCountsRequest, Seq[GetTweetCountsResult]] { request => - Stitch.run( - Stitch.traverse(request.tweetIds)(idToResult(_, request)) - ) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetFieldsHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetFieldsHandler.docx new file mode 100644 index 000000000..b05afcdea Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetFieldsHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetFieldsHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetFieldsHandler.scala deleted file mode 100644 index 55ab6cb18..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetFieldsHandler.scala +++ /dev/null @@ -1,395 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.container.thriftscala.MaterializeAsTweetFieldsRequest -import com.twitter.context.TestingSignalsContext -import com.twitter.servo.util.FutureArrow -import com.twitter.spam.rtf.thriftscala.FilteredReason -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.FilteredState -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository.DeletedTweetVisibilityRepository -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala.TweetFieldsResultState -import com.twitter.tweetypie.thriftscala._ - -/** - * Handler for the `getTweetFields` endpoint. - */ -object GetTweetFieldsHandler { - type Type = GetTweetFieldsRequest => Future[Seq[GetTweetFieldsResult]] - - def apply( - tweetRepo: TweetResultRepository.Type, - deletedTweetVisibilityRepo: DeletedTweetVisibilityRepository.Type, - containerAsGetTweetFieldsResultRepo: CreativesContainerMaterializationRepository.GetTweetFieldsType, - stats: StatsReceiver, - shouldMaterializeContainers: Gate[Unit] - ): Type = { - FutureArrow[GetTweetFieldsRequest, Seq[GetTweetFieldsResult]] { request => - val queryOptions = toTweetQueryOptions(request.options) - - Stitch.run( - Stitch.traverse(request.tweetIds) { id => - tweetRepo(id, queryOptions).liftToTry.flatMap { tweetResult => - toGetTweetFieldsResult( - id, - tweetResult, - request.options, - deletedTweetVisibilityRepo, - containerAsGetTweetFieldsResultRepo, - stats, - shouldMaterializeContainers - ) - } - } - ) - } - } - - /** - * Converts a `GetTweetFieldsOptions` into an internal `TweetQuery.Options`. - */ - def toTweetQueryOptions(options: GetTweetFieldsOptions): TweetQuery.Options = { - val includes = options.tweetIncludes - val shouldSkipCache = TestingSignalsContext().flatMap(_.simulateBackPressure).nonEmpty - val cacheControl = - if (shouldSkipCache) CacheControl.NoCache - else if (options.doNotCache) CacheControl.ReadOnlyCache - else CacheControl.ReadWriteCache - - TweetQuery.Options( - include = TweetQuery - .Include( - tweetFields = includes.collect { - case TweetInclude.TweetFieldId(id) => id - case TweetInclude.CountsFieldId(_) => Tweet.CountsField.id - case TweetInclude.MediaEntityFieldId(_) => Tweet.MediaField.id - }.toSet, - countsFields = includes.collect { case TweetInclude.CountsFieldId(id) => id }.toSet, - mediaFields = includes.collect { case TweetInclude.MediaEntityFieldId(id) => id }.toSet, - quotedTweet = options.includeQuotedTweet, - pastedMedia = true - ).also( - /** - * Always fetching underlying creatives container id. see - * [[hydrateCreativeContainerBackedTweet]] for more detail. - */ - tweetFields = Seq(Tweet.UnderlyingCreativesContainerIdField.id) - ), - cacheControl = cacheControl, - enforceVisibilityFiltering = options.visibilityPolicy == TweetVisibilityPolicy.UserVisible, - safetyLevel = options.safetyLevel.getOrElse(SafetyLevel.FilterNone), - forUserId = options.forUserId, - languageTag = options.languageTag.getOrElse("en"), - cardsPlatformKey = options.cardsPlatformKey, - extensionsArgs = options.extensionsArgs, - forExternalConsumption = true, - simpleQuotedTweet = options.simpleQuotedTweet - ) - } - - def toGetTweetFieldsResult( - tweetId: TweetId, - res: Try[TweetResult], - options: GetTweetFieldsOptions, - deletedTweetVisibilityRepo: DeletedTweetVisibilityRepository.Type, - containerAsGetTweetFieldsResultRepo: CreativesContainerMaterializationRepository.GetTweetFieldsType, - stats: StatsReceiver, - shouldMaterializeContainers: Gate[Unit] - ): Stitch[GetTweetFieldsResult] = { - val measureRacyReads: TweetId => Unit = trackLossyReadsAfterWrite( - stats.stat("racy_reads", "get_tweet_fields"), - Duration.fromSeconds(3) - ) - - res match { - case Throw(NotFound) => - measureRacyReads(tweetId) - Stitch.value(GetTweetFieldsResult(tweetId, NotFoundResultState)) - - case Throw(ex) => - val resultStateStitch = failureResultState(ex) match { - case notFoundResultState @ TweetFieldsResultState.NotFound(_) => - deletedTweetVisibilityRepo( - DeletedTweetVisibilityRepository.VisibilityRequest( - ex, - tweetId, - options.safetyLevel, - options.forUserId, - isInnerQuotedTweet = false - ) - ).map(withVisibilityFilteredReason(notFoundResultState, _)) - case res => Stitch.value(res) - } - resultStateStitch.map(res => GetTweetFieldsResult(tweetId, res)) - case Return(r) => - toTweetFieldsResult( - r, - options, - deletedTweetVisibilityRepo, - containerAsGetTweetFieldsResultRepo, - stats, - shouldMaterializeContainers - ).flatMap { getTweetFieldsResult => - hydrateCreativeContainerBackedTweet( - r.value.tweet.underlyingCreativesContainerId, - getTweetFieldsResult, - options, - containerAsGetTweetFieldsResultRepo, - tweetId, - stats, - shouldMaterializeContainers - ) - } - } - } - - private def failureResultState(ex: Throwable): TweetFieldsResultState = - ex match { - case FilteredState.Unavailable.TweetDeleted => DeletedResultState - case FilteredState.Unavailable.BounceDeleted => BounceDeletedResultState - case FilteredState.Unavailable.SourceTweetNotFound(d) => notFoundResultState(deleted = d) - case FilteredState.Unavailable.Author.NotFound => NotFoundResultState - case fs: FilteredState.HasFilteredReason => toFilteredState(fs.filteredReason) - case OverCapacity(_) => toFailedState(overcapacity = true, None) - case _ => toFailedState(overcapacity = false, Some(ex.toString)) - } - - private val NotFoundResultState = TweetFieldsResultState.NotFound(TweetFieldsResultNotFound()) - - private val DeletedResultState = TweetFieldsResultState.NotFound( - TweetFieldsResultNotFound(deleted = true) - ) - - private val BounceDeletedResultState = TweetFieldsResultState.NotFound( - TweetFieldsResultNotFound(deleted = true, bounceDeleted = true) - ) - - def notFoundResultState(deleted: Boolean): TweetFieldsResultState.NotFound = - if (deleted) DeletedResultState else NotFoundResultState - - private def toFailedState( - overcapacity: Boolean, - message: Option[String] - ): TweetFieldsResultState = - TweetFieldsResultState.Failed(TweetFieldsResultFailed(overcapacity, message)) - - private def toFilteredState(reason: FilteredReason): TweetFieldsResultState = - TweetFieldsResultState.Filtered( - TweetFieldsResultFiltered(reason = reason) - ) - - /** - * Converts a `TweetResult` into a `GetTweetFieldsResult`. For retweets, missing or filtered source - * tweets cause the retweet to be treated as missing or filtered. - */ - private def toTweetFieldsResult( - tweetResult: TweetResult, - options: GetTweetFieldsOptions, - deletedTweetVisibilityRepo: DeletedTweetVisibilityRepository.Type, - creativesContainerRepo: CreativesContainerMaterializationRepository.GetTweetFieldsType, - stats: StatsReceiver, - shouldMaterializeContainers: Gate[Unit] - ): Stitch[GetTweetFieldsResult] = { - val primaryResultState = toTweetFieldsResultState(tweetResult, options) - val quotedResultStateStitch = primaryResultState match { - case TweetFieldsResultState.Found(_) if options.includeQuotedTweet => - val tweetData = tweetResult.value.sourceTweetResult - .getOrElse(tweetResult) - .value - tweetData.quotedTweetResult - .map { - case QuotedTweetResult.NotFound => Stitch.value(NotFoundResultState) - case QuotedTweetResult.Filtered(state) => - val resultState = failureResultState(state) - - (tweetData.tweet.quotedTweet, resultState) match { - //When QT exists => contribute VF filtered reason to result state - case (Some(qt), notFoundResultState @ TweetFieldsResultState.NotFound(_)) => - deletedTweetVisibilityRepo( - DeletedTweetVisibilityRepository.VisibilityRequest( - state, - qt.tweetId, - options.safetyLevel, - options.forUserId, - isInnerQuotedTweet = true - ) - ).map(withVisibilityFilteredReason(notFoundResultState, _)) - //When QT is absent => result state without filtered reason - case _ => Stitch.value(resultState) - } - case QuotedTweetResult.Found(res) => - Stitch - .value(toTweetFieldsResultState(res, options)) - .flatMap { resultState => - hydrateCreativeContainerBackedTweet( - creativesContainerId = res.value.tweet.underlyingCreativesContainerId, - originalGetTweetFieldsResult = GetTweetFieldsResult( - tweetId = res.value.tweet.id, - tweetResult = resultState, - ), - getTweetFieldsRequestOptions = options, - creativesContainerRepo = creativesContainerRepo, - res.value.tweet.id, - stats, - shouldMaterializeContainers - ) - } - .map(_.tweetResult) - } - //Quoted tweet result not requested - case _ => None - } - - quotedResultStateStitch - .map(qtStitch => qtStitch.map(Some(_))) - .getOrElse(Stitch.None) - .map(qtResult => - GetTweetFieldsResult( - tweetId = tweetResult.value.tweet.id, - tweetResult = primaryResultState, - quotedTweetResult = qtResult - )) - } - - /** - * @return a copy of resultState with filtered reason when @param filteredReasonOpt is present - */ - private def withVisibilityFilteredReason( - resultState: TweetFieldsResultState.NotFound, - filteredReasonOpt: Option[FilteredReason] - ): TweetFieldsResultState.NotFound = { - filteredReasonOpt match { - case Some(fs) => - resultState.copy( - notFound = resultState.notFound.copy( - filteredReason = Some(fs) - )) - case _ => resultState - } - } - - private def toTweetFieldsResultState( - tweetResult: TweetResult, - options: GetTweetFieldsOptions - ): TweetFieldsResultState = { - val tweetData = tweetResult.value - val suppressReason = tweetData.suppress.map(_.filteredReason) - val tweetFailedFields = tweetResult.state.failedFields - val sourceTweetFailedFields = - tweetData.sourceTweetResult.map(_.state.failedFields).getOrElse(Set()) - val sourceTweetOpt = tweetData.sourceTweetResult.map(_.value.tweet) - val sourceTweetSuppressReason = - tweetData.sourceTweetResult.flatMap(_.value.suppress.map(_.filteredReason)) - val isTweetPartial = tweetFailedFields.nonEmpty || sourceTweetFailedFields.nonEmpty - - val tweetFoundResult = tweetData.sourceTweetResult match { - case None => - // if `sourceTweetResult` is empty, this isn't a retweet - TweetFieldsResultFound( - tweet = tweetData.tweet, - suppressReason = suppressReason - ) - case Some(r) => - // if the source tweet result state is Found, merge that into the primary result - TweetFieldsResultFound( - tweet = tweetData.tweet, - retweetedTweet = sourceTweetOpt.filter(_ => options.includeRetweetedTweet), - suppressReason = suppressReason.orElse(sourceTweetSuppressReason) - ) - } - - if (isTweetPartial) { - TweetFieldsResultState.Failed( - TweetFieldsResultFailed( - overCapacity = false, - message = Some( - "Failed to load: " + (tweetFailedFields ++ sourceTweetFailedFields).mkString(", ")), - partial = Some( - TweetFieldsPartial( - found = tweetFoundResult, - missingFields = tweetFailedFields, - sourceTweetMissingFields = sourceTweetFailedFields - ) - ) - ) - ) - } else { - TweetFieldsResultState.Found( - tweetFoundResult - ) - } - } - - /** - * if tweet data is backed by creatives container, it'll be hydrated from creatives - * container service. - */ - private def hydrateCreativeContainerBackedTweet( - creativesContainerId: Option[Long], - originalGetTweetFieldsResult: GetTweetFieldsResult, - getTweetFieldsRequestOptions: GetTweetFieldsOptions, - creativesContainerRepo: CreativesContainerMaterializationRepository.GetTweetFieldsType, - tweetId: Long, - stats: StatsReceiver, - shouldMaterializeContainers: Gate[Unit] - ): Stitch[GetTweetFieldsResult] = { - // creatives container backed tweet stats - val ccTweetMaterialized = stats.scope("creatives_container", "get_tweet_fields") - val ccTweetMaterializeRequests = ccTweetMaterialized.counter("requests") - val ccTweetMaterializeSuccess = ccTweetMaterialized.counter("success") - val ccTweetMaterializeFailed = ccTweetMaterialized.counter("failed") - val ccTweetMaterializeFiltered = ccTweetMaterialized.scope("filtered") - - ( - creativesContainerId, - originalGetTweetFieldsResult.tweetResult, - getTweetFieldsRequestOptions.disableTweetMaterialization, - shouldMaterializeContainers() - ) match { - // 1. creatives container backed tweet is determined by `underlyingCreativesContainerId` field presence. - // 2. if the frontend tweet is suppressed by any reason, respect that and not do this hydration. - // (this logic can be revisited and improved further) - case (None, _, _, _) => - Stitch.value(originalGetTweetFieldsResult) - case (Some(_), _, _, false) => - ccTweetMaterializeFiltered.counter("decider_suppressed").incr() - Stitch.value { - GetTweetFieldsResult( - tweetId = tweetId, - tweetResult = TweetFieldsResultState.NotFound(TweetFieldsResultNotFound()) - ) - } - case (Some(containerId), TweetFieldsResultState.Found(_), false, _) => - ccTweetMaterializeRequests.incr() - val materializationRequest = - MaterializeAsTweetFieldsRequest(containerId, tweetId, Some(originalGetTweetFieldsResult)) - creativesContainerRepo( - materializationRequest, - getTweetFieldsRequestOptions - ).onSuccess(_ => ccTweetMaterializeSuccess.incr()) - .onFailure(_ => ccTweetMaterializeFailed.incr()) - .handle { - case ex => - GetTweetFieldsResult( - tweetId = tweetId, - tweetResult = failureResultState(ex) - ) - } - case (Some(_), _, true, _) => - ccTweetMaterializeFiltered.counter("suppressed").incr() - Stitch.value( - GetTweetFieldsResult( - tweetId = tweetId, - tweetResult = TweetFieldsResultState.NotFound(TweetFieldsResultNotFound()) - ) - ) - case (Some(_), state, _, _) => - ccTweetMaterializeFiltered.counter(state.getClass.getName).incr() - Stitch.value(originalGetTweetFieldsResult) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetsHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetsHandler.docx new file mode 100644 index 000000000..3fd7e5608 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetsHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetsHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetsHandler.scala deleted file mode 100644 index f0f144dd5..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/GetTweetsHandler.scala +++ /dev/null @@ -1,415 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.container.thriftscala.MaterializeAsTweetRequest -import com.twitter.context.TestingSignalsContext -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.servo.exception.thriftscala.ClientErrorCause -import com.twitter.servo.util.FutureArrow -import com.twitter.spam.rtf.thriftscala.FilteredReason -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.additionalfields.AdditionalFields -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -/** - * Handler for the `getTweets` endpoint. - */ -object GetTweetsHandler { - type Type = FutureArrow[GetTweetsRequest, Seq[GetTweetResult]] - - /** - * A `TweetQuery.Include` instance with options set as the default base options - * for the `getTweets` endpoint. - */ - val BaseInclude: TweetQuery.Include = - TweetQuery.Include( - tweetFields = Set( - Tweet.CoreDataField.id, - Tweet.UrlsField.id, - Tweet.MentionsField.id, - Tweet.MediaField.id, - Tweet.HashtagsField.id, - Tweet.CashtagsField.id, - Tweet.TakedownCountryCodesField.id, - Tweet.TakedownReasonsField.id, - Tweet.DeviceSourceField.id, - Tweet.LanguageField.id, - Tweet.ContributorField.id, - Tweet.QuotedTweetField.id, - Tweet.UnderlyingCreativesContainerIdField.id, - ), - pastedMedia = true - ) - - def apply( - tweetRepo: TweetResultRepository.Type, - creativesContainerRepo: CreativesContainerMaterializationRepository.GetTweetType, - deletedTweetVisibilityRepo: DeletedTweetVisibilityRepository.Type, - stats: StatsReceiver, - shouldMaterializeContainers: Gate[Unit] - ): Type = { - FutureArrow[GetTweetsRequest, Seq[GetTweetResult]] { request => - val requestOptions = request.options.getOrElse(GetTweetOptions()) - - val invalidAdditionalFields = - requestOptions.additionalFieldIds.filter(!AdditionalFields.isAdditionalFieldId(_)) - - if (invalidAdditionalFields.nonEmpty) { - Future.exception( - ClientError( - ClientErrorCause.BadRequest, - "Requested additional fields contain invalid field id " + - s"${invalidAdditionalFields.mkString(", ")}. Additional fields ids must be greater than 100." - ) - ) - } else { - val opts = toTweetQueryOptions(requestOptions) - val measureRacyReads: TweetId => Unit = trackLossyReadsAfterWrite( - stats.stat("racy_reads", "get_tweets"), - Duration.fromSeconds(3) - ) - - Stitch.run( - Stitch.traverse(request.tweetIds) { id => - tweetRepo(id, opts).liftToTry - .flatMap { - case Throw(NotFound) => - measureRacyReads(id) - - Stitch.value(GetTweetResult(id, StatusState.NotFound)) - case Throw(ex) => - failureResult(deletedTweetVisibilityRepo, id, requestOptions, ex) - case Return(r) => - toGetTweetResult( - deletedTweetVisibilityRepo, - creativesContainerRepo, - requestOptions, - tweetResult = r, - includeSourceTweet = requestOptions.includeSourceTweet, - includeQuotedTweet = requestOptions.includeQuotedTweet, - stats, - shouldMaterializeContainers - ) - }.flatMap { getTweetResult => - // check if tweet data is backed by creatives container and needs to be hydrated from creatives - // container service. - hydrateCreativeContainerBackedTweet( - getTweetResult, - requestOptions, - creativesContainerRepo, - stats, - shouldMaterializeContainers - ) - } - } - ) - } - } - } - - def toTweetQueryOptions(options: GetTweetOptions): TweetQuery.Options = { - val shouldSkipCache = TestingSignalsContext().flatMap(_.simulateBackPressure).nonEmpty - val cacheControl = - if (shouldSkipCache) CacheControl.NoCache - else if (options.doNotCache) CacheControl.ReadOnlyCache - else CacheControl.ReadWriteCache - - val countsFields = toCountsFields(options) - val mediaFields = toMediaFields(options) - - TweetQuery.Options( - include = BaseInclude.also( - tweetFields = toTweetFields(options, countsFields), - countsFields = countsFields, - mediaFields = mediaFields, - quotedTweet = Some(options.includeQuotedTweet) - ), - cacheControl = cacheControl, - cardsPlatformKey = options.cardsPlatformKey, - excludeReported = options.excludeReported, - enforceVisibilityFiltering = !options.bypassVisibilityFiltering, - safetyLevel = options.safetyLevel.getOrElse(SafetyLevel.FilterDefault), - forUserId = options.forUserId, - languageTag = options.languageTag, - extensionsArgs = options.extensionsArgs, - forExternalConsumption = true, - simpleQuotedTweet = options.simpleQuotedTweet - ) - } - - private def toTweetFields(opts: GetTweetOptions, countsFields: Set[FieldId]): Set[FieldId] = { - val bldr = Set.newBuilder[FieldId] - - bldr ++= opts.additionalFieldIds - - if (opts.includePlaces) bldr += Tweet.PlaceField.id - if (opts.forUserId.nonEmpty) { - if (opts.includePerspectivals) bldr += Tweet.PerspectiveField.id - if (opts.includeConversationMuted) bldr += Tweet.ConversationMutedField.id - } - if (opts.includeCards && opts.cardsPlatformKey.isEmpty) bldr += Tweet.CardsField.id - if (opts.includeCards && opts.cardsPlatformKey.nonEmpty) bldr += Tweet.Card2Field.id - if (opts.includeProfileGeoEnrichment) bldr += Tweet.ProfileGeoEnrichmentField.id - - if (countsFields.nonEmpty) bldr += Tweet.CountsField.id - - if (opts.includeCardUri) bldr += Tweet.CardReferenceField.id - - bldr.result() - } - - private def toCountsFields(opts: GetTweetOptions): Set[FieldId] = { - val bldr = Set.newBuilder[FieldId] - - if (opts.includeRetweetCount) bldr += StatusCounts.RetweetCountField.id - if (opts.includeReplyCount) bldr += StatusCounts.ReplyCountField.id - if (opts.includeFavoriteCount) bldr += StatusCounts.FavoriteCountField.id - if (opts.includeQuoteCount) bldr += StatusCounts.QuoteCountField.id - - bldr.result() - } - - private def toMediaFields(opts: GetTweetOptions): Set[FieldId] = { - if (opts.includeMediaAdditionalMetadata) - Set(MediaEntity.AdditionalMetadataField.id) - else - Set.empty - } - - /** - * Converts a `TweetResult` into a `GetTweetResult`. - */ - def toGetTweetResult( - deletedTweetVisibilityRepo: DeletedTweetVisibilityRepository.Type, - creativesContainerRepo: CreativesContainerMaterializationRepository.GetTweetType, - options: GetTweetOptions, - tweetResult: TweetResult, - includeSourceTweet: Boolean, - includeQuotedTweet: Boolean, - stats: StatsReceiver, - shouldMaterializeContainers: Gate[Unit] - ): Stitch[GetTweetResult] = { - val tweetData = tweetResult.value - - // only include missing fields if non empty - def asMissingFields(set: Set[FieldByPath]): Option[Set[FieldByPath]] = - if (set.isEmpty) None else Some(set) - - val missingFields = asMissingFields(tweetResult.state.failedFields) - - val sourceTweetResult = - tweetData.sourceTweetResult - .filter(_ => includeSourceTweet) - - val sourceTweetData = tweetData.sourceTweetResult - .getOrElse(tweetResult) - .value - val quotedTweetResult: Option[QuotedTweetResult] = sourceTweetData.quotedTweetResult - .filter(_ => includeQuotedTweet) - - val qtFilteredReasonStitch = - ((sourceTweetData.tweet.quotedTweet, quotedTweetResult) match { - case (Some(quotedTweet), Some(QuotedTweetResult.Filtered(filteredState))) => - deletedTweetVisibilityRepo( - DeletedTweetVisibilityRepository.VisibilityRequest( - filteredState, - quotedTweet.tweetId, - options.safetyLevel, - options.forUserId, - isInnerQuotedTweet = true - ) - ) - case _ => Stitch.None - }) - //Use quotedTweetResult filtered reason when VF filtered reason is not present - .map(fsOpt => fsOpt.orElse(quotedTweetResult.flatMap(_.filteredReason))) - - val suppress = tweetData.suppress.orElse(tweetData.sourceTweetResult.flatMap(_.value.suppress)) - - val quotedTweetStitch: Stitch[Option[Tweet]] = - quotedTweetResult match { - // check if quote tweet is backed by creatives container and needs to be hydrated from creatives - // container service. detail see go/creatives-containers-tdd - case Some(QuotedTweetResult.Found(tweetResult)) => - hydrateCreativeContainerBackedTweet( - originalGetTweetResult = GetTweetResult( - tweetId = tweetResult.value.tweet.id, - tweetState = StatusState.Found, - tweet = Some(tweetResult.value.tweet) - ), - getTweetRequestOptions = options, - creativesContainerRepo = creativesContainerRepo, - stats = stats, - shouldMaterializeContainers - ).map(_.tweet) - case _ => - Stitch.value( - quotedTweetResult - .flatMap(_.toOption) - .map(_.value.tweet) - ) - } - - Stitch.join(qtFilteredReasonStitch, quotedTweetStitch).map { - case (qtFilteredReason, quotedTweet) => - GetTweetResult( - tweetId = tweetData.tweet.id, - tweetState = - if (suppress.nonEmpty) StatusState.Suppress - else if (missingFields.nonEmpty) StatusState.Partial - else StatusState.Found, - tweet = Some(tweetData.tweet), - missingFields = missingFields, - filteredReason = suppress.map(_.filteredReason), - sourceTweet = sourceTweetResult.map(_.value.tweet), - sourceTweetMissingFields = sourceTweetResult - .map(_.state.failedFields) - .flatMap(asMissingFields), - quotedTweet = quotedTweet, - quotedTweetMissingFields = quotedTweetResult - .flatMap(_.toOption) - .map(_.state.failedFields) - .flatMap(asMissingFields), - quotedTweetFilteredReason = qtFilteredReason - ) - } - } - - private[this] val AuthorAccountIsInactive = FilteredReason.AuthorAccountIsInactive(true) - - def failureResult( - deletedTweetVisibilityRepo: DeletedTweetVisibilityRepository.Type, - tweetId: TweetId, - options: GetTweetOptions, - ex: Throwable - ): Stitch[GetTweetResult] = { - def deletedState(deleted: Boolean, statusState: StatusState) = - if (deleted && options.enableDeletedState) { - statusState - } else { - StatusState.NotFound - } - - ex match { - case FilteredState.Unavailable.Author.Deactivated => - Stitch.value(GetTweetResult(tweetId, StatusState.DeactivatedUser)) - case FilteredState.Unavailable.Author.NotFound => - Stitch.value(GetTweetResult(tweetId, StatusState.NotFound)) - case FilteredState.Unavailable.Author.Offboarded => - Stitch.value( - GetTweetResult(tweetId, StatusState.Drop, filteredReason = Some(AuthorAccountIsInactive))) - case FilteredState.Unavailable.Author.Suspended => - Stitch.value(GetTweetResult(tweetId, StatusState.SuspendedUser)) - case FilteredState.Unavailable.Author.Protected => - Stitch.value(GetTweetResult(tweetId, StatusState.ProtectedUser)) - case FilteredState.Unavailable.Author.Unsafe => - Stitch.value(GetTweetResult(tweetId, StatusState.Drop)) - //Handle delete state with optional FilteredReason - case FilteredState.Unavailable.TweetDeleted => - deletedTweetVisibilityRepo( - DeletedTweetVisibilityRepository.VisibilityRequest( - ex, - tweetId, - options.safetyLevel, - options.forUserId, - isInnerQuotedTweet = false - ) - ).map(filteredReasonOpt => { - val deleteState = deletedState(deleted = true, StatusState.Deleted) - GetTweetResult(tweetId, deleteState, filteredReason = filteredReasonOpt) - }) - - case FilteredState.Unavailable.BounceDeleted => - deletedTweetVisibilityRepo( - DeletedTweetVisibilityRepository.VisibilityRequest( - ex, - tweetId, - options.safetyLevel, - options.forUserId, - isInnerQuotedTweet = false - ) - ).map(filteredReasonOpt => { - val deleteState = deletedState(deleted = true, StatusState.BounceDeleted) - GetTweetResult(tweetId, deleteState, filteredReason = filteredReasonOpt) - }) - - case FilteredState.Unavailable.SourceTweetNotFound(d) => - deletedTweetVisibilityRepo( - DeletedTweetVisibilityRepository.VisibilityRequest( - ex, - tweetId, - options.safetyLevel, - options.forUserId, - isInnerQuotedTweet = false - ) - ).map(filteredReasonOpt => { - val deleteState = deletedState(d, StatusState.Deleted) - GetTweetResult(tweetId, deleteState, filteredReason = filteredReasonOpt) - }) - case FilteredState.Unavailable.Reported => - Stitch.value(GetTweetResult(tweetId, StatusState.ReportedTweet)) - case fs: FilteredState.HasFilteredReason => - Stitch.value( - GetTweetResult(tweetId, StatusState.Drop, filteredReason = Some(fs.filteredReason))) - case OverCapacity(_) => Stitch.value(GetTweetResult(tweetId, StatusState.OverCapacity)) - case _ => Stitch.value(GetTweetResult(tweetId, StatusState.Failed)) - } - } - - private def hydrateCreativeContainerBackedTweet( - originalGetTweetResult: GetTweetResult, - getTweetRequestOptions: GetTweetOptions, - creativesContainerRepo: CreativesContainerMaterializationRepository.GetTweetType, - stats: StatsReceiver, - shouldMaterializeContainers: Gate[Unit] - ): Stitch[GetTweetResult] = { - // creatives container backed tweet stats - val ccTweetMaterialized = stats.scope("creatives_container", "get_tweets") - val ccTweetMaterializeFiltered = ccTweetMaterialized.scope("filtered") - val ccTweetMaterializeSuccess = ccTweetMaterialized.counter("success") - val ccTweetMaterializeFailed = ccTweetMaterialized.counter("failed") - val ccTweetMaterializeRequests = ccTweetMaterialized.counter("requests") - - val tweetId = originalGetTweetResult.tweetId - val tweetState = originalGetTweetResult.tweetState - val underlyingCreativesContainerId = - originalGetTweetResult.tweet.flatMap(_.underlyingCreativesContainerId) - ( - tweetState, - underlyingCreativesContainerId, - getTweetRequestOptions.disableTweetMaterialization, - shouldMaterializeContainers() - ) match { - // 1. creatives container backed tweet is determined by `underlyingCreativesContainerId` field presence. - // 2. if the frontend tweet is suppressed by any reason, respect that and not do this hydration. - // (this logic can be revisited and improved further) - case (_, None, _, _) => - Stitch.value(originalGetTweetResult) - case (_, Some(_), _, false) => - ccTweetMaterializeFiltered.counter("decider_suppressed").incr() - Stitch.value(GetTweetResult(tweetId, StatusState.NotFound)) - case (StatusState.Found, Some(containerId), false, _) => - ccTweetMaterializeRequests.incr() - val materializationRequest = - MaterializeAsTweetRequest(containerId, tweetId, Some(originalGetTweetResult)) - creativesContainerRepo( - materializationRequest, - Some(getTweetRequestOptions) - ).onSuccess(_ => ccTweetMaterializeSuccess.incr()) - .onFailure(_ => ccTweetMaterializeFailed.incr()) - .handle { - case _ => GetTweetResult(tweetId, StatusState.Failed) - } - case (_, Some(_), true, _) => - ccTweetMaterializeFiltered.counter("suppressed").incr() - Stitch.value(GetTweetResult(tweetId, StatusState.NotFound)) - case (state, Some(_), _, _) => - ccTweetMaterializeFiltered.counter(state.name).incr() - Stitch.value(originalGetTweetResult) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/HandlerError.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/HandlerError.docx new file mode 100644 index 000000000..ece947e87 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/HandlerError.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/HandlerError.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/HandlerError.scala deleted file mode 100644 index 6ec0fc611..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/HandlerError.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.servo.exception.thriftscala.ClientErrorCause -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.FilteredState.Unavailable._ - -private[tweetypie] object HandlerError { - - def translateNotFoundToClientError[U](tweetId: TweetId): PartialFunction[Throwable, Stitch[U]] = { - case NotFound => - Stitch.exception(HandlerError.tweetNotFound(tweetId)) - case TweetDeleted | BounceDeleted => - Stitch.exception(HandlerError.tweetNotFound(tweetId, true)) - case SourceTweetNotFound(deleted) => - Stitch.exception(HandlerError.tweetNotFound(tweetId, deleted)) - } - - def tweetNotFound(tweetId: TweetId, deleted: Boolean = false): ClientError = - ClientError( - ClientErrorCause.BadRequest, - s"tweet ${if (deleted) "deleted" else "not found"}: $tweetId" - ) - - def userNotFound(userId: UserId): ClientError = - ClientError(ClientErrorCause.BadRequest, s"user not found: $userId") - - def tweetNotFoundException(tweetId: TweetId): Future[Nothing] = - Future.exception(tweetNotFound(tweetId)) - - def userNotFoundException(userId: UserId): Future[Nothing] = - Future.exception(userNotFound(userId)) - - def getRequired[A, B]( - optionFutureArrow: FutureArrow[A, Option[B]], - notFound: A => Future[B] - ): FutureArrow[A, B] = - FutureArrow(key => - optionFutureArrow(key).flatMap { - case Some(x) => Future.value(x) - case None => notFound(key) - }) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/MediaBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/MediaBuilder.docx new file mode 100644 index 000000000..354273ef1 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/MediaBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/MediaBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/MediaBuilder.scala deleted file mode 100644 index 560c51304..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/MediaBuilder.scala +++ /dev/null @@ -1,176 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.mediaservices.commons.mediainformation.thriftscala.UserDefinedProductMetadata -import com.twitter.mediaservices.commons.thriftscala.MediaKey -import com.twitter.mediaservices.commons.tweetmedia.thriftscala._ -import com.twitter.servo.util.FutureArrow -import com.twitter.tco_util.TcoSlug -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.media._ -import com.twitter.tweetypie.serverutil.ExceptionCounter -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.tweettext.Offset - -object CreateMediaTco { - import UpstreamFailure._ - - case class Request( - tweetId: TweetId, - userId: UserId, - userScreenName: String, - isProtected: Boolean, - createdAt: Time, - isVideo: Boolean, - dark: Boolean) - - type Type = FutureArrow[Request, Media.MediaTco] - - def apply(urlShortener: UrlShortener.Type): Type = - FutureArrow[Request, Media.MediaTco] { req => - val expandedUrl = MediaUrl.Permalink(req.userScreenName, req.tweetId, req.isVideo) - val shortenCtx = - UrlShortener.Context( - userId = req.userId, - userProtected = req.isProtected, - tweetId = req.tweetId, - createdAt = req.createdAt, - dark = req.dark - ) - - urlShortener((expandedUrl, shortenCtx)) - .flatMap { metadata => - metadata.shortUrl match { - case TcoSlug(slug) => - Future.value( - Media.MediaTco( - expandedUrl, - metadata.shortUrl, - MediaUrl.Display.fromTcoSlug(slug) - ) - ) - - case _ => - // should never get here, since shortened urls from talon - // always start with "http://t.co/", just in case... - Future.exception(MediaShortenUrlMalformedFailure) - } - } - .rescue { - case UrlShortener.InvalidUrlError => - // should never get here, since media expandedUrl should always be a valid - // input to talon. - Future.exception(MediaExpandedUrlNotValidFailure) - } - } -} - -object MediaBuilder { - private val log = Logger(getClass) - - case class Request( - mediaUploadIds: Seq[MediaId], - text: String, - tweetId: TweetId, - userId: UserId, - userScreenName: String, - isProtected: Boolean, - createdAt: Time, - dark: Boolean = false, - productMetadata: Option[Map[MediaId, UserDefinedProductMetadata]] = None) - - case class Result(updatedText: String, mediaEntities: Seq[MediaEntity], mediaKeys: Seq[MediaKey]) - - type Type = FutureArrow[Request, Result] - - def apply( - processMedia: MediaClient.ProcessMedia, - createMediaTco: CreateMediaTco.Type, - stats: StatsReceiver - ): Type = - FutureArrow[Request, Result] { - case Request( - mediaUploadIds, - text, - tweetId, - userId, - screenName, - isProtected, - createdAt, - dark, - productMetadata - ) => - for { - mediaKeys <- processMedia( - ProcessMediaRequest( - mediaUploadIds, - userId, - tweetId, - isProtected, - productMetadata - ) - ) - mediaTco <- createMediaTco( - CreateMediaTco.Request( - tweetId, - userId, - screenName, - isProtected, - createdAt, - mediaKeys.exists(MediaKeyClassifier.isVideo(_)), - dark - ) - ) - } yield produceResult(text, mediaTco, isProtected, mediaKeys) - }.countExceptions( - ExceptionCounter(stats) - ) - .onFailure[Request] { (req, ex) => log.info(req.toString, ex) } - .translateExceptions { - case e: MediaExceptions.MediaClientException => - TweetCreateFailure.State(TweetCreateState.InvalidMedia, Some(e.getMessage)) - } - - def produceResult( - text: String, - mediaTco: Media.MediaTco, - userIsProtected: Boolean, - mediaKeys: Seq[MediaKey] - ): Result = { - - val newText = - if (text == "") mediaTco.url - else text + " " + mediaTco.url - - val to = Offset.CodePoint.length(newText) - val from = to - Offset.CodePoint.length(mediaTco.url) - - val mediaEntities = - mediaKeys.map { mediaKey => - MediaEntity( - mediaKey = Some(mediaKey), - fromIndex = from.toShort, - toIndex = to.toShort, - url = mediaTco.url, - displayUrl = mediaTco.displayUrl, - expandedUrl = mediaTco.expandedUrl, - mediaId = mediaKey.mediaId, - mediaPath = "", // to be hydrated - mediaUrl = null, // to be hydrated - mediaUrlHttps = null, // to be hydrated - nsfw = false, // deprecated - sizes = Set( - MediaSize( - sizeType = MediaSizeType.Orig, - resizeMethod = MediaResizeMethod.Fit, - deprecatedContentType = MediaKeyUtil.contentType(mediaKey), - width = -1, // to be hydrated - height = -1 // to be hydrated - ) - ) - ) - } - - Result(newText, mediaEntities, mediaKeys) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/PostTweet.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/PostTweet.docx new file mode 100644 index 000000000..051f2e1de Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/PostTweet.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/PostTweet.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/PostTweet.scala deleted file mode 100644 index 2ee6d1063..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/PostTweet.scala +++ /dev/null @@ -1,395 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.context.thriftscala.FeatureContext -import com.twitter.tweetypie.backends.LimiterService -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.serverutil.ExceptionCounter -import com.twitter.tweetypie.store.InsertTweet -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.util.TweetCreationLock.{Key => TweetCreationLockKey} - -object PostTweet { - type Type[R] = FutureArrow[R, PostTweetResult] - - /** - * A type-class to abstract over tweet creation requests. - */ - trait RequestView[R] { - def isDark(req: R): Boolean - def sourceTweetId(req: R): Option[TweetId] - def options(req: R): Option[WritePathHydrationOptions] - def userId(req: R): UserId - def uniquenessId(req: R): Option[Long] - def returnSuccessOnDuplicate(req: R): Boolean - def returnDuplicateTweet(req: R): Boolean = - returnSuccessOnDuplicate(req) || uniquenessId(req).nonEmpty - def lockKey(req: R): TweetCreationLockKey - def geo(req: R): Option[TweetCreateGeo] - def featureContext(req: R): Option[FeatureContext] - def additionalContext(req: R): Option[collection.Map[TweetCreateContextKey, String]] - def transientContext(req: R): Option[TransientCreateContext] - def additionalFields(req: R): Option[Tweet] - def duplicateState: TweetCreateState - def scope: String - def isNullcast(req: R): Boolean - def creativesContainerId(req: R): Option[CreativesContainerId] - def noteTweetMentionedUserIds(req: R): Option[Seq[Long]] - } - - /** - * An implementation of `RequestView` for `PostTweetRequest`. - */ - implicit object PostTweetRequestView extends RequestView[PostTweetRequest] { - def isDark(req: PostTweetRequest): Boolean = req.dark - def sourceTweetId(req: PostTweetRequest): None.type = None - def options(req: PostTweetRequest): Option[WritePathHydrationOptions] = req.hydrationOptions - def userId(req: PostTweetRequest): UserId = req.userId - def uniquenessId(req: PostTweetRequest): Option[Long] = req.uniquenessId - def returnSuccessOnDuplicate(req: PostTweetRequest) = false - def lockKey(req: PostTweetRequest): TweetCreationLockKey = TweetCreationLockKey.byRequest(req) - def geo(req: PostTweetRequest): Option[TweetCreateGeo] = req.geo - def featureContext(req: PostTweetRequest): Option[FeatureContext] = req.featureContext - def additionalContext( - req: PostTweetRequest - ): Option[collection.Map[TweetCreateContextKey, String]] = req.additionalContext - def transientContext(req: PostTweetRequest): Option[TransientCreateContext] = - req.transientContext - def additionalFields(req: PostTweetRequest): Option[Tweet] = req.additionalFields - def duplicateState: TweetCreateState.Duplicate.type = TweetCreateState.Duplicate - def scope = "tweet" - def isNullcast(req: PostTweetRequest): Boolean = req.nullcast - def creativesContainerId(req: PostTweetRequest): Option[CreativesContainerId] = - req.underlyingCreativesContainerId - def noteTweetMentionedUserIds(req: PostTweetRequest): Option[Seq[Long]] = - req.noteTweetOptions match { - case Some(noteTweetOptions) => noteTweetOptions.mentionedUserIds - case _ => None - } - } - - /** - * An implementation of `RequestView` for `RetweetRequest`. - */ - implicit object RetweetRequestView extends RequestView[RetweetRequest] { - def isDark(req: RetweetRequest): Boolean = req.dark - def sourceTweetId(req: RetweetRequest): None.type = None - def options(req: RetweetRequest): Option[WritePathHydrationOptions] = req.hydrationOptions - def userId(req: RetweetRequest): UserId = req.userId - def uniquenessId(req: RetweetRequest): Option[Long] = req.uniquenessId - def returnSuccessOnDuplicate(req: RetweetRequest): Boolean = req.returnSuccessOnDuplicate - def lockKey(req: RetweetRequest): TweetCreationLockKey = - req.uniquenessId match { - case Some(id) => TweetCreationLockKey.byUniquenessId(req.userId, id) - case None => TweetCreationLockKey.bySourceTweetId(req.userId, req.sourceStatusId) - } - def geo(req: RetweetRequest): None.type = None - def featureContext(req: RetweetRequest): Option[FeatureContext] = req.featureContext - def additionalContext(req: RetweetRequest): None.type = None - def transientContext(req: RetweetRequest): None.type = None - def additionalFields(req: RetweetRequest): Option[Tweet] = req.additionalFields - def duplicateState: TweetCreateState.AlreadyRetweeted.type = TweetCreateState.AlreadyRetweeted - def scope = "retweet" - def isNullcast(req: RetweetRequest): Boolean = req.nullcast - def creativesContainerId(req: RetweetRequest): Option[CreativesContainerId] = None - def noteTweetMentionedUserIds(req: RetweetRequest): Option[Seq[Long]] = None - } - - /** - * A `Filter` is used to decorate a `FutureArrow` that has a known return type - * and an input type for which there is a `RequestView` type-class instance. - */ - trait Filter[Res] { self => - type T[Req] = FutureArrow[Req, Res] - - /** - * Wraps a base arrow with additional behavior. - */ - def apply[Req: RequestView](base: T[Req]): T[Req] - - /** - * Composes two filter. The resulting filter itself composes FutureArrows. - */ - def andThen(next: Filter[Res]): Filter[Res] = - new Filter[Res] { - def apply[Req: RequestView](base: T[Req]): T[Req] = - next(self(base)) - } - } - - /** - * This filter attempts to prevent some race-condition related duplicate tweet creations, - * via use of a `TweetCreateLock`. When a duplicate is detected, this filter can synthesize - * a successful `PostTweetResult` if applicable, or return the appropriate coded response. - */ - object DuplicateHandler { - def apply( - tweetCreationLock: TweetCreationLock, - getTweets: GetTweetsHandler.Type, - stats: StatsReceiver - ): Filter[PostTweetResult] = - new Filter[PostTweetResult] { - def apply[R: RequestView](base: T[R]): T[R] = { - val view = implicitly[RequestView[R]] - val notFoundCount = stats.counter(view.scope, "not_found") - val foundCounter = stats.counter(view.scope, "found") - - FutureArrow.rec[R, PostTweetResult] { self => req => - val duplicateKey = view.lockKey(req) - - // attempts to find the duplicate tweet. - // - // if `returnDupTweet` is true and we find the tweet, then we return a - // successful `PostTweetResult` with that tweet. if we don't find the - // tweet, we throw an `InternalServerError`. - // - // if `returnDupTweet` is false and we find the tweet, then we return - // the appropriate duplicate state. if we don't find the tweet, then - // we unlock the duplicate key and try again. - def duplicate(tweetId: TweetId, returnDupTweet: Boolean) = - findDuplicate(tweetId, req).flatMap { - case Some(postTweetResult) => - foundCounter.incr() - if (returnDupTweet) Future.value(postTweetResult) - else Future.value(PostTweetResult(state = view.duplicateState)) - - case None => - notFoundCount.incr() - if (returnDupTweet) { - // If we failed to load the tweet, but we know that it - // should exist, then return an InternalServerError, so that - // the client treats it as a failed tweet creation req. - Future.exception( - InternalServerError("Failed to load duplicate existing tweet: " + tweetId) - ) - } else { - // Assume the lock is stale if we can't load the tweet. It's - // possible that the lock is not stale, but the tweet is not - // yet available, which requires that it not be present in - // cache and not yet available from the backend. This means - // that the failure mode is to allow tweeting if we can't - // determine the state, but it should be rare that we can't - // determine it. - tweetCreationLock.unlock(duplicateKey).before(self(req)) - } - } - - tweetCreationLock(duplicateKey, view.isDark(req), view.isNullcast(req)) { - base(req) - }.rescue { - case TweetCreationInProgress => - Future.value(PostTweetResult(state = TweetCreateState.Duplicate)) - - // if tweetCreationLock detected a duplicate, look up the duplicate - // and return the appropriate result - case DuplicateTweetCreation(tweetId) => - duplicate(tweetId, view.returnDuplicateTweet(req)) - - // it's possible that tweetCreationLock didn't find a duplicate for a - // retweet attempt, but `RetweetBuilder` did. - case TweetCreateFailure.AlreadyRetweeted(tweetId) if view.returnDuplicateTweet(req) => - duplicate(tweetId, true) - } - } - } - - private def findDuplicate[R: RequestView]( - tweetId: TweetId, - req: R - ): Future[Option[PostTweetResult]] = { - val view = implicitly[RequestView[R]] - val readRequest = - GetTweetsRequest( - tweetIds = Seq(tweetId), - // Assume that the defaults are OK for all of the hydration - // options except the ones that are explicitly set in the - // req. - options = Some( - GetTweetOptions( - forUserId = Some(view.userId(req)), - includePerspectivals = true, - includeCards = view.options(req).exists(_.includeCards), - cardsPlatformKey = view.options(req).flatMap(_.cardsPlatformKey) - ) - ) - ) - - getTweets(readRequest).map { - case Seq(result) => - if (result.tweetState == StatusState.Found) { - // If the tweet was successfully found, then convert the - // read result into a successful write result. - Some( - PostTweetResult( - TweetCreateState.Ok, - result.tweet, - // if the retweet is really old, the retweet perspective might no longer - // be available, but we want to maintain the invariant that the `postRetweet` - // endpoint always returns a source tweet with the correct perspective. - result.sourceTweet.map { srcTweet => - TweetLenses.perspective - .update(_.map(_.copy(retweeted = true, retweetId = Some(tweetId)))) - .apply(srcTweet) - }, - result.quotedTweet - ) - ) - } else { - None - } - } - } - } - } - - /** - * A `Filter` that applies rate limiting to failing requests. - */ - object RateLimitFailures { - def apply( - validateLimit: RateLimitChecker.Validate, - incrementSuccess: LimiterService.IncrementByOne, - incrementFailure: LimiterService.IncrementByOne - ): Filter[TweetBuilderResult] = - new Filter[TweetBuilderResult] { - def apply[R: RequestView](base: T[R]): T[R] = { - val view = implicitly[RequestView[R]] - - FutureArrow[R, TweetBuilderResult] { req => - val userId = view.userId(req) - val dark = view.isDark(req) - val contributorUserId: Option[UserId] = getContributor(userId).map(_.userId) - - validateLimit((userId, dark)) - .before { - base(req).onFailure { _ => - // We don't increment the failure rate limit if the failure - // was from the failure rate limit so that the user can't - // get in a loop where tweet creation is never attempted. We - // don't increment it if the creation is dark because there - // is no way to perform a dark tweet creation through the - // API, so it's most likey some kind of test traffic like - // tap-compare. - if (!dark) incrementFailure(userId, contributorUserId) - } - } - .onSuccess { resp => - // If we return a silent failure, then we want to - // increment the rate limit as if the tweet was fully - // created, because we want it to appear that way to the - // user whose creation silently failed. - if (resp.isSilentFail) incrementSuccess(userId, contributorUserId) - } - } - } - } - } - - /** - * A `Filter` for counting non-`TweetCreateFailure` failures. - */ - object CountFailures { - def apply[Res](stats: StatsReceiver, scopeSuffix: String = "_builder"): Filter[Res] = - new Filter[Res] { - def apply[R: RequestView](base: T[R]): T[R] = { - val view = implicitly[RequestView[R]] - val exceptionCounter = ExceptionCounter(stats.scope(view.scope + scopeSuffix)) - base.onFailure { - case (_, _: TweetCreateFailure) => - case (_, ex) => exceptionCounter(ex) - } - } - } - } - - /** - * A `Filter` for logging failures. - */ - object LogFailures extends Filter[PostTweetResult] { - private[this] val failedTweetCreationsLogger = Logger( - "com.twitter.tweetypie.FailedTweetCreations" - ) - - def apply[R: RequestView](base: T[R]): T[R] = - FutureArrow[R, PostTweetResult] { req => - base(req).onFailure { - case failure => failedTweetCreationsLogger.info(s"request: $req\nfailure: $failure") - } - } - } - - /** - * A `Filter` for converting a thrown `TweetCreateFailure` into a `PostTweetResult`. - */ - object RescueTweetCreateFailure extends Filter[PostTweetResult] { - def apply[R: RequestView](base: T[R]): T[R] = - FutureArrow[R, PostTweetResult] { req => - base(req).rescue { - case failure: TweetCreateFailure => Future.value(failure.toPostTweetResult) - } - } - } - - /** - * Builds a base handler for `PostTweetRequest` and `RetweetRequest`. The handler - * calls an underlying tweet builder, creates a `InsertTweet.Event`, hydrates - * that, passes it to `tweetStore`, and then converts it to a `PostTweetResult`. - */ - object Handler { - def apply[R: RequestView]( - tweetBuilder: FutureArrow[R, TweetBuilderResult], - hydrateInsertEvent: FutureArrow[InsertTweet.Event, InsertTweet.Event], - tweetStore: InsertTweet.Store, - ): Type[R] = { - FutureArrow { req => - for { - bldrRes <- tweetBuilder(req) - event <- hydrateInsertEvent(toInsertTweetEvent(req, bldrRes)) - _ <- Future.when(!event.dark)(tweetStore.insertTweet(event)) - } yield toPostTweetResult(event) - } - } - - /** - * Converts a request/`TweetBuilderResult` pair into an `InsertTweet.Event`. - */ - def toInsertTweetEvent[R: RequestView]( - req: R, - bldrRes: TweetBuilderResult - ): InsertTweet.Event = { - val view = implicitly[RequestView[R]] - InsertTweet.Event( - tweet = bldrRes.tweet, - user = bldrRes.user, - sourceTweet = bldrRes.sourceTweet, - sourceUser = bldrRes.sourceUser, - parentUserId = bldrRes.parentUserId, - timestamp = bldrRes.createdAt, - dark = view.isDark(req) || bldrRes.isSilentFail, - hydrateOptions = view.options(req).getOrElse(WritePathHydrationOptions()), - featureContext = view.featureContext(req), - initialTweetUpdateRequest = bldrRes.initialTweetUpdateRequest, - geoSearchRequestId = for { - geo <- view.geo(req) - searchRequestID <- geo.geoSearchRequestId - } yield { - GeoSearchRequestId(requestID = searchRequestID.id) - }, - additionalContext = view.additionalContext(req), - transientContext = view.transientContext(req), - noteTweetMentionedUserIds = view.noteTweetMentionedUserIds(req) - ) - } - - /** - * Converts an `InsertTweet.Event` into a successful `PostTweetResult`. - */ - def toPostTweetResult(event: InsertTweet.Event): PostTweetResult = - PostTweetResult( - TweetCreateState.Ok, - Some(event.tweet), - sourceTweet = event.sourceTweet, - quotedTweet = event.quotedTweet - ) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetDeleteEventBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetDeleteEventBuilder.docx new file mode 100644 index 000000000..d9a2bcbc8 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetDeleteEventBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetDeleteEventBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetDeleteEventBuilder.scala deleted file mode 100644 index 834cda148..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetDeleteEventBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.repository.TweetRepository -import com.twitter.tweetypie.store.QuotedTweetDelete -import com.twitter.tweetypie.thriftscala.QuotedTweetDeleteRequest - -/** - * Create the appropriate QuotedTweetDelete.Event for a QuotedTweetDelete request. - */ -object QuotedTweetDeleteEventBuilder { - type Type = QuotedTweetDeleteRequest => Future[Option[QuotedTweetDelete.Event]] - - val queryOptions: TweetQuery.Options = - TweetQuery.Options(GetTweetsHandler.BaseInclude) - - def apply(tweetRepo: TweetRepository.Optional): Type = - request => - Stitch.run( - tweetRepo(request.quotingTweetId, queryOptions).map { - _.map { quotingTweet => - QuotedTweetDelete.Event( - quotingTweetId = request.quotingTweetId, - quotingUserId = getUserId(quotingTweet), - quotedTweetId = request.quotedTweetId, - quotedUserId = request.quotedUserId, - timestamp = Time.now - ) - } - } - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetTakedownEventBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetTakedownEventBuilder.docx new file mode 100644 index 000000000..9dacf9cda Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetTakedownEventBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetTakedownEventBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetTakedownEventBuilder.scala deleted file mode 100644 index 7a44845a8..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/QuotedTweetTakedownEventBuilder.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.repository.TweetRepository -import com.twitter.tweetypie.store.QuotedTweetTakedown -import com.twitter.tweetypie.thriftscala.QuotedTweetTakedownRequest - -/** - * Create the appropriate QuotedTweetTakedown.Event for a QuotedTweetTakedown request. - */ -object QuotedTweetTakedownEventBuilder { - type Type = QuotedTweetTakedownRequest => Future[Option[QuotedTweetTakedown.Event]] - - val queryOptions: TweetQuery.Options = - TweetQuery.Options(GetTweetsHandler.BaseInclude) - - def apply(tweetRepo: TweetRepository.Optional): Type = - request => - Stitch.run( - tweetRepo(request.quotingTweetId, queryOptions).map { - _.map { quotingTweet => - QuotedTweetTakedown.Event( - quotingTweetId = request.quotingTweetId, - quotingUserId = getUserId(quotingTweet), - quotedTweetId = request.quotedTweetId, - quotedUserId = request.quotedUserId, - takedownCountryCodes = request.takedownCountryCodes, - takedownReasons = request.takedownReasons, - timestamp = Time.now - ) - } - } - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RateLimitChecker.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RateLimitChecker.docx new file mode 100644 index 000000000..81c0ffc87 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RateLimitChecker.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RateLimitChecker.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RateLimitChecker.scala deleted file mode 100644 index cac90aab6..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RateLimitChecker.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.servo.util.FutureArrow -import com.twitter.tweetypie.backends.LimiterService -import com.twitter.tweetypie.core.TweetCreateFailure -import com.twitter.tweetypie.thriftscala.TweetCreateState.RateLimitExceeded - -object RateLimitChecker { - type Dark = Boolean - type GetRemaining = FutureArrow[(UserId, Dark), Int] - type Validate = FutureArrow[(UserId, Dark), Unit] - - def getMaxMediaTags(minRemaining: LimiterService.MinRemaining, maxMediaTags: Int): GetRemaining = - FutureArrow { - case (userId, dark) => - if (dark) Future.value(maxMediaTags) - else { - val contributorUserId = getContributor(userId).map(_.userId) - minRemaining(userId, contributorUserId) - .map(_.min(maxMediaTags)) - .handle { case _ => maxMediaTags } - } - } - - def validate( - hasRemaining: LimiterService.HasRemaining, - featureStats: StatsReceiver, - rateLimitEnabled: () => Boolean - ): Validate = { - val exceededCounter = featureStats.counter("exceeded") - val checkedCounter = featureStats.counter("checked") - FutureArrow { - case (userId, dark) => - if (dark || !rateLimitEnabled()) { - Future.Unit - } else { - checkedCounter.incr() - val contributorUserId = getContributor(userId).map(_.userId) - hasRemaining(userId, contributorUserId).map { - case false => - exceededCounter.incr() - throw TweetCreateFailure.State(RateLimitExceeded) - case _ => () - } - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReplyBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReplyBuilder.docx new file mode 100644 index 000000000..2fbed47c2 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReplyBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReplyBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReplyBuilder.scala deleted file mode 100644 index 2e1963074..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReplyBuilder.scala +++ /dev/null @@ -1,633 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.TweetCreateFailure -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.serverutil.ExceptionCounter -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.tweettext.Offset -import com.twitter.twittertext.Extractor -import scala.annotation.tailrec -import scala.collection.JavaConverters._ -import scala.collection.mutable -import scala.util.control.NoStackTrace - -object ReplyBuilder { - private val extractor = new Extractor - private val InReplyToTweetNotFound = - TweetCreateFailure.State(TweetCreateState.InReplyToTweetNotFound) - - case class Request( - authorId: UserId, - authorScreenName: String, - inReplyToTweetId: Option[TweetId], - tweetText: String, - prependImplicitMentions: Boolean, - enableTweetToNarrowcasting: Boolean, - excludeUserIds: Seq[UserId], - spamResult: Spam.Result, - batchMode: Option[BatchComposeMode]) - - /** - * This case class contains the fields that are shared between legacy and simplified replies. - */ - case class BaseResult( - reply: Reply, - conversationId: Option[ConversationId], - selfThreadMetadata: Option[SelfThreadMetadata], - community: Option[Communities] = None, - exclusiveTweetControl: Option[ExclusiveTweetControl] = None, - trustedFriendsControl: Option[TrustedFriendsControl] = None, - editControl: Option[EditControl] = None) { - // Creates a Result by providing the fields that differ between legacy and simplified replies. - def toResult( - tweetText: String, - directedAtMetadata: DirectedAtUserMetadata, - visibleStart: Offset.CodePoint = Offset.CodePoint(0), - ): Result = - Result( - reply, - tweetText, - directedAtMetadata, - conversationId, - selfThreadMetadata, - visibleStart, - community, - exclusiveTweetControl, - trustedFriendsControl, - editControl - ) - } - - /** - * @param reply the Reply object to include in the tweet. - * @param tweetText updated tweet text which may include prepended at-mentions, trimmed - * @param directedAtMetadata see DirectedAtHydrator for usage. - * @param conversationId conversation id to assign to the tweet. - * @param selfThreadMetadata returns the result of `SelfThreadBuilder` - * @param visibleStart offset into `tweetText` separating hideable at-mentions from the - * visible text. - */ - case class Result( - reply: Reply, - tweetText: String, - directedAtMetadata: DirectedAtUserMetadata, - conversationId: Option[ConversationId] = None, - selfThreadMetadata: Option[SelfThreadMetadata] = None, - visibleStart: Offset.CodePoint = Offset.CodePoint(0), - community: Option[Communities] = None, - exclusiveTweetControl: Option[ExclusiveTweetControl] = None, - trustedFriendsControl: Option[TrustedFriendsControl] = None, - editControl: Option[EditControl] = None) { - - /** - * @param finalText final tweet text after any server-side additions. - * @return true iff the final tweet text consists exclusively of a hidden reply mention prefix. - * When this happens there's no content to the reply and thus the tweet creation should - * fail. - */ - def replyTextIsEmpty(finalText: String): Boolean = { - - // Length of the tweet text originally output via ReplyBuilder.Result before server-side - // additions (e.g. media, quoted-tweet URLs) - val origTextLength = Offset.CodePoint.length(tweetText) - - // Length of the tweet text after server-side additions. - val finalTextLength = Offset.CodePoint.length(finalText) - - val prefixWasEntireText = origTextLength == visibleStart - val textLenUnchanged = origTextLength == finalTextLength - - prefixWasEntireText && textLenUnchanged - } - } - - type Type = Request => Future[Option[Result]] - - private object InvalidUserException extends NoStackTrace - - /** - * A user ID and screen name used for building replies. - */ - private case class User(id: UserId, screenName: String) - - /** - * Captures the in-reply-to tweet, its author, and if the user is attempting to reply to a - * retweet, then that retweet and its author. - */ - private case class ReplySource( - srcTweet: Tweet, - srcUser: User, - retweet: Option[Tweet] = None, - rtUser: Option[User] = None) { - private val photoTaggedUsers: Seq[User] = - srcTweet.mediaTags - .map(_.tagMap.values.flatten) - .getOrElse(Nil) - .map(toUser) - .toSeq - - private def toUser(mt: MediaTag): User = - mt match { - case MediaTag(_, Some(id), Some(screenName), _) => User(id, screenName) - case _ => throw InvalidUserException - } - - private def toUser(e: MentionEntity): User = - e match { - case MentionEntity(_, _, screenName, Some(id), _, _) => User(id, screenName) - case _ => throw InvalidUserException - } - - private def toUser(d: DirectedAtUser) = User(d.userId, d.screenName) - - def allCardUsers(authorUser: User, cardUsersFinder: CardUsersFinder.Type): Future[Set[UserId]] = - Stitch.run( - cardUsersFinder( - CardUsersFinder.Request( - cardReference = getCardReference(srcTweet), - urls = getUrls(srcTweet).map(_.url), - perspectiveUserId = authorUser.id - ) - ) - ) - - def srcTweetMentionedUsers: Seq[User] = getMentions(srcTweet).map(toUser) - - private trait ReplyType { - - val allExcludedUserIds: Set[UserId] - - def directedAt: Option[User] - def requiredTextMention: Option[User] - - def isExcluded(u: User): Boolean = allExcludedUserIds.contains(u.id) - - def buildPrefix(otherMentions: Seq[User], maxImplicits: Int): String = { - val seen = new mutable.HashSet[UserId] - seen ++= allExcludedUserIds - // Never exclude the required mention - seen --= requiredTextMention.map(_.id) - - (requiredTextMention.toSeq ++ otherMentions) - .filter(u => seen.add(u.id)) - .take(maxImplicits.max(requiredTextMention.size)) - .map(u => s"@${u.screenName}") - .mkString(" ") - } - } - - private case class SelfReply( - allExcludedUserIds: Set[UserId], - enableTweetToNarrowcasting: Boolean) - extends ReplyType { - - private def srcTweetDirectedAt: Option[User] = getDirectedAtUser(srcTweet).map(toUser) - - override def directedAt: Option[User] = - if (!enableTweetToNarrowcasting) None - else Seq.concat(rtUser, srcTweetDirectedAt).find(!isExcluded(_)) - - override def requiredTextMention: Option[User] = - // Make sure the directedAt user is in the text to avoid confusion - directedAt - } - - private case class BatchSubsequentReply(allExcludedUserIds: Set[UserId]) extends ReplyType { - - override def directedAt: Option[User] = None - - override def requiredTextMention: Option[User] = None - - override def buildPrefix(otherMentions: Seq[User], maxImplicits: Int): String = "" - } - - private case class RegularReply( - allExcludedUserIds: Set[UserId], - enableTweetToNarrowcasting: Boolean) - extends ReplyType { - - override def directedAt: Option[User] = - Some(srcUser) - .filterNot(isExcluded) - .filter(_ => enableTweetToNarrowcasting) - - override def requiredTextMention: Option[User] = - // Include the source tweet's author as a mention in the reply, even if the reply is not - // narrowcasted to that user. All non-self-reply tweets require this mention. - Some(srcUser) - } - - /** - * Computes an implicit mention prefix to add to the tweet text as well as any directed-at user. - * - * The first implicit mention is the source-tweet's author unless the reply is a self-reply, in - * which case it inherits the DirectedAtUser from the source tweet, though the current author is - * never added. This mention, if it exists, is the only mention that may be used to direct-at a - * user and is the user that ends up in DirectedAtUserMetadata. If the user replied to a - * retweet and the reply doesn't explicitly mention the retweet author, then the retweet author - * will be next, followed by source tweet mentions and source tweet photo-tagged users. - * - * Users in excludedScreenNames originate from the PostTweetRequest and are filtered out of any - * non-leading mention. - * - * Note on maxImplicits: - * This method returns at most 'maxImplicits' mentions unless 'maxImplicits' is 0 and a - * directed-at mention is required, in which case it returns 1. If this happens the reply may - * fail downstream validation checks (e.g. TweetBuilder). With 280 visible character limit it's - * theoretically possible to explicitly mention 93 users (280 / 3) but this bug shouldn't really - * be an issue because: - * 1.) Most replies don't have 50 explicit mentions - * 2.) TOO-clients have switched to batchMode=Subsequent for self-replies which disable - source tweet's directed-at user inheritance - * 3.) Requests rarely are rejected due to mention_limit_exceeded - * If this becomes a problem we could reopen the mention limit discussion, specifically if the - * backend should allow 51 while the explicit limit remains at 50. - * - * Note on batchMode: - * Implicit mention prefix will be empty string if batchMode is BatchSubsequent. This is to - * support batch composer. - */ - def implicitMentionPrefixAndDAU( - maxImplicits: Int, - excludedUsers: Seq[User], - author: User, - enableTweetToNarrowcasting: Boolean, - batchMode: Option[BatchComposeMode] - ): (String, Option[User]) = { - def allExcludedUserIds = - (excludedUsers ++ Seq(author)).map(_.id).toSet - - val replyType = - if (author.id == srcUser.id) { - if (batchMode.contains(BatchComposeMode.BatchSubsequent)) { - BatchSubsequentReply(allExcludedUserIds) - } else { - SelfReply(allExcludedUserIds, enableTweetToNarrowcasting) - } - } else { - RegularReply(allExcludedUserIds, enableTweetToNarrowcasting) - } - - val prefix = - replyType.buildPrefix( - otherMentions = List.concat(rtUser, srcTweetMentionedUsers, photoTaggedUsers), - maxImplicits = maxImplicits - ) - - (prefix, replyType.directedAt) - } - - /** - * Finds the longest possible prefix of whitespace separated @-mentions, restricted to - * @-mentions that are derived from the reply chain. - */ - def hideablePrefix( - text: String, - cardUsers: Seq[User], - explicitMentions: Seq[Extractor.Entity] - ): Offset.CodePoint = { - val allowedMentions = - (srcTweetMentionedUsers.toSet + srcUser ++ rtUser.toSet ++ photoTaggedUsers ++ cardUsers) - .map(_.screenName.toLowerCase) - val len = Offset.CodeUnit.length(text) - - // To allow NO-BREAK SPACE' (U+00A0) in the prefix need .isSpaceChar - def isWhitespace(c: Char) = c.isWhitespace || c.isSpaceChar - - @tailrec - def skipWs(offset: Offset.CodeUnit): Offset.CodeUnit = - if (offset == len || !isWhitespace(text.charAt(offset.toInt))) offset - else skipWs(offset.incr) - - @tailrec - def go(offset: Offset.CodeUnit, mentions: Stream[Extractor.Entity]): Offset.CodeUnit = - if (offset == len) offset - else { - mentions match { - // if we are at the next mention, and it is allowed, skip past and recurse - case next #:: tail if next.getStart == offset.toInt => - if (!allowedMentions.contains(next.getValue.toLowerCase)) offset - else go(skipWs(Offset.CodeUnit(next.getEnd)), tail) - // we found non-mention text - case _ => offset - } - } - - go(Offset.CodeUnit(0), explicitMentions.toStream).toCodePoint(text) - } - } - - private def replyToUser(user: User, inReplyToStatusId: Option[TweetId] = None): Reply = - Reply( - inReplyToUserId = user.id, - inReplyToScreenName = Some(user.screenName), - inReplyToStatusId = inReplyToStatusId - ) - - /** - * A builder that generates reply from `inReplyToTweetId` or tweet text - * - * There are two kinds of "reply": - * 1. reply to tweet, which is generated from `inReplyToTweetId`. - * - * A valid reply-to-tweet satisfies the following conditions: - * 1). the tweet that is in-reply-to exists (and is visible to the user creating the tweet) - * 2). the author of the in-reply-to tweet is mentioned anywhere in the tweet, or - * this is a tweet that is in reply to the author's own tweet - * - * 2. reply to user, is generated when the tweet text starts with @user_name. This is only - * attempted if PostTweetRequest.enableTweetToNarrowcasting is true (default). - */ - def apply( - userIdentityRepo: UserIdentityRepository.Type, - tweetRepo: TweetRepository.Optional, - replyCardUsersFinder: CardUsersFinder.Type, - selfThreadBuilder: SelfThreadBuilder, - relationshipRepo: RelationshipRepository.Type, - unmentionedEntitiesRepo: UnmentionedEntitiesRepository.Type, - enableRemoveUnmentionedImplicits: Gate[Unit], - stats: StatsReceiver, - maxMentions: Int - ): Type = { - val exceptionCounters = ExceptionCounter(stats) - val modeScope = stats.scope("mode") - val compatModeCounter = modeScope.counter("compat") - val simpleModeCounter = modeScope.counter("simple") - - def getUser(key: UserKey): Future[Option[User]] = - Stitch.run( - userIdentityRepo(key) - .map(ident => User(ident.id, ident.screenName)) - .liftNotFoundToOption - ) - - def getUsers(userIds: Seq[UserId]): Future[Seq[ReplyBuilder.User]] = - Stitch.run( - Stitch - .traverse(userIds)(id => userIdentityRepo(UserKey(id)).liftNotFoundToOption) - .map(_.flatten) - .map { identities => identities.map { ident => User(ident.id, ident.screenName) } } - ) - - val tweetQueryIncludes = - TweetQuery.Include( - tweetFields = Set( - Tweet.CoreDataField.id, - Tweet.CardReferenceField.id, - Tweet.CommunitiesField.id, - Tweet.MediaTagsField.id, - Tweet.MentionsField.id, - Tweet.UrlsField.id, - Tweet.EditControlField.id - ) ++ selfThreadBuilder.requiredReplySourceFields.map(_.id) - ) - - def tweetQueryOptions(forUserId: UserId) = - TweetQuery.Options( - tweetQueryIncludes, - forUserId = Some(forUserId), - enforceVisibilityFiltering = true - ) - - def getTweet(tweetId: TweetId, forUserId: UserId): Future[Option[Tweet]] = - Stitch.run(tweetRepo(tweetId, tweetQueryOptions(forUserId))) - - def checkBlockRelationship(authorId: UserId, result: Result): Future[Unit] = { - val inReplyToBlocksTweeter = - RelationshipKey.blocks( - sourceId = result.reply.inReplyToUserId, - destinationId = authorId - ) - - Stitch.run(relationshipRepo(inReplyToBlocksTweeter)).flatMap { - case true => Future.exception(InReplyToTweetNotFound) - case false => Future.Unit - } - } - - def checkIPIPolicy(request: Request, reply: Reply): Future[Unit] = { - if (request.spamResult == Spam.DisabledByIpiPolicy) { - Future.exception(Spam.DisabledByIpiFailure(reply.inReplyToScreenName)) - } else { - Future.Unit - } - } - - def getUnmentionedUsers(replySource: ReplySource): Future[Seq[UserId]] = { - if (enableRemoveUnmentionedImplicits()) { - val srcDirectedAt = replySource.srcTweet.directedAtUserMetadata.flatMap(_.userId) - val srcTweetMentions = replySource.srcTweet.mentions.getOrElse(Nil).flatMap(_.userId) - val idsToCheck = srcTweetMentions ++ srcDirectedAt - - val conversationId = replySource.srcTweet.coreData.flatMap(_.conversationId) - conversationId match { - case Some(cid) if idsToCheck.nonEmpty => - stats.counter("unmentioned_implicits_check").incr() - Stitch - .run(unmentionedEntitiesRepo(cid, idsToCheck)).liftToTry.map { - case Return(Some(unmentionedUserIds)) => - unmentionedUserIds - case _ => Seq[UserId]() - } - case _ => Future.Nil - - } - } else { - Future.Nil - } - } - - /** - * Constructs a `ReplySource` for the given `tweetId`, which captures the source tweet to be - * replied to, its author, and if `tweetId` is for a retweet of the source tweet, then also - * that retweet and its author. If the source tweet (or a retweet of it), or a corresponding - * author, can't be found or isn't visible to the replier, then `InReplyToTweetNotFound` is - * thrown. - */ - def getReplySource(tweetId: TweetId, forUserId: UserId): Future[ReplySource] = - for { - tweet <- getTweet(tweetId, forUserId).flatMap { - case None => Future.exception(InReplyToTweetNotFound) - case Some(t) => Future.value(t) - } - - user <- getUser(UserKey(getUserId(tweet))).flatMap { - case None => Future.exception(InReplyToTweetNotFound) - case Some(u) => Future.value(u) - } - - res <- getShare(tweet) match { - case None => Future.value(ReplySource(tweet, user)) - case Some(share) => - // if the user is replying to a retweet, find the retweet source tweet, - // then update with the retweet and author. - getReplySource(share.sourceStatusId, forUserId) - .map(_.copy(retweet = Some(tweet), rtUser = Some(user))) - } - } yield res - - /** - * Computes a `Result` for the reply-to-tweet case. If `inReplyToTweetId` is for a retweet, - * the reply will be computed against the source tweet. If `prependImplicitMentions` is true - * and source tweet can't be found or isn't visible to replier, then this method will return - * a `InReplyToTweetNotFound` failure. If `prependImplicitMentions` is false, then the reply - * text must either mention the source tweet user, or it must be a reply to self; if both of - * those conditions fail, then `None` is returned. - */ - def makeReplyToTweet( - inReplyToTweetId: TweetId, - text: String, - author: User, - prependImplicitMentions: Boolean, - enableTweetToNarrowcasting: Boolean, - excludeUserIds: Seq[UserId], - batchMode: Option[BatchComposeMode] - ): Future[Option[Result]] = { - val explicitMentions: Seq[Extractor.Entity] = - extractor.extractMentionedScreennamesWithIndices(text).asScala.toSeq - val mentionedScreenNames = - explicitMentions.map(_.getValue.toLowerCase).toSet - - /** - * If `prependImplicitMentions` is true, or the reply author is the same as the in-reply-to - * author, then the reply text doesn't have to mention the in-reply-to author. Otherwise, - * check that the text contains a mention of the reply author. - */ - def isValidReplyTo(inReplyToUser: User): Boolean = - prependImplicitMentions || - (inReplyToUser.id == author.id) || - mentionedScreenNames.contains(inReplyToUser.screenName.toLowerCase) - - getReplySource(inReplyToTweetId, author.id) - .flatMap { replySrc => - val baseResult = BaseResult( - reply = replyToUser(replySrc.srcUser, Some(replySrc.srcTweet.id)), - conversationId = getConversationId(replySrc.srcTweet), - selfThreadMetadata = selfThreadBuilder.build(author.id, replySrc.srcTweet), - community = replySrc.srcTweet.communities, - // Reply tweets retain the same exclusive - // tweet controls as the tweet being replied to. - exclusiveTweetControl = replySrc.srcTweet.exclusiveTweetControl, - trustedFriendsControl = replySrc.srcTweet.trustedFriendsControl, - editControl = replySrc.srcTweet.editControl - ) - - if (isValidReplyTo(replySrc.srcUser)) { - if (prependImplicitMentions) { - - // Simplified Replies mode - append server-side generated prefix to passed in text - simpleModeCounter.incr() - // remove the in-reply-to tweet author from the excluded users, in-reply-to tweet author will always be a directedAtUser - val filteredExcludedIds = - excludeUserIds.filterNot(uid => uid == TweetLenses.userId(replySrc.srcTweet)) - for { - unmentionedUserIds <- getUnmentionedUsers(replySrc) - excludedUsers <- getUsers(filteredExcludedIds ++ unmentionedUserIds) - (prefix, directedAtUser) = replySrc.implicitMentionPrefixAndDAU( - maxImplicits = math.max(0, maxMentions - explicitMentions.size), - excludedUsers = excludedUsers, - author = author, - enableTweetToNarrowcasting = enableTweetToNarrowcasting, - batchMode = batchMode - ) - } yield { - // prefix or text (or both) can be empty strings. Add " " separator and adjust - // prefix length only when both prefix and text are non-empty. - val textChunks = Seq(prefix, text).map(_.trim).filter(_.nonEmpty) - val tweetText = textChunks.mkString(" ") - val visibleStart = - if (textChunks.size == 2) { - Offset.CodePoint.length(prefix + " ") - } else { - Offset.CodePoint.length(prefix) - } - - Some( - baseResult.toResult( - tweetText = tweetText, - directedAtMetadata = DirectedAtUserMetadata(directedAtUser.map(_.id)), - visibleStart = visibleStart - ) - ) - } - } else { - // Backwards-compatibility mode - walk from beginning of text until find visibleStart - compatModeCounter.incr() - for { - cardUserIds <- replySrc.allCardUsers(author, replyCardUsersFinder) - cardUsers <- getUsers(cardUserIds.toSeq) - optUserIdentity <- extractReplyToUser(text) - directedAtUserId = optUserIdentity.map(_.id).filter(_ => enableTweetToNarrowcasting) - } yield { - Some( - baseResult.toResult( - tweetText = text, - directedAtMetadata = DirectedAtUserMetadata(directedAtUserId), - visibleStart = replySrc.hideablePrefix(text, cardUsers, explicitMentions), - ) - ) - } - } - } else { - Future.None - } - } - .handle { - // if `getReplySource` throws this exception, but we aren't computing implicit - // mentions, then we fall back to the reply-to-user case instead of reply-to-tweet - case InReplyToTweetNotFound if !prependImplicitMentions => None - } - } - - def makeReplyToUser(text: String): Future[Option[Result]] = - extractReplyToUser(text).map(_.map { user => - Result(replyToUser(user), text, DirectedAtUserMetadata(Some(user.id))) - }) - - def extractReplyToUser(text: String): Future[Option[User]] = - Option(extractor.extractReplyScreenname(text)) match { - case None => Future.None - case Some(screenName) => getUser(UserKey(screenName)) - } - - FutureArrow[Request, Option[Result]] { request => - exceptionCounters { - (request.inReplyToTweetId.filter(_ > 0) match { - case None => - Future.None - - case Some(tweetId) => - makeReplyToTweet( - tweetId, - request.tweetText, - User(request.authorId, request.authorScreenName), - request.prependImplicitMentions, - request.enableTweetToNarrowcasting, - request.excludeUserIds, - request.batchMode - ) - }).flatMap { - case Some(r) => - // Ensure that the author of this reply is not blocked by - // the user who they are replying to. - checkBlockRelationship(request.authorId, r) - .before(checkIPIPolicy(request, r.reply)) - .before(Future.value(Some(r))) - - case None if request.enableTweetToNarrowcasting => - // We don't check the block relationship when the tweet is - // not part of a conversation (which is to say, we allow - // directed-at tweets from a blocked user.) These tweets - // will not cause notifications for the blocking user, - // despite the presence of the reply struct. - makeReplyToUser(request.tweetText) - - case None => - Future.None - } - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RetweetBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RetweetBuilder.docx new file mode 100644 index 000000000..d839517c7 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RetweetBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RetweetBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RetweetBuilder.scala deleted file mode 100644 index e14eecc84..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/RetweetBuilder.scala +++ /dev/null @@ -1,352 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.flockdb.client._ -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.additionalfields.AdditionalFields.setAdditionalFields -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.thriftscala.entities.EntityExtractor -import com.twitter.tweetypie.tweettext.Truncator -import com.twitter.tweetypie.util.CommunityUtil -import com.twitter.tweetypie.util.EditControlUtil - -case class SourceTweetRequest( - tweetId: TweetId, - user: User, - hydrateOptions: WritePathHydrationOptions) - -object RetweetBuilder { - import TweetBuilder._ - import UpstreamFailure._ - - type Type = FutureArrow[RetweetRequest, TweetBuilderResult] - - val SGSTestRole = "socialgraph" - - val log: Logger = Logger(getClass) - - /** - * Retweets text gets RT and username prepended - */ - def composeRetweetText(text: String, sourceUser: User): String = - composeRetweetText(text, sourceUser.profile.get.screenName) - - /** - * Retweets text gets RT and username prepended - */ - def composeRetweetText(text: String, screenName: String): String = - Truncator.truncateForRetweet("RT @" + screenName + ": " + text) - - // We do not want to allow community tweets to be retweeted. - def validateNotCommunityTweet(sourceTweet: Tweet): Future[Unit] = - if (CommunityUtil.hasCommunity(sourceTweet.communities)) { - Future.exception(TweetCreateFailure.State(TweetCreateState.CommunityRetweetNotAllowed)) - } else { - Future.Unit - } - - // We do not want to allow Trusted Friends tweets to be retweeted. - def validateNotTrustedFriendsTweet(sourceTweet: Tweet): Future[Unit] = - sourceTweet.trustedFriendsControl match { - case Some(trustedFriendsControl) => - Future.exception(TweetCreateFailure.State(TweetCreateState.TrustedFriendsRetweetNotAllowed)) - case None => - Future.Unit - } - - // We do not want to allow retweet of a stale version of a tweet in an edit chain. - def validateStaleTweet(sourceTweet: Tweet): Future[Unit] = { - if (!EditControlUtil.isLatestEdit(sourceTweet.editControl, sourceTweet.id).getOrElse(true)) { - Future.exception(TweetCreateFailure.State(TweetCreateState.StaleTweetRetweetNotAllowed)) - } else { - // the source tweet does not have any edit control or the source tweet is the latest tweet - Future.Unit - } - } - - /** - * Builds the RetweetBuilder - */ - def apply( - validateRequest: RetweetRequest => Future[Unit], - tweetIdGenerator: TweetIdGenerator, - tweetRepo: TweetRepository.Type, - userRepo: UserRepository.Type, - tflock: TFlockClient, - deviceSourceRepo: DeviceSourceRepository.Type, - validateUpdateRateLimit: RateLimitChecker.Validate, - spamChecker: Spam.Checker[RetweetSpamRequest] = Spam.DoNotCheckSpam, - updateUserCounts: (User, Tweet) => Future[User], - superFollowRelationsRepo: StratoSuperFollowRelationsRepository.Type, - unretweetEdits: TweetDeletePathHandler.UnretweetEdits, - setEditWindowToSixtyMinutes: Gate[Unit] - ): RetweetBuilder.Type = { - val entityExtactor = EntityExtractor.mutationAll.endo - - val sourceTweetRepo: SourceTweetRequest => Stitch[Tweet] = - req => { - tweetRepo( - req.tweetId, - WritePathQueryOptions.retweetSourceTweet(req.user, req.hydrateOptions) - ).rescue { - case _: FilteredState => Stitch.NotFound - } - .rescue { - convertRepoExceptions(TweetCreateState.SourceTweetNotFound, TweetLookupFailure(_)) - } - } - - val getUser = userLookup(userRepo) - val getSourceUser = sourceUserLookup(userRepo) - val getDeviceSource = deviceSourceLookup(deviceSourceRepo) - - /** - * We exempt SGS test users from the check to get them through Block v2 testing. - */ - def isSGSTestRole(user: User): Boolean = - user.roles.exists { roles => roles.roles.contains(SGSTestRole) } - - def validateCanRetweet( - user: User, - sourceUser: User, - sourceTweet: Tweet, - request: RetweetRequest - ): Future[Unit] = - Future - .join( - validateNotCommunityTweet(sourceTweet), - validateNotTrustedFriendsTweet(sourceTweet), - validateSourceUserRetweetable(user, sourceUser), - validateStaleTweet(sourceTweet), - Future.when(!request.dark) { - if (request.returnSuccessOnDuplicate) - failWithRetweetIdIfAlreadyRetweeted(user, sourceTweet) - else - validateNotAlreadyRetweeted(user, sourceTweet) - } - ) - .unit - - def validateSourceUserRetweetable(user: User, sourceUser: User): Future[Unit] = - if (sourceUser.profile.isEmpty) - Future.exception(UserProfileEmptyException) - else if (sourceUser.safety.isEmpty) - Future.exception(UserSafetyEmptyException) - else if (sourceUser.view.isEmpty) - Future.exception(UserViewEmptyException) - else if (user.id != sourceUser.id && sourceUser.safety.get.isProtected) - Future.exception(TweetCreateFailure.State(TweetCreateState.CannotRetweetProtectedTweet)) - else if (sourceUser.safety.get.deactivated) - Future.exception(TweetCreateFailure.State(TweetCreateState.CannotRetweetDeactivatedUser)) - else if (sourceUser.safety.get.suspended) - Future.exception(TweetCreateFailure.State(TweetCreateState.CannotRetweetSuspendedUser)) - else if (sourceUser.view.get.blockedBy && !isSGSTestRole(user)) - Future.exception(TweetCreateFailure.State(TweetCreateState.CannotRetweetBlockingUser)) - else if (sourceUser.profile.get.screenName.isEmpty) - Future.exception( - TweetCreateFailure.State(TweetCreateState.CannotRetweetUserWithoutScreenName) - ) - else - Future.Unit - - def tflockGraphContains( - graph: StatusGraph, - fromId: Long, - toId: Long, - dir: Direction - ): Future[Boolean] = - tflock.contains(graph, fromId, toId, dir).rescue { - case ex: OverCapacity => Future.exception(ex) - case ex => Future.exception(TFlockLookupFailure(ex)) - } - - def getRetweetIdFromTflock(sourceTweetId: TweetId, userId: UserId): Future[Option[Long]] = - tflock - .selectAll( - Select( - sourceId = sourceTweetId, - graph = RetweetsGraph, - direction = Forward - ).intersect( - Select( - sourceId = userId, - graph = UserTimelineGraph, - direction = Forward - ) - ) - ) - .map(_.headOption) - - def validateNotAlreadyRetweeted(user: User, sourceTweet: Tweet): Future[Unit] = - // use the perspective object from TLS if available, otherwise, check with tflock - (sourceTweet.perspective match { - case Some(perspective) => - Future.value(perspective.retweeted) - case None => - // we have to query the RetweetSourceGraph in the Reverse order because - // it is only defined in that direction, instead of bi-directionally - tflockGraphContains(RetweetSourceGraph, user.id, sourceTweet.id, Reverse) - }).flatMap { - case true => - Future.exception(TweetCreateFailure.State(TweetCreateState.AlreadyRetweeted)) - case false => Future.Unit - } - - def failWithRetweetIdIfAlreadyRetweeted(user: User, sourceTweet: Tweet): Future[Unit] = - // use the perspective object from TLS if available, otherwise, check with tflock - (sourceTweet.perspective.flatMap(_.retweetId) match { - case Some(tweetId) => Future.value(Some(tweetId)) - case None => - getRetweetIdFromTflock(sourceTweet.id, user.id) - }).flatMap { - case None => Future.Unit - case Some(tweetId) => - Future.exception(TweetCreateFailure.AlreadyRetweeted(tweetId)) - } - - def validateContributor(contributorIdOpt: Option[UserId]): Future[Unit] = - if (contributorIdOpt.isDefined) - Future.exception(TweetCreateFailure.State(TweetCreateState.ContributorNotSupported)) - else - Future.Unit - - case class RetweetSource(sourceTweet: Tweet, parentUserId: UserId) - - /** - * Recursively follows a retweet chain to the root source tweet. Also returns user id from the - * first walked tweet as the 'parentUserId'. - * In practice, the depth of the chain should never be greater than 2 because - * share.sourceStatusId should always reference the root (unlike share.parentStatusId). - */ - def findRetweetSource( - tweetId: TweetId, - forUser: User, - hydrateOptions: WritePathHydrationOptions - ): Future[RetweetSource] = - Stitch - .run(sourceTweetRepo(SourceTweetRequest(tweetId, forUser, hydrateOptions))) - .flatMap { tweet => - getShare(tweet) match { - case None => Future.value(RetweetSource(tweet, getUserId(tweet))) - case Some(share) => - findRetweetSource(share.sourceStatusId, forUser, hydrateOptions) - .map(_.copy(parentUserId = getUserId(tweet))) - } - } - - FutureArrow { request => - for { - () <- validateRequest(request) - userFuture = Stitch.run(getUser(request.userId)) - tweetIdFuture = tweetIdGenerator() - devsrcFuture = Stitch.run(getDeviceSource(request.createdVia)) - user <- userFuture - tweetId <- tweetIdFuture - devsrc <- devsrcFuture - rtSource <- findRetweetSource( - request.sourceStatusId, - user, - request.hydrationOptions.getOrElse(WritePathHydrationOptions(simpleQuotedTweet = true)) - ) - sourceTweet = rtSource.sourceTweet - sourceUser <- Stitch.run(getSourceUser(getUserId(sourceTweet), request.userId)) - - // We want to confirm that a user is actually allowed to - // retweet an Exclusive Tweet (only available to super followers) - () <- StratoSuperFollowRelationsRepository.Validate( - sourceTweet.exclusiveTweetControl, - user.id, - superFollowRelationsRepo) - - () <- validateUser(user) - () <- validateUpdateRateLimit((user.id, request.dark)) - () <- validateContributor(request.contributorUserId) - () <- validateCanRetweet(user, sourceUser, sourceTweet, request) - () <- unretweetEdits(sourceTweet.editControl, sourceTweet.id, user.id) - - spamRequest = RetweetSpamRequest( - retweetId = tweetId, - sourceUserId = getUserId(sourceTweet), - sourceTweetId = sourceTweet.id, - sourceTweetText = getText(sourceTweet), - sourceUserName = sourceUser.profile.map(_.screenName), - safetyMetaData = request.safetyMetaData - ) - - spamResult <- spamChecker(spamRequest) - - safety = user.safety.get - - share = Share( - sourceStatusId = sourceTweet.id, - sourceUserId = sourceUser.id, - parentStatusId = request.sourceStatusId - ) - - retweetText = composeRetweetText(getText(sourceTweet), sourceUser) - createdAt = SnowflakeId(tweetId).time - - coreData = TweetCoreData( - userId = request.userId, - text = retweetText, - createdAtSecs = createdAt.inSeconds, - createdVia = devsrc.internalName, - share = Some(share), - hasTakedown = safety.hasTakedown, - trackingId = request.trackingId, - nsfwUser = safety.nsfwUser, - nsfwAdmin = safety.nsfwAdmin, - narrowcast = request.narrowcast, - nullcast = request.nullcast - ) - - retweet = Tweet( - id = tweetId, - coreData = Some(coreData), - contributor = getContributor(request.userId), - editControl = Some( - EditControl.Initial( - EditControlUtil - .makeEditControlInitial( - tweetId = tweetId, - createdAt = createdAt, - setEditWindowToSixtyMinutes = setEditWindowToSixtyMinutes - ) - .initial - .copy(isEditEligible = Some(false)) - ) - ), - ) - - retweetWithEntities = entityExtactor(retweet) - retweetWithAdditionalFields = setAdditionalFields( - retweetWithEntities, - request.additionalFields - ) - // update the perspective and counts fields of the source tweet to reflect the effects - // of the user performing a retweet, even though those effects haven't happened yet. - updatedSourceTweet = sourceTweet.copy( - perspective = sourceTweet.perspective.map { - _.copy(retweeted = true, retweetId = Some(retweet.id)) - }, - counts = sourceTweet.counts.map { c => c.copy(retweetCount = c.retweetCount.map(_ + 1)) } - ) - - user <- updateUserCounts(user, retweetWithAdditionalFields) - } yield { - TweetBuilderResult( - tweet = retweetWithAdditionalFields, - user = user, - createdAt = createdAt, - sourceTweet = Some(updatedSourceTweet), - sourceUser = Some(sourceUser), - parentUserId = Some(rtSource.parentUserId), - isSilentFail = spamResult == Spam.SilentFail - ) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReverseGeocoder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReverseGeocoder.docx new file mode 100644 index 000000000..80f8a474d Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReverseGeocoder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReverseGeocoder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReverseGeocoder.scala deleted file mode 100644 index 8a675a8ce..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ReverseGeocoder.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.geoduck.backend.hydration.thriftscala.HydrationContext -import com.twitter.geoduck.common.thriftscala.Constants -import com.twitter.geoduck.common.thriftscala.PlaceQuery -import com.twitter.geoduck.common.thriftscala.PlaceQueryFields -import com.twitter.geoduck.service.common.clientmodules.GeoduckGeohashLocate -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.geoduck.util.primitives.LatLon -import com.twitter.geoduck.util.primitives.{Geohash => GDGeohash} -import com.twitter.geoduck.util.primitives.{Place => GDPlace} -import com.twitter.servo.util.FutureArrow -import com.twitter.tweetypie.repository.GeoduckPlaceConverter -import com.twitter.tweetypie.{thriftscala => TP} - -object ReverseGeocoder { - val log: Logger = Logger(getClass) - - private def validatingRGC(rgc: ReverseGeocoder): ReverseGeocoder = - FutureArrow { - case (coords: TP.GeoCoordinates, language: PlaceLanguage) => - if (LatLon.isValid(coords.latitude, coords.longitude)) - rgc((coords, language)) - else - Future.None - } - - /** - * create a Geo backed ReverseGeocoder - */ - def fromGeoduck(geohashLocate: GeoduckGeohashLocate): ReverseGeocoder = - validatingRGC( - FutureArrow { - case (geo: TP.GeoCoordinates, language: PlaceLanguage) => - if (log.isDebugEnabled) { - log.debug("RGC'ing " + geo.toString() + " with geoduck") - } - - val hydrationContext = - HydrationContext( - placeFields = Set[PlaceQueryFields]( - PlaceQueryFields.PlaceNames - ) - ) - - val gh = GDGeohash(LatLon(lat = geo.latitude, lon = geo.longitude)) - val placeQuery = PlaceQuery(placeTypes = Some(Constants.ConsumerPlaceTypes)) - - geohashLocate - .locateGeohashes(Seq(gh.toThrift), placeQuery, hydrationContext) - .onFailure { case ex => log.warn("failed to rgc " + geo.toString(), ex) } - .map { - (resp: Seq[Try[LocationResponse]]) => - resp.headOption.flatMap { - case Throw(ex) => - log.warn("rgc failed for coords: " + geo.toString(), ex) - None - case Return(locationResponse) => - GDPlace.tryLocationResponse(locationResponse) match { - case Throw(ex) => - log - .warn("rgc failed in response handling for coords: " + geo.toString(), ex) - None - case Return(tplaces) => - GDPlace.pickConsumerLocation(tplaces).map { place: GDPlace => - if (log.isDebugEnabled) { - log.debug("successfully rgc'd " + geo + " to " + place.id) - } - GeoduckPlaceConverter(language, place) - } - } - - } - } - } - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowRetweetSpamChecker.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowRetweetSpamChecker.docx new file mode 100644 index 000000000..d16b111ca Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowRetweetSpamChecker.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowRetweetSpamChecker.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowRetweetSpamChecker.scala deleted file mode 100644 index 3c7a78fd9..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowRetweetSpamChecker.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.finagle.tracing.Trace -import com.twitter.service.gen.scarecrow.thriftscala.Retweet -import com.twitter.service.gen.scarecrow.thriftscala.TieredAction -import com.twitter.service.gen.scarecrow.thriftscala.TieredActionResult -import com.twitter.spam.features.thriftscala.SafetyMetaData -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.TweetCreateFailure -import com.twitter.tweetypie.repository.RetweetSpamCheckRepository -import com.twitter.tweetypie.thriftscala.TweetCreateState - -case class RetweetSpamRequest( - retweetId: TweetId, - sourceUserId: UserId, - sourceTweetId: TweetId, - sourceTweetText: String, - sourceUserName: Option[String], - safetyMetaData: Option[SafetyMetaData]) - -/** - * Use the Scarecrow service as the spam checker for retweets. - */ -object ScarecrowRetweetSpamChecker { - val log: Logger = Logger(getClass) - - def requestToScarecrowRetweet(req: RetweetSpamRequest): Retweet = - Retweet( - id = req.retweetId, - sourceUserId = req.sourceUserId, - text = req.sourceTweetText, - sourceTweetId = req.sourceTweetId, - safetyMetaData = req.safetyMetaData - ) - - def apply( - stats: StatsReceiver, - repo: RetweetSpamCheckRepository.Type - ): Spam.Checker[RetweetSpamRequest] = { - - def handler(request: RetweetSpamRequest): Spam.Checker[TieredAction] = - Spam.handleScarecrowResult(stats) { - case (TieredActionResult.NotSpam, _, _) => Spam.AllowFuture - case (TieredActionResult.SilentFail, _, _) => Spam.SilentFailFuture - case (TieredActionResult.UrlSpam, _, denyMessage) => - Future.exception(TweetCreateFailure.State(TweetCreateState.UrlSpam, denyMessage)) - case (TieredActionResult.Deny, _, denyMessage) => - Future.exception(TweetCreateFailure.State(TweetCreateState.Spam, denyMessage)) - case (TieredActionResult.DenyByIpiPolicy, _, denyMessage) => - Future.exception(Spam.DisabledByIpiFailure(request.sourceUserName, denyMessage)) - case (TieredActionResult.RateLimit, _, denyMessage) => - Future.exception( - TweetCreateFailure.State(TweetCreateState.SafetyRateLimitExceeded, denyMessage)) - case (TieredActionResult.Bounce, Some(b), _) => - Future.exception(TweetCreateFailure.Bounced(b)) - } - - req => { - Trace.record("com.twitter.tweetypie.ScarecrowRetweetSpamChecker.retweetId=" + req.retweetId) - Stitch.run(repo(requestToScarecrowRetweet(req))).flatMap(handler(req)) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowTweetSpamChecker.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowTweetSpamChecker.docx new file mode 100644 index 000000000..f5d9e067e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowTweetSpamChecker.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowTweetSpamChecker.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowTweetSpamChecker.scala deleted file mode 100644 index 5db66c4dc..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScarecrowTweetSpamChecker.scala +++ /dev/null @@ -1,106 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.finagle.tracing.Trace -import com.twitter.relevance.feature_store.thriftscala.FeatureData -import com.twitter.relevance.feature_store.thriftscala.FeatureValue -import com.twitter.service.gen.scarecrow.thriftscala.TieredAction -import com.twitter.service.gen.scarecrow.thriftscala.TieredActionResult -import com.twitter.service.gen.scarecrow.thriftscala.TweetContext -import com.twitter.service.gen.scarecrow.thriftscala.TweetNew -import com.twitter.spam.features.thriftscala.SafetyMetaData -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.TweetCreateFailure -import com.twitter.tweetypie.handler.Spam.Checker -import com.twitter.tweetypie.repository.TweetSpamCheckRepository -import com.twitter.tweetypie.thriftscala.TweetCreateState -import com.twitter.tweetypie.thriftscala.TweetMediaTags - -case class TweetSpamRequest( - tweetId: TweetId, - userId: UserId, - text: String, - mediaTags: Option[TweetMediaTags], - safetyMetaData: Option[SafetyMetaData], - inReplyToTweetId: Option[TweetId], - quotedTweetId: Option[TweetId], - quotedTweetUserId: Option[UserId]) - -/** - * Use the Scarecrow service as the spam checker for tweets. - */ -object ScarecrowTweetSpamChecker { - val log: Logger = Logger(getClass) - - private def requestToScarecrowTweet(req: TweetSpamRequest): TweetNew = { - // compile additional input features for the spam check - val mediaTaggedUserIds = { - val mediaTags = req.mediaTags.getOrElse(TweetMediaTags()) - mediaTags.tagMap.values.flatten.flatMap(_.userId).toSet - } - - val additionalInputFeatures = { - val mediaTaggedUserFeatures = if (mediaTaggedUserIds.nonEmpty) { - Seq( - "mediaTaggedUsers" -> FeatureData(Some(FeatureValue.LongSetValue(mediaTaggedUserIds))), - "victimIds" -> FeatureData(Some(FeatureValue.LongSetValue(mediaTaggedUserIds))) - ) - } else { - Seq.empty - } - - val quotedTweetIdFeature = req.quotedTweetId.map { quotedTweetId => - "quotedTweetId" -> FeatureData(Some(FeatureValue.LongValue(quotedTweetId))) - } - - val quotedTweetUserIdFeature = req.quotedTweetUserId.map { quotedTweetUserId => - "quotedTweetUserId" -> FeatureData(Some(FeatureValue.LongValue(quotedTweetUserId))) - } - - val featureMap = - (mediaTaggedUserFeatures ++ quotedTweetIdFeature ++ quotedTweetUserIdFeature).toMap - - if (featureMap.nonEmpty) Some(featureMap) else None - } - - TweetNew( - id = req.tweetId, - userId = req.userId, - text = req.text, - additionalInputFeatures = additionalInputFeatures, - safetyMetaData = req.safetyMetaData, - inReplyToStatusId = req.inReplyToTweetId - ) - } - - private def tieredActionHandler(stats: StatsReceiver): Checker[TieredAction] = - Spam.handleScarecrowResult(stats) { - case (TieredActionResult.NotSpam, _, _) => Spam.AllowFuture - case (TieredActionResult.SilentFail, _, _) => Spam.SilentFailFuture - case (TieredActionResult.DenyByIpiPolicy, _, _) => Spam.DisabledByIpiPolicyFuture - case (TieredActionResult.UrlSpam, _, denyMessage) => - Future.exception(TweetCreateFailure.State(TweetCreateState.UrlSpam, denyMessage)) - case (TieredActionResult.Deny, _, denyMessage) => - Future.exception(TweetCreateFailure.State(TweetCreateState.Spam, denyMessage)) - case (TieredActionResult.Captcha, _, denyMessage) => - Future.exception(TweetCreateFailure.State(TweetCreateState.SpamCaptcha, denyMessage)) - case (TieredActionResult.RateLimit, _, denyMessage) => - Future.exception( - TweetCreateFailure.State(TweetCreateState.SafetyRateLimitExceeded, denyMessage)) - case (TieredActionResult.Bounce, Some(b), _) => - Future.exception(TweetCreateFailure.Bounced(b)) - } - - def fromSpamCheckRepository( - stats: StatsReceiver, - repo: TweetSpamCheckRepository.Type - ): Spam.Checker[TweetSpamRequest] = { - val handler = tieredActionHandler(stats) - req => { - Trace.record("com.twitter.tweetypie.ScarecrowTweetSpamChecker.userId=" + req.userId) - Stitch.run(repo(requestToScarecrowTweet(req), TweetContext.Creation)).flatMap { resp => - handler(resp.tieredAction) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScrubGeoEventBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScrubGeoEventBuilder.docx new file mode 100644 index 000000000..6868c4321 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScrubGeoEventBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScrubGeoEventBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScrubGeoEventBuilder.scala deleted file mode 100644 index 77c3b2bb3..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/ScrubGeoEventBuilder.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.store.ScrubGeo -import com.twitter.tweetypie.store.ScrubGeoUpdateUserTimestamp -import com.twitter.tweetypie.thriftscala.DeleteLocationData -import com.twitter.tweetypie.thriftscala.GeoScrub - -/** - * Create the appropriate ScrubGeo.Event for a GeoScrub request. - */ -object ScrubGeoEventBuilder { - val userQueryOptions: UserQueryOptions = - UserQueryOptions( - Set(UserField.Safety, UserField.Roles), - UserVisibility.All - ) - - private def userLoader( - stats: StatsReceiver, - userRepo: UserRepository.Optional - ): UserId => Future[Option[User]] = { - val userNotFoundCounter = stats.counter("user_not_found") - userId => - Stitch.run( - userRepo(UserKey(userId), userQueryOptions) - .onSuccess(userOpt => if (userOpt.isEmpty) userNotFoundCounter.incr()) - ) - } - - object UpdateUserTimestamp { - type Type = DeleteLocationData => Future[ScrubGeoUpdateUserTimestamp.Event] - - def apply( - stats: StatsReceiver, - userRepo: UserRepository.Optional, - ): Type = { - val timestampDiffStat = stats.stat("now_delta_ms") - val loadUser = userLoader(stats, userRepo) - request: DeleteLocationData => - loadUser(request.userId).map { userOpt => - // delta between users requesting deletion and the time we publish to TweetEvents - timestampDiffStat.add((Time.now.inMillis - request.timestampMs).toFloat) - ScrubGeoUpdateUserTimestamp.Event( - userId = request.userId, - timestamp = Time.fromMilliseconds(request.timestampMs), - optUser = userOpt - ) - } - } - } - - object ScrubTweets { - type Type = GeoScrub => Future[ScrubGeo.Event] - - def apply(stats: StatsReceiver, userRepo: UserRepository.Optional): Type = { - val loadUser = userLoader(stats, userRepo) - geoScrub => - loadUser(geoScrub.userId).map { userOpt => - ScrubGeo.Event( - tweetIdSet = geoScrub.statusIds.toSet, - userId = geoScrub.userId, - enqueueMax = geoScrub.hosebirdEnqueue, - optUser = userOpt, - timestamp = Time.now - ) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SelfThreadBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SelfThreadBuilder.docx new file mode 100644 index 000000000..478477c7b Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SelfThreadBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SelfThreadBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SelfThreadBuilder.scala deleted file mode 100644 index adc2c5739..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SelfThreadBuilder.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.tweetypie.thriftscala.Reply -import com.twitter.tweetypie.thriftscala.SelfThreadMetadata -import org.apache.thrift.protocol.TField - -trait SelfThreadBuilder { - def requiredReplySourceFields: Set[TField] = - Set( - Tweet.CoreDataField, // for Reply and ConversationId - Tweet.SelfThreadMetadataField // for continuing existing self-threads - ) - - def build(authorUserId: UserId, replySourceTweet: Tweet): Option[SelfThreadMetadata] -} - -/** - * SelfThreadBuilder is used to build metadata for self-threads (tweetstorms). - * - * This builder is invoked from ReplyBuilder on tweets that pass in a inReplyToStatusId and create - * a Reply. The invocation is done inside ReplyBuilder as ReplyBuilder has already loaded the - * "reply source tweet" which has all the information needed to determine the self-thread metadata. - * - * Note that Tweet.SelfThreadMetadata schema supports representing two types of self-threads: - * 1. root self-thread : self-thread that begins alone and does not start with replying to another - * tweet. This self-thread has a self-thread ID equal to the conversation ID. - * 2. reply self-thread : self-thread that begins as a reply to another user's tweet. - * This self-thread has a self-thread ID equal to the first tweet in the - * current self-reply chain which will not equal the conversation ID. - * - * Currently only type #1 "root self-thread" is handled. - */ -object SelfThreadBuilder { - - def apply(stats: StatsReceiver): SelfThreadBuilder = { - // We want to keep open the possibility for differentiation between root - // self-threads (current functionality) and reply self-threads (possible - // future functionality). - val rootThreadStats = stats.scope("root_thread") - - // A tweet becomes a root of a self-thread only after the first self-reply - // is created. root_thread/start is incr()d during the write-path of the - // self-reply tweet, when it is known that the first/root tweet has not - // yet been assigned a SelfThreadMetadata. The write-path of the second - // tweet does not add the SelfThreadMetadata to the first tweet - that - // happens asynchronously by the SelfThreadDaemon. - val rootThreadStartCounter = rootThreadStats.counter("start") - - // root_thread/continue provides visibility into the frequency of - // continuation tweets off leaf tweets in a tweet storm. Also incr()d in - // the special case of a reply to the root tweet, which does not yet have a - // SelfThreadMetadata(isLeaf=true). - val rootThreadContinueCounter = rootThreadStats.counter("continue") - - // root_thread/branch provides visibility into how frequently self-threads - // get branched - that is, when the author self-replies to a non-leaf tweet - // in an existing thread. Knowing the frequency of branching will help us - // determine the priority of accounting for branching in various - // tweet-delete use cases. Currently we do not fix up the root tweet's - // SelfThreadMetadata when its reply tweets are deleted. - val rootThreadBranchCounter = rootThreadStats.counter("branch") - - def observeSelfThreadMetrics(replySourceSTM: Option[SelfThreadMetadata]): Unit = { - replySourceSTM match { - case Some(SelfThreadMetadata(_, isLeaf)) => - if (isLeaf) rootThreadContinueCounter.incr() - else rootThreadBranchCounter.incr() - case None => - rootThreadStartCounter.incr() - } - } - - new SelfThreadBuilder { - - override def build( - authorUserId: UserId, - replySourceTweet: Tweet - ): Option[SelfThreadMetadata] = { - // the "reply source tweet"'s author must match the current author - if (getUserId(replySourceTweet) == authorUserId) { - val replySourceSTM = getSelfThreadMetadata(replySourceTweet) - - observeSelfThreadMetrics(replySourceSTM) - - // determine if replySourceTweet stands alone (non-reply) - getReply(replySourceTweet) match { - case None | Some(Reply(None, _, _)) => - // 'replySourceTweet' started a new self-thread that stands alone - // which happens when there's no Reply or the Reply does not have - // inReplyToStatusId (directed-at user) - - // requiredReplySourceFields requires coreData and conversationId - // is required so this would have previously thrown an exception - // in ReplyBuilder if the read was partial - val convoId = replySourceTweet.coreData.get.conversationId.get - Some(SelfThreadMetadata(id = convoId, isLeaf = true)) - - case _ => - // 'replySourceTweet' was also a reply-to-tweet, so continue any - // self-thread by inheriting any SelfThreadMetadata it has - // (though always setting isLeaf to true) - replySourceSTM.map(_.copy(isLeaf = true)) - } - } else { - // Replying to a different user currently never creates a self-thread - // as all self-threads must start at the root (and match conversation - // ID). - // - // In the future replying to a different user *might* be part of a - // self-thread but we wouldn't mark it as such until the *next* tweet - // is created (at which time the self_thread daemon goes back and - // marks the first tweet as in the self-thread. - None - } - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetAdditionalFieldsBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetAdditionalFieldsBuilder.docx new file mode 100644 index 000000000..d7ad09c63 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetAdditionalFieldsBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetAdditionalFieldsBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetAdditionalFieldsBuilder.scala deleted file mode 100644 index 423543d8f..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetAdditionalFieldsBuilder.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.repository.TweetRepository -import com.twitter.tweetypie.repository.UserKey -import com.twitter.tweetypie.repository.UserQueryOptions -import com.twitter.tweetypie.repository.UserRepository -import com.twitter.tweetypie.repository.UserVisibility -import com.twitter.tweetypie.store.AsyncSetAdditionalFields -import com.twitter.tweetypie.store.SetAdditionalFields -import com.twitter.tweetypie.store.TweetStoreEventOrRetry -import com.twitter.tweetypie.thriftscala.AsyncSetAdditionalFieldsRequest -import com.twitter.tweetypie.thriftscala.SetAdditionalFieldsRequest - -object SetAdditionalFieldsBuilder { - type Type = SetAdditionalFieldsRequest => Future[SetAdditionalFields.Event] - - val tweetOptions: TweetQuery.Options = TweetQuery.Options(include = GetTweetsHandler.BaseInclude) - - def apply(tweetRepo: TweetRepository.Type): Type = { - def getTweet(tweetId: TweetId) = - Stitch.run( - tweetRepo(tweetId, tweetOptions) - .rescue(HandlerError.translateNotFoundToClientError(tweetId)) - ) - - request => { - getTweet(request.additionalFields.id).map { tweet => - SetAdditionalFields.Event( - additionalFields = request.additionalFields, - userId = getUserId(tweet), - timestamp = Time.now - ) - } - } - } -} - -object AsyncSetAdditionalFieldsBuilder { - type Type = AsyncSetAdditionalFieldsRequest => Future[ - TweetStoreEventOrRetry[AsyncSetAdditionalFields.Event] - ] - - val userQueryOpts: UserQueryOptions = UserQueryOptions(Set(UserField.Safety), UserVisibility.All) - - def apply(userRepo: UserRepository.Type): Type = { - def getUser(userId: UserId): Future[User] = - Stitch.run( - userRepo(UserKey.byId(userId), userQueryOpts) - .rescue { case NotFound => Stitch.exception(HandlerError.userNotFound(userId)) } - ) - - request => - getUser(request.userId).map { user => - AsyncSetAdditionalFields.Event.fromAsyncRequest(request, user) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetRetweetVisibilityHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetRetweetVisibilityHandler.docx new file mode 100644 index 000000000..01ce4e48f Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetRetweetVisibilityHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetRetweetVisibilityHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetRetweetVisibilityHandler.scala deleted file mode 100644 index 48dc91014..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/SetRetweetVisibilityHandler.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.tweetypie.store.SetRetweetVisibility -import com.twitter.tweetypie.thriftscala.SetRetweetVisibilityRequest -import com.twitter.tweetypie.thriftscala.Share -import com.twitter.tweetypie.thriftscala.Tweet - -/** - * Create a [[SetRetweetVisibility.Event]] from a [[SetRetweetVisibilityRequest]] and then - * pipe the event to [[store.SetRetweetVisibility]]. The event contains the information - * to determine if a retweet should be included in its source tweet's retweet count. - * - * Showing/hiding a retweet count is done by calling TFlock to modify an edge's state between - * `Positive` <--> `Archived` in the RetweetsGraph(6) and modifying the count in cache directly. - */ -object SetRetweetVisibilityHandler { - type Type = SetRetweetVisibilityRequest => Future[Unit] - - def apply( - tweetGetter: TweetId => Future[Option[Tweet]], - setRetweetVisibilityStore: SetRetweetVisibility.Event => Future[Unit] - ): Type = - req => - tweetGetter(req.retweetId).map { - case Some(retweet) => - getShare(retweet).map { share: Share => - val event = SetRetweetVisibility.Event( - retweetId = req.retweetId, - visible = req.visible, - srcId = share.sourceStatusId, - retweetUserId = getUserId(retweet), - srcTweetUserId = share.sourceUserId, - timestamp = Time.now - ) - setRetweetVisibilityStore(event) - } - - case None => - // No-op if either the retweet has been deleted or has no source id. - // If deleted, then we do not want to accidentally undelete a legitimately deleted retweets. - // If no source id, then we do not know the source tweet to modify its count. - Unit - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/Spam.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/Spam.docx new file mode 100644 index 000000000..7974d6e1a Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/Spam.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/Spam.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/Spam.scala deleted file mode 100644 index 088f9b8a9..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/Spam.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.botmaker.thriftscala.BotMakerResponse -import com.twitter.bouncer.thriftscala.Bounce -import com.twitter.finagle.tracing.Trace -import com.twitter.relevance.feature_store.thriftscala.FeatureData -import com.twitter.relevance.feature_store.thriftscala.FeatureValue.StrValue -import com.twitter.service.gen.scarecrow.thriftscala.TieredAction -import com.twitter.service.gen.scarecrow.thriftscala.TieredActionResult -import com.twitter.tweetypie.core.TweetCreateFailure -import com.twitter.tweetypie.thriftscala.TweetCreateState - -object Spam { - sealed trait Result - case object Allow extends Result - case object SilentFail extends Result - case object DisabledByIpiPolicy extends Result - - val AllowFuture: Future[Allow.type] = Future.value(Allow) - val SilentFailFuture: Future[SilentFail.type] = Future.value(SilentFail) - val DisabledByIpiPolicyFuture: Future[DisabledByIpiPolicy.type] = - Future.value(DisabledByIpiPolicy) - - def DisabledByIpiFailure( - userName: Option[String], - customDenyMessage: Option[String] = None - ): TweetCreateFailure.State = { - val errorMsg = (customDenyMessage, userName) match { - case (Some(denyMessage), _) => denyMessage - case (_, Some(name)) => s"Some actions on this ${name} Tweet have been disabled by Twitter." - case _ => "Some actions on this Tweet have been disabled by Twitter." - } - TweetCreateFailure.State(TweetCreateState.DisabledByIpiPolicy, Some(errorMsg)) - } - - type Checker[T] = T => Future[Result] - - /** - * Dummy spam checker that always allows requests. - */ - val DoNotCheckSpam: Checker[AnyRef] = _ => AllowFuture - - def gated[T](gate: Gate[Unit])(checker: Checker[T]): Checker[T] = - req => if (gate()) checker(req) else AllowFuture - - def selected[T](gate: Gate[Unit])(ifTrue: Checker[T], ifFalse: Checker[T]): Checker[T] = - req => gate.select(ifTrue, ifFalse)()(req) - - def withEffect[T](check: Checker[T], effect: T => Unit): T => Future[Result] = { t: T => - effect(t) - check(t) - } - - /** - * Wrapper that implicitly allows retweet or tweet creation when spam - * checking fails. - */ - def allowOnException[T](checker: Checker[T]): Checker[T] = - req => - checker(req).rescue { - case e: TweetCreateFailure => Future.exception(e) - case _ => AllowFuture - } - - /** - * Handler for scarecrow result to be used by a Checker. - */ - def handleScarecrowResult( - stats: StatsReceiver - )( - handler: PartialFunction[(TieredActionResult, Option[Bounce], Option[String]), Future[Result]] - ): Checker[TieredAction] = - result => { - stats.scope("scarecrow_result").counter(result.resultCode.name).incr() - Trace.record("com.twitter.tweetypie.Spam.scarecrow_result=" + result.resultCode.name) - /* - * A bot can return a custom DenyMessage - * - * If it does, we substitute this for the 'message' in the ValidationError. - */ - val customDenyMessage: Option[String] = for { - botMakeResponse: BotMakerResponse <- result.botMakerResponse - outputFeatures <- botMakeResponse.outputFeatures - denyMessageFeature: FeatureData <- outputFeatures.get("DenyMessage") - denyMessageFeatureValue <- denyMessageFeature.featureValue - denyMessage <- denyMessageFeatureValue match { - case stringValue: StrValue => - Some(stringValue.strValue) - case _ => - None - } - } yield denyMessage - handler.applyOrElse( - (result.resultCode, result.bounce, customDenyMessage), - withEffect(DoNotCheckSpam, (_: AnyRef) => stats.counter("unexpected_result").incr()) - ) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TakedownHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TakedownHandler.docx new file mode 100644 index 000000000..c823deec6 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TakedownHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TakedownHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TakedownHandler.scala deleted file mode 100644 index e729e3cce..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TakedownHandler.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.servo.util.FutureArrow -import com.twitter.takedown.util.TakedownReasons._ -import com.twitter.tweetypie.store.Takedown -import com.twitter.tweetypie.thriftscala.TakedownRequest -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.tweetypie.util.Takedowns - -/** - * This handler processes TakedownRequest objects sent to Tweetypie's takedown endpoint. - * The request object specifies which takedown countries are being added and which are - * being removed. It also includes side effect flags for setting the tweet's has_takedown - * bit, scribing to Guano, and enqueuing to EventBus. For more information about inputs - * to the takedown endpoint, see the TakedownRequest documentation in the thrift definition. - */ -object TakedownHandler { - type Type = FutureArrow[TakedownRequest, Unit] - - def apply( - getTweet: FutureArrow[TweetId, Tweet], - getUser: FutureArrow[UserId, User], - writeTakedown: FutureEffect[Takedown.Event] - ): Type = { - FutureArrow { request => - for { - tweet <- getTweet(request.tweetId) - user <- getUser(getUserId(tweet)) - userHasTakedowns = user.takedowns.map(userTakedownsToReasons).exists(_.nonEmpty) - - existingTweetReasons = Takedowns.fromTweet(tweet).reasons - - reasonsToRemove = (request.countriesToRemove.map(countryCodeToReason) ++ - request.reasonsToRemove.map(normalizeReason)).distinct.sortBy(_.toString) - - reasonsToAdd = (request.countriesToAdd.map(countryCodeToReason) ++ - request.reasonsToAdd.map(normalizeReason)).distinct.sortBy(_.toString) - - updatedTweetTakedowns = - (existingTweetReasons ++ reasonsToAdd) - .filterNot(reasonsToRemove.contains) - .toSeq - .sortBy(_.toString) - - (cs, rs) = Takedowns.partitionReasons(updatedTweetTakedowns) - - updatedTweet = Lens.setAll( - tweet, - // these fields are cached on the Tweet in CachingTweetStore and written in - // ManhattanTweetStore - TweetLenses.hasTakedown -> (updatedTweetTakedowns.nonEmpty || userHasTakedowns), - TweetLenses.tweetypieOnlyTakedownCountryCodes -> Some(cs).filter(_.nonEmpty), - TweetLenses.tweetypieOnlyTakedownReasons -> Some(rs).filter(_.nonEmpty) - ) - - _ <- writeTakedown.when(tweet != updatedTweet) { - Takedown.Event( - tweet = updatedTweet, - timestamp = Time.now, - user = Some(user), - takedownReasons = updatedTweetTakedowns, - reasonsToAdd = reasonsToAdd, - reasonsToRemove = reasonsToRemove, - auditNote = request.auditNote, - host = request.host, - byUserId = request.byUserId, - eventbusEnqueue = request.eventbusEnqueue, - scribeForAudit = request.scribeForAudit, - updateCodesAndReasons = true - ) - } - } yield () - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetBuilder.docx new file mode 100644 index 000000000..b69111107 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetBuilder.scala deleted file mode 100644 index 98bb33064..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetBuilder.scala +++ /dev/null @@ -1,1180 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.featureswitches.v2.FeatureSwitchResults -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.gizmoduck.thriftscala.AccessPolicy -import com.twitter.gizmoduck.thriftscala.LabelValue -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.additionalfields.AdditionalFields._ -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.jiminy.tweetypie.NudgeBuilder -import com.twitter.tweetypie.jiminy.tweetypie.NudgeBuilderRequest -import com.twitter.tweetypie.media.Media -import com.twitter.tweetypie.repository.StratoCommunityAccessRepository.CommunityAccess -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.serverutil.DeviceSourceParser -import com.twitter.tweetypie.serverutil.ExtendedTweetMetadataBuilder -import com.twitter.tweetypie.store._ -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.thriftscala.entities.EntityExtractor -import com.twitter.tweetypie.tweettext._ -import com.twitter.tweetypie.util.CommunityAnnotation -import com.twitter.tweetypie.util.CommunityUtil -import com.twitter.twittertext.Regex.{VALID_URL => UrlPattern} -import com.twitter.twittertext.TwitterTextParser - -case class TweetBuilderResult( - tweet: Tweet, - user: User, - createdAt: Time, - sourceTweet: Option[Tweet] = None, - sourceUser: Option[User] = None, - parentUserId: Option[UserId] = None, - isSilentFail: Boolean = false, - geoSearchRequestId: Option[GeoSearchRequestId] = None, - initialTweetUpdateRequest: Option[InitialTweetUpdateRequest] = None) - -object TweetBuilder { - import GizmoduckUserCountsUpdatingStore.isUserTweet - import PostTweet._ - import Preprocessor._ - import TweetCreateState.{Spam => CreateStateSpam, _} - import TweetText._ - import UpstreamFailure._ - - type Type = FutureArrow[PostTweetRequest, TweetBuilderResult] - - val log: Logger = Logger(getClass) - - private[this] val _unitMutation = Future.value(Mutation.unit[Any]) - def MutationUnitFuture[T]: Future[Mutation[T]] = _unitMutation.asInstanceOf[Future[Mutation[T]]] - - case class MissingConversationId(inReplyToTweetId: TweetId) extends RuntimeException - - case class TextVisibility( - visibleTextRange: Option[TextRange], - totalTextDisplayLength: Offset.DisplayUnit, - visibleText: String) { - val isExtendedTweet: Boolean = totalTextDisplayLength.toInt > OriginalMaxDisplayLength - - /** - * Going forward we will be moving away from quoted-tweets urls in tweet text, but we - * have a backwards-compat layer in Tweetypie which adds the QT url to text to provide - * support for all clients to read in a backwards-compatible way until they upgrade. - * - * Tweets can become extended as their display length can go beyond 140 - * after adding the QT short url. Therefore, we are adding below function - * to account for legacy formatting during read-time and generate a self-permalink. - */ - def isExtendedWithExtraChars(extraChars: Int): Boolean = - totalTextDisplayLength.toInt > (OriginalMaxDisplayLength - extraChars) - } - - /** Max number of users that can be tagged on a single tweet */ - val MaxMediaTagCount = 10 - - val MobileWebApp = "oauth:49152" - val M2App = "oauth:3033294" - val M5App = "oauth:3033300" - - val TestRateLimitUserRole = "stresstest" - - /** - * The fields to fetch for the user creating the tweet. - */ - val userFields: Set[UserField] = - Set( - UserField.Profile, - UserField.ProfileDesign, - UserField.Account, - UserField.Safety, - UserField.Counts, - UserField.Roles, - UserField.UrlEntities, - UserField.Labels - ) - - /** - * The fields to fetch for the user of the source tweet in a retweet. - */ - val sourceUserFields: Set[UserField] = - userFields + UserField.View - - /** - * Converts repository exceptions into an API-compatible exception type - */ - def convertRepoExceptions[A]( - notFoundState: TweetCreateState, - failureHandler: Throwable => Throwable - ): PartialFunction[Throwable, Stitch[A]] = { - // stitch.NotFound is converted to the supplied TweetCreateState, wrapped in TweetCreateFailure - case NotFound => Stitch.exception(TweetCreateFailure.State(notFoundState)) - // OverCapacity exceptions should not be translated and should bubble up to the top - case ex: OverCapacity => Stitch.exception(ex) - // Other exceptions are wrapped in the supplied failureHandler - case ex => Stitch.exception(failureHandler(ex)) - } - - /** - * Adapts a UserRepository to a Repository for looking up a single user and that - * fails with an appropriate TweetCreateFailure if the user is not found. - */ - def userLookup(userRepo: UserRepository.Type): UserId => Stitch[User] = { - val opts = UserQueryOptions(queryFields = userFields, visibility = UserVisibility.All) - - userId => - userRepo(UserKey(userId), opts) - .rescue(convertRepoExceptions[User](UserNotFound, UserLookupFailure(_))) - } - - /** - * Adapts a UserRepository to a Repository for looking up a single user and that - * fails with an appropriate TweetCreateFailure if the user is not found. - */ - def sourceUserLookup(userRepo: UserRepository.Type): (UserId, UserId) => Stitch[User] = { - val opts = UserQueryOptions(queryFields = sourceUserFields, visibility = UserVisibility.All) - - (userId, forUserId) => - userRepo(UserKey(userId), opts.copy(forUserId = Some(forUserId))) - .rescue(convertRepoExceptions[User](SourceUserNotFound, UserLookupFailure(_))) - } - - /** - * Any fields that are loaded on the user via TweetBuilder/RetweetBuilder, but which should not - * be included on the user in the async-insert actions (such as hosebird) should be removed here. - * - * This will include perspectival fields that were loaded relative to the user creating the tweet. - */ - def scrubUserInAsyncInserts: User => User = - user => user.copy(view = None) - - /** - * Any fields that are loaded on the source user via TweetBuilder/RetweetBuilder, but which - * should not be included on the user in the async-insert actions (such as hosebird) should - * be removed here. - * - * This will include perspectival fields that were loaded relative to the user creating the tweet. - */ - def scrubSourceUserInAsyncInserts: User => User = - // currently the same as scrubUserInAsyncInserts, could be different in the future - scrubUserInAsyncInserts - - /** - * Any fields that are loaded on the source tweet via RetweetBuilder, but which should not be - * included on the source tweetypie in the async-insert actions (such as hosebird) should - * be removed here. - * - * This will include perspectival fields that were loaded relative to the user creating the tweet. - */ - def scrubSourceTweetInAsyncInserts: Tweet => Tweet = - tweet => tweet.copy(perspective = None, cards = None, card2 = None) - - /** - * Adapts a DeviceSource to a Repository for looking up a single device-source and that - * fails with an appropriate TweetCreateFailure if not found. - */ - def deviceSourceLookup(devSrcRepo: DeviceSourceRepository.Type): DeviceSourceRepository.Type = - appIdStr => { - val result: Stitch[DeviceSource] = - if (DeviceSourceParser.isValid(appIdStr)) { - devSrcRepo(appIdStr) - } else { - Stitch.exception(NotFound) - } - - result.rescue(convertRepoExceptions(DeviceSourceNotFound, DeviceSourceLookupFailure(_))) - } - - /** - * Checks: - * - that we have all the user fields we need - * - that the user is active - * - that they are not a frictionless follower account - */ - def validateUser(user: User): Future[Unit] = - if (user.safety.isEmpty) - Future.exception(UserSafetyEmptyException) - else if (user.profile.isEmpty) - Future.exception(UserProfileEmptyException) - else if (user.safety.get.deactivated) - Future.exception(TweetCreateFailure.State(UserDeactivated)) - else if (user.safety.get.suspended) - Future.exception(TweetCreateFailure.State(UserSuspended)) - else if (user.labels.exists(_.labels.exists(_.labelValue == LabelValue.ReadOnly))) - Future.exception(TweetCreateFailure.State(CreateStateSpam)) - else if (user.userType == UserType.Frictionless) - Future.exception(TweetCreateFailure.State(UserNotFound)) - else if (user.userType == UserType.Soft) - Future.exception(TweetCreateFailure.State(UserNotFound)) - else if (user.safety.get.accessPolicy == AccessPolicy.BounceAll || - user.safety.get.accessPolicy == AccessPolicy.BounceAllPublicWrites) - Future.exception(TweetCreateFailure.State(UserReadonly)) - else - Future.Unit - - def validateCommunityReply( - communities: Option[Communities], - replyResult: Option[ReplyBuilder.Result] - ): Future[Unit] = { - - if (replyResult.flatMap(_.reply.inReplyToStatusId).nonEmpty) { - val rootCommunities = replyResult.flatMap(_.community) - val rootCommunityIds = CommunityUtil.communityIds(rootCommunities) - val replyCommunityIds = CommunityUtil.communityIds(communities) - - if (rootCommunityIds == replyCommunityIds) { - Future.Unit - } else { - Future.exception(TweetCreateFailure.State(CommunityReplyTweetNotAllowed)) - } - } else { - Future.Unit - } - } - - // Project requirements do not allow exclusive tweets to be replies. - // All exclusive tweets must be root tweets. - def validateExclusiveTweetNotReplies( - exclusiveTweetControls: Option[ExclusiveTweetControl], - replyResult: Option[ReplyBuilder.Result] - ): Future[Unit] = { - val isInReplyToTweet = replyResult.exists(_.reply.inReplyToStatusId.isDefined) - if (exclusiveTweetControls.isDefined && isInReplyToTweet) { - Future.exception(TweetCreateFailure.State(SuperFollowsInvalidParams)) - } else { - Future.Unit - } - } - - // Invalid parameters for Exclusive Tweets: - // - Community field set # Tweets can not be both at the same time. - def validateExclusiveTweetParams( - exclusiveTweetControls: Option[ExclusiveTweetControl], - communities: Option[Communities] - ): Future[Unit] = { - if (exclusiveTweetControls.isDefined && CommunityUtil.hasCommunity(communities)) { - Future.exception(TweetCreateFailure.State(SuperFollowsInvalidParams)) - } else { - Future.Unit - } - } - - def validateTrustedFriendsNotReplies( - trustedFriendsControl: Option[TrustedFriendsControl], - replyResult: Option[ReplyBuilder.Result] - ): Future[Unit] = { - val isInReplyToTweet = replyResult.exists(_.reply.inReplyToStatusId.isDefined) - if (trustedFriendsControl.isDefined && isInReplyToTweet) { - Future.exception(TweetCreateFailure.State(TrustedFriendsInvalidParams)) - } else { - Future.Unit - } - } - - def validateTrustedFriendsParams( - trustedFriendsControl: Option[TrustedFriendsControl], - conversationControl: Option[TweetCreateConversationControl], - communities: Option[Communities], - exclusiveTweetControl: Option[ExclusiveTweetControl] - ): Future[Unit] = { - if (trustedFriendsControl.isDefined && - (conversationControl.isDefined || CommunityUtil.hasCommunity( - communities) || exclusiveTweetControl.isDefined)) { - Future.exception(TweetCreateFailure.State(TrustedFriendsInvalidParams)) - } else { - Future.Unit - } - } - - /** - * Checks the weighted tweet text length using twitter-text, as used by clients. - * This should ensure that any tweet the client deems valid will also be deemed - * valid by Tweetypie. - */ - def prevalidateTextLength(text: String, stats: StatsReceiver): Future[Unit] = { - val twitterTextConfig = TwitterTextParser.TWITTER_TEXT_DEFAULT_CONFIG - val twitterTextResult = TwitterTextParser.parseTweet(text, twitterTextConfig) - val textTooLong = !twitterTextResult.isValid && text.length > 0 - - Future.when(textTooLong) { - val weightedLength = twitterTextResult.weightedLength - log.debug( - s"Weighted length too long. weightedLength: $weightedLength" + - s", Tweet text: '${diffshow.show(text)}'" - ) - stats.counter("check_weighted_length/text_too_long").incr() - Future.exception(TweetCreateFailure.State(TextTooLong)) - } - } - - /** - * Checks that the tweet text is neither blank nor too long. - */ - def validateTextLength( - text: String, - visibleText: String, - replyResult: Option[ReplyBuilder.Result], - stats: StatsReceiver - ): Future[Unit] = { - val utf8Length = Offset.Utf8.length(text) - - def visibleTextTooLong = - Offset.DisplayUnit.length(visibleText) > Offset.DisplayUnit(MaxVisibleWeightedEmojiLength) - - def utf8LengthTooLong = - utf8Length > Offset.Utf8(MaxUtf8Length) - - if (isBlank(text)) { - stats.counter("validate_text_length/text_cannot_be_blank").incr() - Future.exception(TweetCreateFailure.State(TextCannotBeBlank)) - } else if (replyResult.exists(_.replyTextIsEmpty(text))) { - stats.counter("validate_text_length/reply_text_cannot_be_blank").incr() - Future.exception(TweetCreateFailure.State(TextCannotBeBlank)) - } else if (visibleTextTooLong) { - // Final check that visible text does not exceed MaxVisibleWeightedEmojiLength - // characters. - // prevalidateTextLength() does some portion of validation as well, most notably - // weighted length on raw, unescaped text. - stats.counter("validate_text_length/text_too_long.visible_length_explicit").incr() - log.debug( - s"Explicit MaxVisibleWeightedLength visible length check failed. " + - s"visibleText: '${diffshow.show(visibleText)}' and " + - s"total text: '${diffshow.show(text)}'" - ) - Future.exception(TweetCreateFailure.State(TextTooLong)) - } else if (utf8LengthTooLong) { - stats.counter("validate_text_length/text_too_long.utf8_length").incr() - Future.exception(TweetCreateFailure.State(TextTooLong)) - } else { - stats.stat("validate_text_length/utf8_length").add(utf8Length.toInt) - Future.Unit - } - } - - def getTextVisibility( - text: String, - replyResult: Option[ReplyBuilder.Result], - urlEntities: Seq[UrlEntity], - mediaEntities: Seq[MediaEntity], - attachmentUrl: Option[String] - ): TextVisibility = { - val totalTextLength = Offset.CodePoint.length(text) - val totalTextDisplayLength = Offset.DisplayUnit.length(text) - - /** - * visibleEnd for multiple scenarios: - * - * normal tweet + media - fromIndex of mediaEntity (hydrated from last media permalink) - * quote tweet + media - fromIndex of mediaEntity - * replies + media - fromIndex of mediaEntity - * normal quote tweet - total text length (visible text range will be None) - * tweets with other attachments (DM deep links) - * fromIndex of the last URL entity - */ - val visibleEnd = mediaEntities.headOption - .map(_.fromIndex) - .orElse(attachmentUrl.flatMap(_ => urlEntities.lastOption).map(_.fromIndex)) - .map(from => (from - 1).max(0)) // for whitespace, unless there is none - .map(Offset.CodePoint(_)) - .getOrElse(totalTextLength) - - val visibleStart = replyResult match { - case Some(rr) => rr.visibleStart.min(visibleEnd) - case None => Offset.CodePoint(0) - } - - if (visibleStart.toInt == 0 && visibleEnd == totalTextLength) { - TextVisibility( - visibleTextRange = None, - totalTextDisplayLength = totalTextDisplayLength, - visibleText = text - ) - } else { - val charFrom = visibleStart.toCodeUnit(text) - val charTo = charFrom.offsetByCodePoints(text, visibleEnd - visibleStart) - val visibleText = text.substring(charFrom.toInt, charTo.toInt) - - TextVisibility( - visibleTextRange = Some(TextRange(visibleStart.toInt, visibleEnd.toInt)), - totalTextDisplayLength = totalTextDisplayLength, - visibleText = visibleText - ) - } - } - - def isValidHashtag(entity: HashtagEntity): Boolean = - TweetText.codePointLength(entity.text) <= TweetText.MaxHashtagLength - - /** - * Validates that the number of various entities are within the limits, and the - * length of hashtags are with the limit. - */ - def validateEntities(tweet: Tweet): Future[Unit] = - if (getMentions(tweet).length > TweetText.MaxMentions) - Future.exception(TweetCreateFailure.State(MentionLimitExceeded)) - else if (getUrls(tweet).length > TweetText.MaxUrls) - Future.exception(TweetCreateFailure.State(UrlLimitExceeded)) - else if (getHashtags(tweet).length > TweetText.MaxHashtags) - Future.exception(TweetCreateFailure.State(HashtagLimitExceeded)) - else if (getCashtags(tweet).length > TweetText.MaxCashtags) - Future.exception(TweetCreateFailure.State(CashtagLimitExceeded)) - else if (getHashtags(tweet).exists(e => !isValidHashtag(e))) - Future.exception(TweetCreateFailure.State(HashtagLengthLimitExceeded)) - else - Future.Unit - - /** - * Update the user to what it should look like after the tweet is created - */ - def updateUserCounts(hasMedia: Tweet => Boolean): (User, Tweet) => Future[User] = - (user: User, tweet: Tweet) => { - val countAsUserTweet = isUserTweet(tweet) - val tweetsDelta = if (countAsUserTweet) 1 else 0 - val mediaTweetsDelta = if (countAsUserTweet && hasMedia(tweet)) 1 else 0 - - Future.value( - user.copy( - counts = user.counts.map { counts => - counts.copy( - tweets = counts.tweets + tweetsDelta, - mediaTweets = counts.mediaTweets.map(_ + mediaTweetsDelta) - ) - } - ) - ) - } - - def validateAdditionalFields[R](implicit view: RequestView[R]): FutureEffect[R] = - FutureEffect[R] { req => - view - .additionalFields(req) - .map(tweet => - unsettableAdditionalFieldIds(tweet) ++ rejectedAdditionalFieldIds(tweet)) match { - case Some(unsettableFieldIds) if unsettableFieldIds.nonEmpty => - Future.exception( - TweetCreateFailure.State( - InvalidAdditionalField, - Some(unsettableAdditionalFieldIdsErrorMessage(unsettableFieldIds)) - ) - ) - case _ => Future.Unit - } - } - - def validateTweetMediaTags( - stats: StatsReceiver, - getUserMediaTagRateLimit: RateLimitChecker.GetRemaining, - userRepo: UserRepository.Optional - ): (Tweet, Boolean) => Future[Mutation[Tweet]] = { - val userRepoWithStats: UserRepository.Optional = - (userKey, queryOptions) => - userRepo(userKey, queryOptions).liftToTry.map { - case Return(res @ Some(_)) => - stats.counter("found").incr() - res - case Return(None) => - stats.counter("not_found").incr() - None - case Throw(_) => - stats.counter("failed").incr() - None - } - - (tweet: Tweet, dark: Boolean) => { - val mediaTags = getMediaTagMap(tweet) - - if (mediaTags.isEmpty) { - MutationUnitFuture - } else { - getUserMediaTagRateLimit((getUserId(tweet), dark)).flatMap { remainingMediaTagCount => - val maxMediaTagCount = math.min(remainingMediaTagCount, MaxMediaTagCount) - - val taggedUserIds = - mediaTags.values.flatten.toSeq.collect { - case MediaTag(MediaTagType.User, Some(userId), _, _) => userId - }.distinct - - val droppedTagCount = taggedUserIds.size - maxMediaTagCount - if (droppedTagCount > 0) stats.counter("over_limit_tags").incr(droppedTagCount) - - val userQueryOpts = - UserQueryOptions( - queryFields = Set(UserField.MediaView), - visibility = UserVisibility.MediaTaggable, - forUserId = Some(getUserId(tweet)) - ) - - val keys = taggedUserIds.take(maxMediaTagCount).map(UserKey.byId) - val keyOpts = keys.map((_, userQueryOpts)) - - Stitch.run { - Stitch - .traverse(keyOpts)(userRepoWithStats.tupled) - .map(_.flatten) - .map { users => - val userMap = users.map(u => u.id -> u).toMap - val mediaTagsMutation = - Mutation[Seq[MediaTag]] { mediaTags => - val validMediaTags = - mediaTags.filter { - case MediaTag(MediaTagType.User, Some(userId), _, _) => - userMap.get(userId).exists(_.mediaView.exists(_.canMediaTag)) - case _ => false - } - val invalidCount = mediaTags.size - validMediaTags.size - - if (invalidCount != 0) { - stats.counter("invalid").incr(invalidCount) - Some(validMediaTags) - } else { - None - } - } - TweetLenses.mediaTagMap.mutation(mediaTagsMutation.liftMapValues) - } - } - } - } - } - } - - def validateCommunityMembership( - communityMembershipRepository: StratoCommunityMembershipRepository.Type, - communityAccessRepository: StratoCommunityAccessRepository.Type, - communities: Option[Communities] - ): Future[Unit] = - communities match { - case Some(Communities(Seq(communityId))) => - Stitch - .run { - communityMembershipRepository(communityId).flatMap { - case true => Stitch.value(None) - case false => - communityAccessRepository(communityId).map { - case Some(CommunityAccess.Public) | Some(CommunityAccess.Closed) => - Some(TweetCreateState.CommunityUserNotAuthorized) - case Some(CommunityAccess.Private) | None => - Some(TweetCreateState.CommunityNotFound) - } - } - }.flatMap { - case None => - Future.Done - case Some(tweetCreateState) => - Future.exception(TweetCreateFailure.State(tweetCreateState)) - } - case Some(Communities(communities)) if communities.length > 1 => - // Not allowed to specify more than one community ID. - Future.exception(TweetCreateFailure.State(TweetCreateState.InvalidAdditionalField)) - case _ => Future.Done - } - - private[this] val CardUriSchemeRegex = "(?i)^(?:card|tombstone):".r - - /** - * Is the given String a URI that is allowed as a card reference - * without a matching URL in the text? - */ - def hasCardsUriScheme(uri: String): Boolean = - CardUriSchemeRegex.findPrefixMatchOf(uri).isDefined - - val InvalidAdditionalFieldEmptyUrlEntities: TweetCreateFailure.State = - TweetCreateFailure.State( - TweetCreateState.InvalidAdditionalField, - Some("url entities are empty") - ) - - val InvalidAdditionalFieldNonMatchingUrlAndShortUrl: TweetCreateFailure.State = - TweetCreateFailure.State( - TweetCreateState.InvalidAdditionalField, - Some("non-matching url and short url") - ) - - val InvalidAdditionalFieldInvalidUri: TweetCreateFailure.State = - TweetCreateFailure.State( - TweetCreateState.InvalidAdditionalField, - Some("invalid URI") - ) - - val InvalidAdditionalFieldInvalidCardUri: TweetCreateFailure.State = - TweetCreateFailure.State( - TweetCreateState.InvalidAdditionalField, - Some("invalid card URI") - ) - - type CardReferenceBuilder = - (Tweet, UrlShortener.Context) => Future[Mutation[Tweet]] - - def cardReferenceBuilder( - cardReferenceValidator: CardReferenceValidationHandler.Type, - urlShortener: UrlShortener.Type - ): CardReferenceBuilder = - (tweet, urlShortenerCtx) => { - getCardReference(tweet) match { - case Some(CardReference(uri)) => - for { - cardUri <- - if (hasCardsUriScheme(uri)) { - // This is an explicit card references that does not - // need a corresponding URL in the text. - Future.value(uri) - } else if (UrlPattern.matcher(uri).matches) { - // The card reference is being used to specify which URL - // card to show. We need to verify that the URL is - // actually in the tweet text, or it can be effectively - // used to bypass the tweet length limit. - val urlEntities = getUrls(tweet) - - if (urlEntities.isEmpty) { - // Fail fast if there can't possibly be a matching URL entity - Future.exception(InvalidAdditionalFieldEmptyUrlEntities) - } else { - // Look for the URL in the expanded URL entities. If - // it is present, then map it to the t.co shortened - // version of the URL. - urlEntities - .collectFirst { - case urlEntity if urlEntity.expanded.exists(_ == uri) => - Future.value(urlEntity.url) - } - .getOrElse { - // The URL may have been altered when it was - // returned from Talon, such as expanding a pasted - // t.co link. In this case, we t.co-ize the link and - // make sure that the corresponding t.co is present - // as a URL entity. - urlShortener((uri, urlShortenerCtx)).flatMap { shortened => - if (urlEntities.exists(_.url == shortened.shortUrl)) { - Future.value(shortened.shortUrl) - } else { - Future.exception(InvalidAdditionalFieldNonMatchingUrlAndShortUrl) - } - } - } - } - } else { - Future.exception(InvalidAdditionalFieldInvalidUri) - } - - validatedCardUri <- cardReferenceValidator((getUserId(tweet), cardUri)).rescue { - case CardReferenceValidationFailedException => - Future.exception(InvalidAdditionalFieldInvalidCardUri) - } - } yield { - TweetLenses.cardReference.mutation( - Mutation[CardReference] { cardReference => - Some(cardReference.copy(cardUri = validatedCardUri)) - }.checkEq.liftOption - ) - } - - case None => - MutationUnitFuture - } - } - - def filterInvalidData( - validateTweetMediaTags: (Tweet, Boolean) => Future[Mutation[Tweet]], - cardReferenceBuilder: CardReferenceBuilder - ): (Tweet, PostTweetRequest, UrlShortener.Context) => Future[Tweet] = - (tweet: Tweet, request: PostTweetRequest, urlShortenerCtx: UrlShortener.Context) => { - Future - .join( - validateTweetMediaTags(tweet, request.dark), - cardReferenceBuilder(tweet, urlShortenerCtx) - ) - .map { - case (mediaMutation, cardRefMutation) => - mediaMutation.also(cardRefMutation).endo(tweet) - } - } - - def apply( - stats: StatsReceiver, - validateRequest: PostTweetRequest => Future[Unit], - validateEdit: EditValidator.Type, - validateUser: User => Future[Unit] = TweetBuilder.validateUser, - validateUpdateRateLimit: RateLimitChecker.Validate, - tweetIdGenerator: TweetIdGenerator, - userRepo: UserRepository.Type, - deviceSourceRepo: DeviceSourceRepository.Type, - communityMembershipRepo: StratoCommunityMembershipRepository.Type, - communityAccessRepo: StratoCommunityAccessRepository.Type, - urlShortener: UrlShortener.Type, - urlEntityBuilder: UrlEntityBuilder.Type, - geoBuilder: GeoBuilder.Type, - replyBuilder: ReplyBuilder.Type, - mediaBuilder: MediaBuilder.Type, - attachmentBuilder: AttachmentBuilder.Type, - duplicateTweetFinder: DuplicateTweetFinder.Type, - spamChecker: Spam.Checker[TweetSpamRequest], - filterInvalidData: (Tweet, PostTweetRequest, UrlShortener.Context) => Future[Tweet], - updateUserCounts: (User, Tweet) => Future[User], - validateConversationControl: ConversationControlBuilder.Validate.Type, - conversationControlBuilder: ConversationControlBuilder.Type, - validateTweetWrite: TweetWriteValidator.Type, - nudgeBuilder: NudgeBuilder.Type, - communitiesValidator: CommunitiesValidator.Type, - collabControlBuilder: CollabControlBuilder.Type, - editControlBuilder: EditControlBuilder.Type, - featureSwitches: FeatureSwitches - ): TweetBuilder.Type = { - val entityExtractor = EntityExtractor.mutationWithoutUrls.endo - val getUser = userLookup(userRepo) - val getDeviceSource = deviceSourceLookup(deviceSourceRepo) - - // create a tco of the permalink for given a tweetId - val permalinkShortener = (tweetId: TweetId, ctx: UrlShortener.Context) => - urlShortener((s"https://twitter.com/i/web/status/$tweetId", ctx)).rescue { - // propagate OverCapacity - case e: OverCapacity => Future.exception(e) - // convert any other failure into UrlShorteningFailure - case e => Future.exception(UrlShorteningFailure(e)) - } - - def extractGeoSearchRequestId(tweetGeoOpt: Option[TweetCreateGeo]): Option[GeoSearchRequestId] = - for { - tweetGeo <- tweetGeoOpt - geoSearchRequestId <- tweetGeo.geoSearchRequestId - } yield GeoSearchRequestId(geoSearchRequestId.id) - - def featureSwitchResults(user: User, stats: StatsReceiver): Option[FeatureSwitchResults] = - TwitterContext() - .flatMap { viewer => - UserViewerRecipient(user, viewer, stats) - }.map { recipient => - featureSwitches.matchRecipient(recipient) - } - - FutureArrow { request => - for { - () <- validateRequest(request) - - (tweetId, user, devsrc) <- Future.join( - tweetIdGenerator().rescue { case t => Future.exception(SnowflakeFailure(t)) }, - Stitch.run(getUser(request.userId)), - Stitch.run(getDeviceSource(request.createdVia)) - ) - - () <- validateUser(user) - () <- validateUpdateRateLimit((user.id, request.dark)) - - // Feature Switch results are calculated once and shared between multiple builders - matchedResults = featureSwitchResults(user, stats) - - () <- validateConversationControl( - ConversationControlBuilder.Validate.Request( - matchedResults = matchedResults, - conversationControl = request.conversationControl, - inReplyToTweetId = request.inReplyToTweetId - ) - ) - - // strip illegal chars, normalize newlines, collapse blank lines, etc. - text = preprocessText(request.text) - - () <- prevalidateTextLength(text, stats) - - attachmentResult <- attachmentBuilder( - AttachmentBuilderRequest( - tweetId = tweetId, - user = user, - mediaUploadIds = request.mediaUploadIds, - cardReference = request.additionalFields.flatMap(_.cardReference), - attachmentUrl = request.attachmentUrl, - remoteHost = request.remoteHost, - darkTraffic = request.dark, - deviceSource = devsrc - ) - ) - - // updated text with appended attachment url, if any. - text <- Future.value( - attachmentResult.attachmentUrl match { - case None => text - case Some(url) => s"$text $url" - } - ) - - spamResult <- spamChecker( - TweetSpamRequest( - tweetId = tweetId, - userId = request.userId, - text = text, - mediaTags = request.additionalFields.flatMap(_.mediaTags), - safetyMetaData = request.safetyMetaData, - inReplyToTweetId = request.inReplyToTweetId, - quotedTweetId = attachmentResult.quotedTweet.map(_.tweetId), - quotedTweetUserId = attachmentResult.quotedTweet.map(_.userId) - ) - ) - - safety = user.safety.get - createdAt = SnowflakeId(tweetId).time - - urlShortenerCtx = UrlShortener.Context( - tweetId = tweetId, - userId = user.id, - createdAt = createdAt, - userProtected = safety.isProtected, - clientAppId = devsrc.clientAppId, - remoteHost = request.remoteHost, - dark = request.dark - ) - - replyRequest = ReplyBuilder.Request( - authorId = request.userId, - authorScreenName = user.profile.map(_.screenName).get, - inReplyToTweetId = request.inReplyToTweetId, - tweetText = text, - prependImplicitMentions = request.autoPopulateReplyMetadata, - enableTweetToNarrowcasting = request.enableTweetToNarrowcasting, - excludeUserIds = request.excludeReplyUserIds.getOrElse(Nil), - spamResult = spamResult, - batchMode = request.transientContext.flatMap(_.batchCompose) - ) - - replyResult <- replyBuilder(replyRequest) - replyOpt = replyResult.map(_.reply) - - replyConversationId <- replyResult match { - case Some(r) if r.reply.inReplyToStatusId.nonEmpty => - r.conversationId match { - case None => - // Throw this specific exception to make it easier to - // count how often we hit this corner case. - Future.exception(MissingConversationId(r.reply.inReplyToStatusId.get)) - case conversationIdOpt => Future.value(conversationIdOpt) - } - case _ => Future.value(None) - } - - // Validate that the current user can reply to this conversation, based on - // the conversation's ConversationControl. - // Note: currently we only validate conversation controls access on replies, - // therefore we use the conversationId from the inReplyToStatus. - // Validate that the exclusive tweet control option is only used by allowed users. - () <- validateTweetWrite( - TweetWriteValidator.Request( - replyConversationId, - request.userId, - request.exclusiveTweetControlOptions, - replyResult.flatMap(_.exclusiveTweetControl), - request.trustedFriendsControlOptions, - replyResult.flatMap(_.trustedFriendsControl), - attachmentResult.quotedTweet, - replyResult.flatMap(_.reply.inReplyToStatusId), - replyResult.flatMap(_.editControl), - request.editOptions - ) - ) - - convoId = replyConversationId match { - case Some(replyConvoId) => replyConvoId - case None => - // This is a root tweet, so the tweet id is the conversation id. - tweetId - } - - () <- nudgeBuilder( - NudgeBuilderRequest( - text = text, - inReplyToTweetId = replyOpt.flatMap(_.inReplyToStatusId), - conversationId = if (convoId == tweetId) None else Some(convoId), - hasQuotedTweet = attachmentResult.quotedTweet.nonEmpty, - nudgeOptions = request.nudgeOptions, - tweetId = Some(tweetId), - ) - ) - - // updated text with implicit reply mentions inserted, if any - text <- Future.value( - replyResult.map(_.tweetText).getOrElse(text) - ) - - // updated text with urls replaced with t.cos - ((text, urlEntities), (geoCoords, placeIdOpt)) <- Future.join( - urlEntityBuilder((text, urlShortenerCtx)) - .map { - case (text, urlEntities) => - UrlEntityBuilder.updateTextAndUrls(text, urlEntities)(partialHtmlEncode) - }, - if (request.geo.isEmpty) - Future.value((None, None)) - else - geoBuilder( - GeoBuilder.Request( - request.geo.get, - user.account.map(_.geoEnabled).getOrElse(false), - user.account.map(_.language).getOrElse("en") - ) - ).map(r => (r.geoCoordinates, r.placeId)) - ) - - // updated text with trailing media url - MediaBuilder.Result(text, mediaEntities, mediaKeys) <- - request.mediaUploadIds.getOrElse(Nil) match { - case Nil => Future.value(MediaBuilder.Result(text, Nil, Nil)) - case ids => - mediaBuilder( - MediaBuilder.Request( - mediaUploadIds = ids, - text = text, - tweetId = tweetId, - userId = user.id, - userScreenName = user.profile.get.screenName, - isProtected = user.safety.get.isProtected, - createdAt = createdAt, - dark = request.dark, - productMetadata = request.mediaMetadata.map(_.toMap) - ) - ) - } - - () <- Future.when(!request.dark) { - val reqInfo = - DuplicateTweetFinder.RequestInfo.fromPostTweetRequest(request, text) - - duplicateTweetFinder(reqInfo).flatMap { - case None => Future.Unit - case Some(duplicateId) => - log.debug(s"timeline_duplicate_check_failed:$duplicateId") - Future.exception(TweetCreateFailure.State(TweetCreateState.Duplicate)) - } - } - - textVisibility = getTextVisibility( - text = text, - replyResult = replyResult, - urlEntities = urlEntities, - mediaEntities = mediaEntities, - attachmentUrl = attachmentResult.attachmentUrl - ) - - () <- validateTextLength( - text = text, - visibleText = textVisibility.visibleText, - replyResult = replyResult, - stats = stats - ) - - communities = - request.additionalFields - .flatMap(CommunityAnnotation.additionalFieldsToCommunityIDs) - .map(ids => Communities(communityIds = ids)) - - rootExclusiveControls = request.exclusiveTweetControlOptions.map { _ => - ExclusiveTweetControl(request.userId) - } - - () <- validateExclusiveTweetNotReplies(rootExclusiveControls, replyResult) - () <- validateExclusiveTweetParams(rootExclusiveControls, communities) - - replyExclusiveControls = replyResult.flatMap(_.exclusiveTweetControl) - - // The userId is pulled off of the request rather than being supplied - // via the ExclusiveTweetControlOptions because additional fields - // can be set by clients to contain any value they want. - // This could include userIds that don't match their actual userId. - // Only one of replyResult or request.exclusiveTweetControlOptions will be defined. - exclusiveTweetControl = replyExclusiveControls.orElse(rootExclusiveControls) - - rootTrustedFriendsControl = request.trustedFriendsControlOptions.map { options => - TrustedFriendsControl(options.trustedFriendsListId) - } - - () <- validateTrustedFriendsNotReplies(rootTrustedFriendsControl, replyResult) - () <- validateTrustedFriendsParams( - rootTrustedFriendsControl, - request.conversationControl, - communities, - exclusiveTweetControl - ) - - replyTrustedFriendsControl = replyResult.flatMap(_.trustedFriendsControl) - - trustedFriendsControl = replyTrustedFriendsControl.orElse(rootTrustedFriendsControl) - - collabControl <- collabControlBuilder( - CollabControlBuilder.Request( - collabControlOptions = request.collabControlOptions, - replyResult = replyResult, - communities = communities, - trustedFriendsControl = trustedFriendsControl, - conversationControl = request.conversationControl, - exclusiveTweetControl = exclusiveTweetControl, - userId = request.userId - )) - - isCollabInvitation = collabControl.isDefined && (collabControl.get match { - case CollabControl.CollabInvitation(_: CollabInvitation) => true - case _ => false - }) - - coreData = TweetCoreData( - userId = request.userId, - text = text, - createdAtSecs = createdAt.inSeconds, - createdVia = devsrc.internalName, - reply = replyOpt, - hasTakedown = safety.hasTakedown, - // We want to nullcast community tweets and CollabInvitations - // This will disable tweet fanout to followers' home timelines, - // and filter the tweets from appearing from the tweeter's profile - // or search results for the tweeter's tweets. - nullcast = - request.nullcast || CommunityUtil.hasCommunity(communities) || isCollabInvitation, - narrowcast = request.narrowcast, - nsfwUser = request.possiblySensitive.getOrElse(safety.nsfwUser), - nsfwAdmin = safety.nsfwAdmin, - trackingId = request.trackingId, - placeId = placeIdOpt, - coordinates = geoCoords, - conversationId = Some(convoId), - // Set hasMedia to true if we know that there is media, - // and leave it unknown if not, so that it will be - // correctly set for pasted media. - hasMedia = if (mediaEntities.nonEmpty) Some(true) else None - ) - - tweet = Tweet( - id = tweetId, - coreData = Some(coreData), - urls = Some(urlEntities), - media = Some(mediaEntities), - mediaKeys = if (mediaKeys.nonEmpty) Some(mediaKeys) else None, - contributor = getContributor(request.userId), - visibleTextRange = textVisibility.visibleTextRange, - selfThreadMetadata = replyResult.flatMap(_.selfThreadMetadata), - directedAtUserMetadata = replyResult.map(_.directedAtMetadata), - composerSource = request.composerSource, - quotedTweet = attachmentResult.quotedTweet, - exclusiveTweetControl = exclusiveTweetControl, - trustedFriendsControl = trustedFriendsControl, - collabControl = collabControl, - noteTweet = request.noteTweetOptions.map(options => - NoteTweet(options.noteTweetId, options.isExpandable)) - ) - - editControl <- editControlBuilder( - EditControlBuilder.Request( - postTweetRequest = request, - tweet = tweet, - matchedResults = matchedResults - ) - ) - - tweet <- Future.value(tweet.copy(editControl = editControl)) - - tweet <- Future.value(entityExtractor(tweet)) - - () <- validateEntities(tweet) - - tweet <- { - val cctlRequest = - ConversationControlBuilder.Request.fromTweet( - tweet, - request.conversationControl, - request.noteTweetOptions.flatMap(_.mentionedUserIds)) - Stitch.run(conversationControlBuilder(cctlRequest)).map { conversationControl => - tweet.copy(conversationControl = conversationControl) - } - } - - tweet <- Future.value( - setAdditionalFields(tweet, request.additionalFields) - ) - () <- validateCommunityMembership(communityMembershipRepo, communityAccessRepo, communities) - () <- validateCommunityReply(communities, replyResult) - () <- communitiesValidator( - CommunitiesValidator.Request(matchedResults, safety.isProtected, communities)) - - tweet <- Future.value(tweet.copy(communities = communities)) - - tweet <- Future.value( - tweet.copy(underlyingCreativesContainerId = request.underlyingCreativesContainerId) - ) - - // For certain tweets we want to write a self-permalink which is used to generate modified - // tweet text for legacy clients that contains a link. NOTE: this permalink is for - // the tweet being created - we also create permalinks for related tweets further down - // e.g. if this tweet is an edit, we might create a permalink for the initial tweet as well - tweet <- { - val isBeyond140 = textVisibility.isExtendedWithExtraChars(attachmentResult.extraChars) - val isEditTweet = request.editOptions.isDefined - val isMixedMedia = Media.isMixedMedia(mediaEntities) - val isNoteTweet = request.noteTweetOptions.isDefined - - if (isBeyond140 || isEditTweet || isMixedMedia || isNoteTweet) - permalinkShortener(tweetId, urlShortenerCtx) - .map { selfPermalink => - tweet.copy( - selfPermalink = Some(selfPermalink), - extendedTweetMetadata = Some(ExtendedTweetMetadataBuilder(tweet, selfPermalink)) - ) - } - else { - Future.value(tweet) - } - } - - // When an edit tweet is created we have to update some information on the - // initial tweet, this object stores info about those updates for use - // in the tweet insert store. - // We update the editControl for each edit tweet and for the first edit tweet - // we update the self permalink. - initialTweetUpdateRequest: Option[InitialTweetUpdateRequest] <- editControl match { - case Some(EditControl.Edit(edit)) => - // Identifies the first edit of an initial tweet - val isFirstEdit = - request.editOptions.map(_.previousTweetId).contains(edit.initialTweetId) - - // A potential permalink for this tweet being created's initial tweet - val selfPermalinkForInitial: Future[Option[ShortenedUrl]] = - if (isFirstEdit) { - // `tweet` is the first edit of an initial tweet, which means - // we need to write a self permalink. We create it here in - // TweetBuilder and pass it through to the tweet store to - // be written to the initial tweet. - permalinkShortener(edit.initialTweetId, urlShortenerCtx).map(Some(_)) - } else { - Future.value(None) - } - - selfPermalinkForInitial.map { link => - Some( - InitialTweetUpdateRequest( - initialTweetId = edit.initialTweetId, - editTweetId = tweet.id, - selfPermalink = link - )) - } - - // This is not an edit this is the initial tweet - so there are no initial - // tweet updates - case _ => Future.value(None) - } - - tweet <- filterInvalidData(tweet, request, urlShortenerCtx) - - () <- validateEdit(tweet, request.editOptions) - - user <- updateUserCounts(user, tweet) - - } yield { - TweetBuilderResult( - tweet, - user, - createdAt, - isSilentFail = spamResult == Spam.SilentFail, - geoSearchRequestId = extractGeoSearchRequestId(request.geo), - initialTweetUpdateRequest = initialTweetUpdateRequest - ) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetCreationLock.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetCreationLock.docx new file mode 100644 index 000000000..fc063a7e5 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetCreationLock.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetCreationLock.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetCreationLock.scala deleted file mode 100644 index a530e95a2..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetCreationLock.scala +++ /dev/null @@ -1,402 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.servo.cache.Cache -import com.twitter.servo.util.Scribe -import com.twitter.tweetypie.serverutil.ExceptionCounter -import com.twitter.tweetypie.thriftscala.PostTweetResult -import com.twitter.tweetypie.util.TweetCreationLock.Key -import com.twitter.tweetypie.util.TweetCreationLock.State -import com.twitter.util.Base64Long -import scala.util.Random -import scala.util.control.NoStackTrace -import scala.util.control.NonFatal - -/** - * This exception is returned from TweetCreationLock if there is an - * in-progress cache entry for this key. It is possible that the key - * exists because the key was not properly cleaned up, but it's - * impossible to differentiate between these cases. We resolve this by - * returning TweetCreationInProgress and having a (relatively) short TTL - * on the cache entry so that the client and/or user may retry. - */ -case object TweetCreationInProgress extends Exception with NoStackTrace - -/** - * Thrown when the TweetCreationLock discovers that there is already - * a tweet with the specified uniqueness id. - */ -case class DuplicateTweetCreation(tweetId: TweetId) extends Exception with NoStackTrace - -trait TweetCreationLock { - def apply( - key: Key, - dark: Boolean, - nullcast: Boolean - )( - insert: => Future[PostTweetResult] - ): Future[PostTweetResult] - def unlock(key: Key): Future[Unit] -} - -object CacheBasedTweetCreationLock { - - /** - * Indicates that setting the lock value failed because the state of - * that key in the cache has been changed (by another process or - * cache eviction). - */ - case object UnexpectedCacheState extends Exception with NoStackTrace - - /** - * Thrown when the process of updating the lock cache failed more - * than the allowed number of times. - */ - case class RetriesExhausted(failures: Seq[Exception]) extends Exception with NoStackTrace - - def shouldRetry(e: Exception): Boolean = - e match { - case TweetCreationInProgress => false - case _: DuplicateTweetCreation => false - case _: RetriesExhausted => false - case _ => true - } - - def ttlChooser(shortTtl: Duration, longTtl: Duration): (Key, State) => Duration = - (_, state) => - state match { - case _: State.AlreadyCreated => longTtl - case _ => shortTtl - } - - /** - * The log format is tab-separated (base 64 tweet_id, base 64 - * uniqueness_id). It's logged this way in order to minimize the - * storage requirement and to make it easy to analyze. Each log line - * should be 24 bytes, including newline. - */ - val formatUniquenessLogEntry: ((String, TweetId)) => String = { - case (uniquenessId, tweetId) => Base64Long.toBase64(tweetId) + "\t" + uniquenessId - } - - /** - * Scribe the uniqueness id paired with the tweet id so that we can - * track the rate of failures of the uniqueness id check by - * detecting multiple tweets created with the same uniqueness id. - * - * Scribe to a test category because we only need to keep this - * information around for long enough to find any duplicates. - */ - val ScribeUniquenessId: FutureEffect[(String, TweetId)] = - Scribe("test_tweetypie_uniqueness_id") contramap formatUniquenessLogEntry - - private[this] val UniquenessIdLog = Logger("com.twitter.tweetypie.handler.UniquenessId") - - /** - * Log the uniqueness ids to a standard logger (for use when it's - * not production traffic). - */ - val LogUniquenessId: FutureEffect[(String, TweetId)] = FutureEffect[(String, TweetId)] { rec => - UniquenessIdLog.info(formatUniquenessLogEntry(rec)) - Future.Unit - } - - private val log = Logger(getClass) -} - -/** - * This class adds locking around Tweet creation, to prevent creating - * duplicate tweets when two identical requests arrive simultaneously. - * A lock is created in cache using the user id and a hash of the tweet text - * in the case of tweets, or the source_status_id in the case of retweets. - * If another process attempts to lock for the same user and hash, the request - * fails as a duplicate. The lock lasts for 10 seconds if it is not deleted. - * Given the hard timeout of 5 seconds on all requests, it should never take - * us longer than 5 seconds to create a request, but we've observed times of up - * to 10 seconds to create statuses for some of our more popular users. - * - * When a request with a uniqueness id is successful, the id of the - * created tweet will be stored in the cache so that subsequent - * requests can retrieve the originally-created tweet rather than - * duplicating creation or getting an exception. - */ -class CacheBasedTweetCreationLock( - cache: Cache[Key, State], - maxTries: Int, - stats: StatsReceiver, - logUniquenessId: FutureEffect[(String, TweetId)]) - extends TweetCreationLock { - import CacheBasedTweetCreationLock._ - - private[this] val eventCounters = stats.scope("event") - - private[this] def event(k: Key, name: String): Unit = { - log.debug(s"$name:$k") - eventCounters.counter(name).incr() - } - - private[this] def retryLoop[A](action: => Future[A]): Future[A] = { - def go(failures: List[Exception]): Future[A] = - if (failures.length >= maxTries) { - Future.exception(RetriesExhausted(failures.reverse)) - } else { - action.rescue { - case e: Exception if shouldRetry(e) => go(e :: failures) - } - } - - go(Nil) - } - - private[this] val lockerExceptions = ExceptionCounter(stats) - - /** - * Obtain the lock for creating a tweet. If this method completes - * without throwing an exception, then the lock value was - * successfully set in cache, which indicates a high probability - * that this is the only process that is attempting to create this - * tweet. (The uncertainty comes from the possibility of lock - * entries missing from the cache.) - * - * @throws TweetCreationInProgress if there is another process - * trying to create this tweet. - * - * @throws DuplicateTweetCreation if a tweet has already been - * created for a duplicate request. The exception has the id of - * the created tweet. - * - * @throws RetriesExhausted if obtaining the lock failed more than - * the requisite number of times. - */ - private[this] def obtainLock(k: Key, token: Long): Future[Time] = retryLoop { - val lockTime = Time.now - - // Get the current state for this key. - cache - .getWithChecksum(Seq(k)) - .flatMap(initialStateKvr => Future.const(initialStateKvr(k))) - .flatMap { - case None => - // Nothing in cache for this key - cache - .add(k, State.InProgress(token, lockTime)) - .flatMap { - case true => Future.value(lockTime) - case false => Future.exception(UnexpectedCacheState) - } - case Some((Throw(e), _)) => - Future.exception(e) - case Some((Return(st), cs)) => - st match { - case State.Unlocked => - // There is an Unlocked entry for this key, which - // implies that a previous attempt was cleaned up. - cache - .checkAndSet(k, State.InProgress(token, lockTime), cs) - .flatMap { - case true => Future.value(lockTime) - case false => Future.exception(UnexpectedCacheState) - } - case State.InProgress(cachedToken, creationStartedTimestamp) => - if (cachedToken == token) { - // There is an in-progress entry for *this process*. This - // can happen on a retry if the `add` actually succeeds - // but the future fails. The retry can return the result - // of the add that we previously tried. - Future.value(creationStartedTimestamp) - } else { - // There is an in-progress entry for *a different - // process*. This implies that there is another tweet - // creation in progress for *this tweet*. - val tweetCreationAge = Time.now - creationStartedTimestamp - k.uniquenessId.foreach { id => - log.info( - "Found an in-progress tweet creation for uniqueness id %s %s ago" - .format(id, tweetCreationAge) - ) - } - stats.stat("in_progress_age_ms").add(tweetCreationAge.inMilliseconds) - Future.exception(TweetCreationInProgress) - } - case State.AlreadyCreated(tweetId, creationStartedTimestamp) => - // Another process successfully created a tweet for this - // key. - val tweetCreationAge = Time.now - creationStartedTimestamp - stats.stat("already_created_age_ms").add(tweetCreationAge.inMilliseconds) - Future.exception(DuplicateTweetCreation(tweetId)) - } - } - } - - /** - * Attempt to remove this process' lock entry from the cache. This - * is done by writing a short-lived tombstone, so that we can ensure - * that we only overwrite the entry if it is still an entry for this - * process instead of another process' entry. - */ - private[this] def cleanupLoop(k: Key, token: Long): Future[Unit] = - retryLoop { - // Instead of deleting the value, we attempt to write Unlocked, - // because we only want to delete it if it was the value that we - // wrote ourselves, and there is no delete call that is - // conditional on the extant value. - cache - .getWithChecksum(Seq(k)) - .flatMap(kvr => Future.const(kvr(k))) - .flatMap { - case None => - // Nothing in the cache for this tweet creation, so cleanup - // is successful. - Future.Unit - - case Some((tryV, cs)) => - // If we failed trying to deserialize the value, then we - // want to let the error bubble up, because there is no good - // recovery procedure, since we can't tell whether the entry - // is ours. - Future.const(tryV).flatMap { - case State.InProgress(presentToken, _) => - if (presentToken == token) { - // This is *our* in-progress marker, so we want to - // overwrite it with the tombstone. If checkAndSet - // returns false, that's OK, because that means - // someone else overwrote the value, and we don't have - // to clean it up anymore. - cache.checkAndSet(k, State.Unlocked, cs).unit - } else { - // Indicates that another request has overwritten our - // state before we cleaned it up. This should only - // happen when our token was cleared from cache and - // another process started a duplicate create. This - // should be very infrequent. We count it just to be - // sure. - event(k, "other_attempt_in_progress") - Future.Unit - } - - case _ => - // Cleanup has succeeded, because we are not responsible - // for the cache entry anymore. - Future.Unit - } - } - }.onSuccess { _ => event(k, "cleanup_attempt_succeeded") } - .handle { - case _ => event(k, "cleanup_attempt_failed") - } - - /** - * Mark that a tweet has been successfully created. Subsequent calls - * to `apply` with this key will receive a DuplicateTweetCreation - * exception with the specified id. - */ - private[this] def creationComplete(k: Key, tweetId: TweetId, lockTime: Time): Future[Unit] = - // Unconditionally set the state because regardless of the - // value present, we know that we want to transition to the - // AlreadyCreated state for this key. - retryLoop(cache.set(k, State.AlreadyCreated(tweetId, lockTime))) - .onSuccess(_ => event(k, "mark_created_succeeded")) - .onFailure { case _ => event(k, "mark_created_failed") } - // If this fails, it's OK for the request to complete - // successfully, because it's more harmful to create the tweet - // and return failure than it is to complete it successfully, - // but fail to honor the uniqueness id next time. - .handle { case NonFatal(_) => } - - private[this] def createWithLock( - k: Key, - create: => Future[PostTweetResult] - ): Future[PostTweetResult] = { - val token = Random.nextLong - event(k, "lock_attempted") - - obtainLock(k, token) - .onSuccess { _ => event(k, "lock_obtained") } - .handle { - // If we run out of retries when trying to get the lock, then - // just go ahead with tweet creation. We should keep an eye on - // how frequently this happens, because this means that the - // only sign that this is happening will be duplicate tweet - // creations. - case RetriesExhausted(failures) => - event(k, "lock_failure_ignored") - // Treat this as the time that we obtained the lock. - Time.now - } - .onFailure { - case e => lockerExceptions(e) - } - .flatMap { lockTime => - create.transform { - case r @ Return(PostTweetResult(_, Some(tweet), _, _, _, _, _)) => - event(k, "create_succeeded") - - k.uniquenessId.foreach { u => logUniquenessId((u, tweet.id)) } - - // Update the lock entry to remember the id of the tweet we - // created and extend the TTL. - creationComplete(k, tweet.id, lockTime).before(Future.const(r)) - case other => - other match { - case Throw(e) => - log.debug(s"Tweet creation failed for key $k", e) - case Return(r) => - log.debug(s"Tweet creation failed for key $k, so unlocking: $r") - } - - event(k, "create_failed") - - // Attempt to clean up the lock after the failed create. - cleanupLoop(k, token).before(Future.const(other)) - } - } - } - - /** - * Make a best-effort attempt at removing the duplicate cache entry - * for this key. If this fails, it is not catastrophic. The worst-case - * behavior should be that the user has to wait for the short TTL to - * elapse before tweeting succeeds. - */ - def unlock(k: Key): Future[Unit] = - retryLoop(cache.delete(k).unit).onSuccess(_ => event(k, "deleted")) - - /** - * Prevent duplicate tweet creation. - * - * Ensures that no more than one tweet creation for the same key is - * happening at the same time. If `create` fails, then the key will - * be removed from the cache. If it succeeds, then the key will be - * retained. - * - * @throws DuplicateTweetCreation if a tweet has already been - * created by a previous request. The exception has the id of the - * created tweet. - * - * @throws TweetCreationInProgress. See the documentation above. - */ - def apply( - k: Key, - isDark: Boolean, - nullcast: Boolean - )( - create: => Future[PostTweetResult] - ): Future[PostTweetResult] = - if (isDark) { - event(k, "dark_create") - create - } else if (nullcast) { - event(k, "nullcast_create") - create - } else { - createWithLock(k, create).onFailure { - // Another process is creating this same tweet (or has already - // created it) - case TweetCreationInProgress => - event(k, "tweet_creation_in_progress") - case _: DuplicateTweetCreation => - event(k, "tweet_already_created") - case _ => - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetDeletePathHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetDeletePathHandler.docx new file mode 100644 index 000000000..e1a4f0835 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetDeletePathHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetDeletePathHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetDeletePathHandler.scala deleted file mode 100644 index e1052a887..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetDeletePathHandler.scala +++ /dev/null @@ -1,811 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.conversions.DurationOps.RichDuration -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.servo.exception.thriftscala.ClientErrorCause -import com.twitter.servo.util.FutureArrow -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.stitch.NotFound -import com.twitter.timelineservice.thriftscala.PerspectiveResult -import com.twitter.timelineservice.{thriftscala => tls} -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.store._ -import com.twitter.tweetypie.thriftscala._ -import com.twitter.util.Time -import com.twitter.util.Try -import Try._ -import com.twitter.spam.rtf.thriftscala.SafetyLabelType -import com.twitter.tweetypie.backends.TimelineService.GetPerspectives -import com.twitter.tweetypie.util.EditControlUtil -import scala.util.control.NoStackTrace - -case class CascadedDeleteNotAvailable(retweetId: TweetId) extends Exception with NoStackTrace { - override def getMessage: String = - s"""|Cascaded delete tweet failed because tweet $retweetId - |is not present in cache or manhattan.""".stripMargin -} - -object TweetDeletePathHandler { - - type DeleteTweets = - (DeleteTweetsRequest, Boolean) => Future[Seq[DeleteTweetResult]] - - type UnretweetEdits = (Option[EditControl], TweetId, UserId) => Future[Unit] - - /** The information from a deleteTweet request that can be inspected by a deleteTweets validator */ - case class DeleteTweetsContext( - byUserId: Option[UserId], - authenticatedUserId: Option[UserId], - tweetAuthorId: UserId, - users: Map[UserId, User], - isUserErasure: Boolean, - expectedErasureUserId: Option[UserId], - tweetIsBounced: Boolean, - isBounceDelete: Boolean) - - /** Provides reason a tweet deletion was allowed */ - sealed trait DeleteAuthorization { def byUserId: Option[UserId] } - - case class AuthorizedByTweetOwner(userId: UserId) extends DeleteAuthorization { - def byUserId: Option[UserId] = Some(userId) - } - case class AuthorizedByTweetContributor(contributorUserId: UserId) extends DeleteAuthorization { - def byUserId: Option[UserId] = Some(contributorUserId) - } - case class AuthorizedByAdmin(adminUserId: UserId) extends DeleteAuthorization { - def byUserId: Option[UserId] = Some(adminUserId) - } - case object AuthorizedByErasure extends DeleteAuthorization { - def byUserId: None.type = None - } - - // Type for a method that receives all the relevant information about a proposed internal tweet - // deletion and can return Future.exception to cancel the delete due to a validation error or - // return a [[DeleteAuthorization]] specifying the reason the deletion is allowed. - type ValidateDeleteTweets = FutureArrow[DeleteTweetsContext, DeleteAuthorization] - - val userFieldsForDelete: Set[UserField] = - Set(UserField.Account, UserField.Profile, UserField.Roles, UserField.Safety) - - val userQueryOptions: UserQueryOptions = - UserQueryOptions( - userFieldsForDelete, - UserVisibility.All - ) - - // user_agent property originates from the client so truncate to a reasonable length - val MaxUserAgentLength = 1000 - - // Age under which we treat not found tweets in - // cascaded_delete_tweet as a temporary condition (the most likely - // explanation being that the tweet has not yet been - // replicated). Tweets older than this we assume are due to - // *permanently* inconsistent data, either spurious edges in tflock or - // tweets that are not loadable from Manhattan. - val MaxCascadedDeleteTweetTemporaryInconsistencyAge: Duration = - 10.minutes -} - -trait TweetDeletePathHandler { - import TweetDeletePathHandler.ValidateDeleteTweets - - def cascadedDeleteTweet(request: CascadedDeleteTweetRequest): Future[Unit] - - def deleteTweets( - request: DeleteTweetsRequest, - isUnretweetEdits: Boolean = false, - ): Future[Seq[DeleteTweetResult]] - - def internalDeleteTweets( - request: DeleteTweetsRequest, - byUserId: Option[UserId], - authenticatedUserId: Option[UserId], - validate: ValidateDeleteTweets, - isUnretweetEdits: Boolean = false - ): Future[Seq[DeleteTweetResult]] - - def unretweetEdits( - optEditControl: Option[EditControl], - excludedTweetId: TweetId, - byUserId: UserId - ): Future[Unit] -} - -/** - * Implementation of TweetDeletePathHandler - */ -class DefaultTweetDeletePathHandler( - stats: StatsReceiver, - tweetResultRepo: TweetResultRepository.Type, - userRepo: UserRepository.Optional, - stratoSafetyLabelsRepo: StratoSafetyLabelsRepository.Type, - lastQuoteOfQuoterRepo: LastQuoteOfQuoterRepository.Type, - tweetStore: TotalTweetStore, - getPerspectives: GetPerspectives) - extends TweetDeletePathHandler { - import TweetDeletePathHandler._ - - val tweetRepo: TweetRepository.Type = TweetRepository.fromTweetResult(tweetResultRepo) - - // attempt to delete tweets was made by someone other than the tweet owner or an admin user - object DeleteTweetsPermissionException extends Exception with NoStackTrace - object ExpectedUserIdMismatchException extends Exception with NoStackTrace - - private[this] val log = Logger("com.twitter.tweetypie.store.TweetDeletions") - - private[this] val cascadeEditDelete = stats.scope("cascade_edit_delete") - private[this] val cascadeEditDeletesEnqueued = cascadeEditDelete.counter("enqueued") - private[this] val cascadeEditDeleteTweets = cascadeEditDelete.counter("tweets") - private[this] val cascadeEditDeleteFailures = cascadeEditDelete.counter("failures") - - private[this] val cascadedDeleteTweet = stats.scope("cascaded_delete_tweet") - private[this] val cascadedDeleteTweetFailures = cascadedDeleteTweet.counter("failures") - private[this] val cascadedDeleteTweetSourceMatch = cascadedDeleteTweet.counter("source_match") - private[this] val cascadedDeleteTweetSourceMismatch = - cascadedDeleteTweet.counter("source_mismatch") - private[this] val cascadedDeleteTweetTweetNotFound = - cascadedDeleteTweet.counter("tweet_not_found") - private[this] val cascadedDeleteTweetTweetNotFoundAge = - cascadedDeleteTweet.stat("tweet_not_found_age") - private[this] val cascadedDeleteTweetUserNotFound = cascadedDeleteTweet.counter("user_not_found") - - private[this] val deleteTweets = stats.scope("delete_tweets") - private[this] val deleteTweetsAuth = deleteTweets.scope("per_tweet_auth") - private[this] val deleteTweetsAuthAttempts = deleteTweetsAuth.counter("attempts") - private[this] val deleteTweetsAuthFailures = deleteTweetsAuth.counter("failures") - private[this] val deleteTweetsAuthSuccessAdmin = deleteTweetsAuth.counter("success_admin") - private[this] val deleteTweetsAuthSuccessByUser = deleteTweetsAuth.counter("success_by_user") - private[this] val deleteTweetsTweets = deleteTweets.counter("tweets") - private[this] val deleteTweetsFailures = deleteTweets.counter("failures") - private[this] val deleteTweetsTweetNotFound = deleteTweets.counter("tweet_not_found") - private[this] val deleteTweetsUserNotFound = deleteTweets.counter("user_not_found") - private[this] val userIdMismatchInTweetDelete = - deleteTweets.counter("expected_actual_user_id_mismatch") - private[this] val bounceDeleteFlagNotSet = - deleteTweets.counter("bounce_delete_flag_not_set") - - private[this] def getUser(userId: UserId): Future[Option[User]] = - Stitch.run(userRepo(UserKey(userId), userQueryOptions)) - - private[this] def getUsersForDeleteTweets(userIds: Seq[UserId]): Future[Map[UserId, User]] = - Stitch.run( - Stitch - .traverse(userIds) { userId => - userRepo(UserKey(userId), userQueryOptions).map { - case Some(u) => Some(userId -> u) - case None => deleteTweetsUserNotFound.incr(); None - } - } - .map(_.flatten.toMap) - ) - - private[this] def getTweet(tweetId: TweetId): Future[Tweet] = - Stitch.run(tweetRepo(tweetId, WritePathQueryOptions.deleteTweetsWithoutEditControl)) - - private[this] def getSingleDeletedTweet( - id: TweetId, - isCascadedEditTweetDeletion: Boolean = false - ): Stitch[Option[TweetData]] = { - val opts = if (isCascadedEditTweetDeletion) { - // Disable edit control hydration if this is cascade delete of edits. - // When edit control is hydrated, the tweet will actually be considered already deleted. - WritePathQueryOptions.deleteTweetsWithoutEditControl - } else { - WritePathQueryOptions.deleteTweets - } - tweetResultRepo(id, opts) - .map(_.value) - .liftToOption { - // We treat the request the same whether the tweet never - // existed or is in one of the already-deleted states by - // just filtering out those tweets. Any tweets that we - // return should be deleted. If the tweet has been - // bounce-deleted, we never want to soft-delete it, and - // vice versa. - case NotFound | FilteredState.Unavailable.TweetDeleted | - FilteredState.Unavailable.BounceDeleted => - true - } - } - - private[this] def getTweetsForDeleteTweets( - ids: Seq[TweetId], - isCascadedEditTweetDeletion: Boolean - ): Future[Map[TweetId, TweetData]] = - Stitch - .run { - Stitch.traverse(ids) { id => - getSingleDeletedTweet(id, isCascadedEditTweetDeletion) - .map { - // When deleting a tweet that has been edited, we want to instead delete the initial version. - // Because the initial tweet will be hydrated in every request, if it is deleted, later - // revisions will be hidden, and cleaned up asynchronously by TP Daemons - - // However, we don't need to do a second lookup if it's already the original tweet - // or if we're doing a cascading edit tweet delete (deleting the entire tweet history) - case Some(tweetData) - if EditControlUtil.isInitialTweet(tweetData.tweet) || - isCascadedEditTweetDeletion => - Stitch.value(Some(tweetData)) - case Some(tweetData) => - getSingleDeletedTweet(EditControlUtil.getInitialTweetId(tweetData.tweet)) - case None => - Stitch.value(None) - // We need to preserve the input tweetId, and the initial TweetData - }.flatten.map(tweetData => (id, tweetData)) - } - } - .map(_.collect { case (tweetId, Some(tweetData)) => (tweetId, tweetData) }.toMap) - - private[this] def getStratoBounceStatuses( - ids: Seq[Long], - isUserErasure: Boolean, - isCascadedEditedTweetDeletion: Boolean - ): Future[Map[TweetId, Boolean]] = { - // Don't load bounce label for user erasure tweet deletion. - // User Erasure deletions cause unnecessary spikes of traffic - // to Strato when we read the bounce label that we don't use. - - // We also want to always delete a bounced tweet if the rest of the - // edit chain is being deleted in a cascaded edit tweet delete - if (isUserErasure || isCascadedEditedTweetDeletion) { - Future.value(ids.map(id => id -> false).toMap) - } else { - Stitch.run( - Stitch - .traverse(ids) { id => - stratoSafetyLabelsRepo(id, SafetyLabelType.Bounce).map { label => - id -> label.isDefined - } - } - .map(_.toMap) - ) - } - } - - /** A suspended/deactivated user can't delete tweets */ - private[this] def userNotSuspendedOrDeactivated(user: User): Try[User] = - user.safety match { - case None => Throw(UpstreamFailure.UserSafetyEmptyException) - case Some(safety) if safety.deactivated => - Throw( - AccessDenied( - s"User deactivated userId: ${user.id}", - errorCause = Some(AccessDeniedCause.UserDeactivated) - ) - ) - case Some(safety) if safety.suspended => - Throw( - AccessDenied( - s"User suspended userId: ${user.id}", - errorCause = Some(AccessDeniedCause.UserSuspended) - ) - ) - case _ => Return(user) - } - - /** - * Ensure that byUser has permission to delete tweet either by virtue of owning the tweet or being - * an admin user. Returns the reason as a DeleteAuthorization or else throws an Exception if not - * authorized. - */ - private[this] def userAuthorizedToDeleteTweet( - byUser: User, - optAuthenticatedUserId: Option[UserId], - tweetAuthorId: UserId - ): Try[DeleteAuthorization] = { - - def hasAdminPrivilege = - byUser.roles.exists(_.rights.contains("delete_user_tweets")) - - deleteTweetsAuthAttempts.incr() - if (byUser.id == tweetAuthorId) { - deleteTweetsAuthSuccessByUser.incr() - optAuthenticatedUserId match { - case Some(uid) => - Return(AuthorizedByTweetContributor(uid)) - case None => - Return(AuthorizedByTweetOwner(byUser.id)) - } - } else if (optAuthenticatedUserId.isEmpty && hasAdminPrivilege) { // contributor may not assume admin role - deleteTweetsAuthSuccessAdmin.incr() - Return(AuthorizedByAdmin(byUser.id)) - } else { - deleteTweetsAuthFailures.incr() - Throw(DeleteTweetsPermissionException) - } - } - - /** - * expected user id is the id provided on the DeleteTweetsRequest that the indicates which user - * owns the tweets they want to delete. The actualUserId is the actual userId on the tweet we are about to delete. - * we check to ensure they are the same as a safety check against accidental deletion of tweets either from user mistakes - * or from corrupted data (e.g bad tflock edges) - */ - private[this] def expectedUserIdMatchesActualUserId( - expectedUserId: UserId, - actualUserId: UserId - ): Try[Unit] = - if (expectedUserId == actualUserId) { - Return.Unit - } else { - userIdMismatchInTweetDelete.incr() - Throw(ExpectedUserIdMismatchException) - } - - /** - * Validation for the normal public tweet delete case, the user must be found and must - * not be suspended or deactivated. - */ - val validateTweetsForPublicDelete: ValidateDeleteTweets = FutureArrow { - ctx: DeleteTweetsContext => - Future.const( - for { - - // byUserId must be present - byUserId <- ctx.byUserId.orThrow( - ClientError(ClientErrorCause.BadRequest, "Missing byUserId") - ) - - // the byUser must be found - byUserOpt = ctx.users.get(byUserId) - byUser <- byUserOpt.orThrow( - ClientError(ClientErrorCause.BadRequest, s"User $byUserId not found") - ) - - _ <- userNotSuspendedOrDeactivated(byUser) - - _ <- validateBounceConditions( - ctx.tweetIsBounced, - ctx.isBounceDelete - ) - - // if there's a contributor, make sure the user is found and not suspended or deactivated - _ <- - ctx.authenticatedUserId - .map { uid => - ctx.users.get(uid) match { - case None => - Throw(ClientError(ClientErrorCause.BadRequest, s"Contributor $uid not found")) - case Some(authUser) => - userNotSuspendedOrDeactivated(authUser) - } - } - .getOrElse(Return.Unit) - - // if the expected user id is present, make sure it matches the user id on the tweet - _ <- - ctx.expectedErasureUserId - .map { expectedUserId => - expectedUserIdMatchesActualUserId(expectedUserId, ctx.tweetAuthorId) - } - .getOrElse(Return.Unit) - - // User must own the tweet or be an admin - deleteAuth <- userAuthorizedToDeleteTweet( - byUser, - ctx.authenticatedUserId, - ctx.tweetAuthorId - ) - } yield deleteAuth - ) - } - - private def validateBounceConditions( - tweetIsBounced: Boolean, - isBounceDelete: Boolean - ): Try[Unit] = { - if (tweetIsBounced && !isBounceDelete) { - bounceDeleteFlagNotSet.incr() - Throw(ClientError(ClientErrorCause.BadRequest, "Cannot normal delete a Bounced Tweet")) - } else { - Return.Unit - } - } - - /** - * Validation for the user erasure case. User may be missing. - */ - val validateTweetsForUserErasureDaemon: ValidateDeleteTweets = FutureArrow { - ctx: DeleteTweetsContext => - Future - .const( - for { - expectedUserId <- ctx.expectedErasureUserId.orThrow( - ClientError( - ClientErrorCause.BadRequest, - "expectedUserId is required for DeleteTweetRequests" - ) - ) - - // It's critical to always check that the userId on the tweet we want to delete matches the - // userId on the erasure request. This prevents us from accidentally deleting tweets not owned by the - // erased user, even if tflock serves us bad data. - validationResult <- expectedUserIdMatchesActualUserId(expectedUserId, ctx.tweetAuthorId) - } yield validationResult - ) - .map(_ => AuthorizedByErasure) - } - - /** - * Fill in missing values of AuditDeleteTweet with values from TwitterContext. - */ - def enrichMissingFromTwitterContext(orig: AuditDeleteTweet): AuditDeleteTweet = { - val viewer = TwitterContext() - orig.copy( - host = orig.host.orElse(viewer.flatMap(_.auditIp)), - clientApplicationId = orig.clientApplicationId.orElse(viewer.flatMap(_.clientApplicationId)), - userAgent = orig.userAgent.orElse(viewer.flatMap(_.userAgent)).map(_.take(MaxUserAgentLength)) - ) - } - - /** - * core delete tweets implementation. - * - * The [[deleteTweets]] method wraps this method and provides validation required - * for a public endpoint. - */ - override def internalDeleteTweets( - request: DeleteTweetsRequest, - byUserId: Option[UserId], - authenticatedUserId: Option[UserId], - validate: ValidateDeleteTweets, - isUnretweetEdits: Boolean = false - ): Future[Seq[DeleteTweetResult]] = { - - val auditDeleteTweet = - enrichMissingFromTwitterContext(request.auditPassthrough.getOrElse(AuditDeleteTweet())) - deleteTweetsTweets.incr(request.tweetIds.size) - for { - tweetDataMap <- getTweetsForDeleteTweets( - request.tweetIds, - request.cascadedEditedTweetDeletion.getOrElse(false) - ) - - userIds: Seq[UserId] = (tweetDataMap.values.map { td => - getUserId(td.tweet) - } ++ byUserId ++ authenticatedUserId).toSeq.distinct - - users <- getUsersForDeleteTweets(userIds) - - stratoBounceStatuses <- getStratoBounceStatuses( - tweetDataMap.keys.toSeq, - request.isUserErasure, - request.cascadedEditedTweetDeletion.getOrElse(false)) - - results <- Future.collect { - request.tweetIds.map { tweetId => - tweetDataMap.get(tweetId) match { - // already deleted, so nothing to do - case None => - deleteTweetsTweetNotFound.incr() - Future.value(DeleteTweetResult(tweetId, TweetDeleteState.Ok)) - case Some(tweetData) => - val tweet: Tweet = tweetData.tweet - val tweetIsBounced = stratoBounceStatuses(tweetId) - val optSourceTweet: Option[Tweet] = tweetData.sourceTweetResult.map(_.value.tweet) - - val validation: Future[(Boolean, DeleteAuthorization)] = for { - isLastQuoteOfQuoter <- isFinalQuoteOfQuoter(tweet) - deleteAuth <- validate( - DeleteTweetsContext( - byUserId = byUserId, - authenticatedUserId = authenticatedUserId, - tweetAuthorId = getUserId(tweet), - users = users, - isUserErasure = request.isUserErasure, - expectedErasureUserId = request.expectedUserId, - tweetIsBounced = tweetIsBounced, - isBounceDelete = request.isBounceDelete - ) - ) - _ <- optSourceTweet match { - case Some(sourceTweet) if !isUnretweetEdits => - // If this is a retweet and this deletion was not triggered by - // unretweetEdits, unretweet edits of the source Tweet - // before deleting the retweet. - // - // deleteAuth will always contain a byUserId except for erasure deletion, - // in which case the retweets will be deleted individually. - deleteAuth.byUserId match { - case Some(userId) => - unretweetEdits(sourceTweet.editControl, sourceTweet.id, userId) - case None => Future.Unit - } - case _ => Future.Unit - } - } yield { - (isLastQuoteOfQuoter, deleteAuth) - } - - validation - .flatMap { - case (isLastQuoteOfQuoter: Boolean, deleteAuth: DeleteAuthorization) => - val isAdminDelete = deleteAuth match { - case AuthorizedByAdmin(_) => true - case _ => false - } - - val event = - DeleteTweet.Event( - tweet = tweet, - timestamp = Time.now, - user = users.get(getUserId(tweet)), - byUserId = deleteAuth.byUserId, - auditPassthrough = Some(auditDeleteTweet), - isUserErasure = request.isUserErasure, - isBounceDelete = request.isBounceDelete && tweetIsBounced, - isLastQuoteOfQuoter = isLastQuoteOfQuoter, - isAdminDelete = isAdminDelete - ) - val numberOfEdits: Int = tweet.editControl - .collect { - case EditControl.Initial(initial) => - initial.editTweetIds.count(_ != tweet.id) - } - .getOrElse(0) - cascadeEditDeletesEnqueued.incr(numberOfEdits) - tweetStore - .deleteTweet(event) - .map(_ => DeleteTweetResult(tweetId, TweetDeleteState.Ok)) - } - .onFailure { _ => - deleteTweetsFailures.incr() - } - .handle { - case ExpectedUserIdMismatchException => - DeleteTweetResult(tweetId, TweetDeleteState.ExpectedUserIdMismatch) - case DeleteTweetsPermissionException => - DeleteTweetResult(tweetId, TweetDeleteState.PermissionError) - } - } - } - } - } yield results - } - - private def isFinalQuoteOfQuoter(tweet: Tweet): Future[Boolean] = { - tweet.quotedTweet match { - case Some(qt) => - Stitch.run { - lastQuoteOfQuoterRepo - .apply(qt.tweetId, getUserId(tweet)) - .liftToTry - .map(_.getOrElse(false)) - } - case None => Future(false) - } - } - - /** - * Validations for the public deleteTweets endpoint. - * - ensures that the byUserId user can be found and is in the correct user state - * - ensures that the tweet is being deleted by the tweet's owner, or by an admin - * If there is a validation error, a future.exception is returned - * - * If the delete request is part of a user erasure, validations are relaxed (the User is allowed to be missing). - */ - val deleteTweetsValidator: ValidateDeleteTweets = - FutureArrow { context => - if (context.isUserErasure) { - validateTweetsForUserErasureDaemon(context) - } else { - validateTweetsForPublicDelete(context) - } - } - - override def deleteTweets( - request: DeleteTweetsRequest, - isUnretweetEdits: Boolean = false, - ): Future[Seq[DeleteTweetResult]] = { - - // For comparison testing we only want to compare the DeleteTweetsRequests that are generated - // in DeleteTweets path and not the call that comes from the Unretweet path - val context = TwitterContext() - internalDeleteTweets( - request, - byUserId = request.byUserId.orElse(context.flatMap(_.userId)), - context.flatMap(_.authenticatedUserId), - deleteTweetsValidator, - isUnretweetEdits - ) - } - - // Cascade delete tweet is the logic for removing tweets that are detached - // from their dependency which has been deleted. They are already filtered - // out from serving, so this operation reconciles storage with the view - // presented by Tweetypie. - // This RPC call is delegated from daemons or batch jobs. Currently there - // are two use-cases when this call is issued: - // * Deleting detached retweets after the source tweet was deleted. - // This is done through RetweetsDeletion daemon and the - // CleanupDetachedRetweets job. - // * Deleting edits of an initial tweet that has been deleted. - // This is done by CascadedEditedTweetDelete daemon. - // Note that, when serving the original delete request for an edit, - // the initial tweet is only deleted, which makes all edits hidden. - override def cascadedDeleteTweet(request: CascadedDeleteTweetRequest): Future[Unit] = { - val contextViewer = TwitterContext() - getTweet(request.tweetId) - .transform { - case Throw( - FilteredState.Unavailable.TweetDeleted | FilteredState.Unavailable.BounceDeleted) => - // The retweet or edit was already deleted via some other mechanism - Future.Unit - - case Throw(NotFound) => - cascadedDeleteTweetTweetNotFound.incr() - val recentlyCreated = - if (SnowflakeId.isSnowflakeId(request.tweetId)) { - val age = Time.now - SnowflakeId(request.tweetId).time - cascadedDeleteTweetTweetNotFoundAge.add(age.inMilliseconds) - age < MaxCascadedDeleteTweetTemporaryInconsistencyAge - } else { - false - } - - if (recentlyCreated) { - // Treat the NotFound as a temporary condition, most - // likely due to replication lag. - Future.exception(CascadedDeleteNotAvailable(request.tweetId)) - } else { - // Treat the NotFound as a permanent inconsistenty, either - // spurious edges in tflock or invalid data in Manhattan. This - // was happening a few times an hour during the time that we - // were not treating it specially. For now, we will just log that - // it happened, but in the longer term, it would be good - // to collect this data and repair the corruption. - log.warn( - Seq( - "cascaded_delete_tweet_old_not_found", - request.tweetId, - request.cascadedFromTweetId - ).mkString("\t") - ) - Future.Done - } - - // Any other FilteredStates should not be thrown because of - // the options that we used to load the tweet, so we will just - // let them bubble up as an internal server error - case Throw(other) => - Future.exception(other) - - case Return(tweet) => - Future - .join( - isFinalQuoteOfQuoter(tweet), - getUser(getUserId(tweet)) - ) - .flatMap { - case (isLastQuoteOfQuoter, user) => - if (user.isEmpty) { - cascadedDeleteTweetUserNotFound.incr() - } - val tweetSourceId = getShare(tweet).map(_.sourceStatusId) - val initialEditId = tweet.editControl.collect { - case EditControl.Edit(edit) => edit.initialTweetId - } - if (initialEditId.contains(request.cascadedFromTweetId)) { - cascadeEditDeleteTweets.incr() - } - if (tweetSourceId.contains(request.cascadedFromTweetId) - || initialEditId.contains(request.cascadedFromTweetId)) { - cascadedDeleteTweetSourceMatch.incr() - val deleteEvent = - DeleteTweet.Event( - tweet = tweet, - timestamp = Time.now, - user = user, - byUserId = contextViewer.flatMap(_.userId), - cascadedFromTweetId = Some(request.cascadedFromTweetId), - auditPassthrough = request.auditPassthrough, - isUserErasure = false, - // cascaded deletes of retweets or edits have not been through a bouncer flow, - // so are not considered to be "bounce deleted". - isBounceDelete = false, - isLastQuoteOfQuoter = isLastQuoteOfQuoter, - isAdminDelete = false - ) - tweetStore - .deleteTweet(deleteEvent) - .onFailure { _ => - if (initialEditId.contains(request.cascadedFromTweetId)) { - cascadeEditDeleteFailures.incr() - } - } - } else { - cascadedDeleteTweetSourceMismatch.incr() - log.warn( - Seq( - "cascaded_from_tweet_id_source_mismatch", - request.tweetId, - request.cascadedFromTweetId, - tweetSourceId.orElse(initialEditId).getOrElse("-") - ).mkString("\t") - ) - Future.Done - } - } - } - .onFailure(_ => cascadedDeleteTweetFailures.incr()) - } - - // Given a list of edit Tweet ids and a user id, find the retweet ids of those edit ids from the given user - private def editTweetIdRetweetsFromUser( - editTweetIds: Seq[TweetId], - byUserId: UserId - ): Future[Seq[TweetId]] = { - if (editTweetIds.isEmpty) { - Future.value(Seq()) - } else { - getPerspectives( - Seq(tls.PerspectiveQuery(byUserId, editTweetIds)) - ).map { res: Seq[PerspectiveResult] => - res.headOption.toSeq - .flatMap(_.perspectives.flatMap(_.retweetId)) - } - } - } - - /* This function is called from three places - - * 1. When Tweetypie gets a request to retweet the latest version of an edit chain, all the - * previous revisons should be unretweeted. - * i.e. On Retweet of the latest tweet - unretweets all the previous revisions for this user. - * - create A - * - retweet A'(retweet of A) - * - create edit B(edit of A) - * - retweet B' => Deletes A' - * - * 2. When Tweetypie gets an unretweet request for a source tweet that is an edit tweet, all - * the versions of the edit chain is retweeted. - * i.e. On unretweet of any version in the edit chain - unretweets all the revisions for this user - * - create A - * - retweet A' - * - create B - * - unretweet B => Deletes A' (& also any B' if it existed) - * - * 3. When Tweetypie gets a delete request for a retweet, say A1. & if A happens to the source - * tweet for A1 & if A is an edit tweet, then the entire edit chain should be unretweeted & not - * A. i.e. On delete of a retweet - unretweet all the revisions for this user. - * - create A - * - retweet A' - * - create B - * - delete A' => Deletes A' (& also any B' if it existed) - * - * The following function has two failure scenarios - - * i. when it fails to get perspectives of any of the edit tweets. - * ii. the deletion of any of the retweets of these edits fail. - * - * In either of this scenario, we fail the entire request & the error bubbles up to the top. - * Note: The above unretweet of edits only happens for the current user. - * In normal circumstances, a maximum of one Tweet in the edit chain will have been retweeted, - * but we don't know which one it was. Additionally, there may be circumstances where - * unretweet failed, and we end up with multiple versions retweeted. For these reasons, - * we always unretweet all the revisions (except for `excludedTweetId`). - * This is a no-op if none of these versions have been retweeted. - * */ - override def unretweetEdits( - optEditControl: Option[EditControl], - excludedTweetId: TweetId, - byUserId: UserId - ): Future[Unit] = { - - val editTweetIds: Seq[TweetId] = - EditControlUtil.getEditTweetIds(optEditControl).get().filter(_ != excludedTweetId) - - (editTweetIdRetweetsFromUser(editTweetIds, byUserId).flatMap { tweetIds => - if (tweetIds.nonEmpty) { - deleteTweets( - DeleteTweetsRequest(tweetIds = tweetIds, byUserId = Some(byUserId)), - isUnretweetEdits = true - ) - } else { - Future.Nil - } - }).unit - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetWriteValidator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetWriteValidator.docx new file mode 100644 index 000000000..b5840bd88 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetWriteValidator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetWriteValidator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetWriteValidator.scala deleted file mode 100644 index 2164b8a84..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/TweetWriteValidator.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.tweetypie.handler - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.TweetCreateFailure -import com.twitter.tweetypie.repository.ConversationControlRepository -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.thriftscala.ExclusiveTweetControl -import com.twitter.tweetypie.thriftscala.ExclusiveTweetControlOptions -import com.twitter.tweetypie.thriftscala.QuotedTweet -import com.twitter.tweetypie.thriftscala.TrustedFriendsControl -import com.twitter.tweetypie.thriftscala.TrustedFriendsControlOptions -import com.twitter.tweetypie.thriftscala.TweetCreateState -import com.twitter.tweetypie.FutureEffect -import com.twitter.tweetypie.Gate -import com.twitter.tweetypie.TweetId -import com.twitter.tweetypie.UserId -import com.twitter.tweetypie.thriftscala.EditControl -import com.twitter.tweetypie.thriftscala.EditOptions -import com.twitter.visibility.writer.interfaces.tweets.TweetWriteEnforcementLibrary -import com.twitter.visibility.writer.interfaces.tweets.TweetWriteEnforcementRequest -import com.twitter.visibility.writer.models.ActorContext -import com.twitter.visibility.writer.Allow -import com.twitter.visibility.writer.Deny -import com.twitter.visibility.writer.DenyExclusiveTweetReply -import com.twitter.visibility.writer.DenyStaleTweetQuoteTweet -import com.twitter.visibility.writer.DenyStaleTweetReply -import com.twitter.visibility.writer.DenySuperFollowsCreate -import com.twitter.visibility.writer.DenyTrustedFriendsCreate -import com.twitter.visibility.writer.DenyTrustedFriendsQuoteTweet -import com.twitter.visibility.writer.DenyTrustedFriendsReply - -object TweetWriteValidator { - case class Request( - conversationId: Option[TweetId], - userId: UserId, - exclusiveTweetControlOptions: Option[ExclusiveTweetControlOptions], - replyToExclusiveTweetControl: Option[ExclusiveTweetControl], - trustedFriendsControlOptions: Option[TrustedFriendsControlOptions], - inReplyToTrustedFriendsControl: Option[TrustedFriendsControl], - quotedTweetOpt: Option[QuotedTweet], - inReplyToTweetId: Option[TweetId], - inReplyToEditControl: Option[EditControl], - editOptions: Option[EditOptions]) - - type Type = FutureEffect[Request] - - def apply( - convoCtlRepo: ConversationControlRepository.Type, - tweetWriteEnforcementLibrary: TweetWriteEnforcementLibrary, - enableExclusiveTweetControlValidation: Gate[Unit], - enableTrustedFriendsControlValidation: Gate[Unit], - enableStaleTweetValidation: Gate[Unit] - ): FutureEffect[Request] = - FutureEffect[Request] { request => - // We are creating up an empty TweetQuery.Options here so we can use the default - // CacheControl value and avoid hard coding it here. - val queryOptions = TweetQuery.Options(TweetQuery.Include()) - Stitch.run { - for { - convoCtl <- request.conversationId match { - case Some(convoId) => - convoCtlRepo( - convoId, - queryOptions.cacheControl - ) - case None => - Stitch.value(None) - } - - result <- tweetWriteEnforcementLibrary( - TweetWriteEnforcementRequest( - rootConversationControl = convoCtl, - convoId = request.conversationId, - exclusiveTweetControlOptions = request.exclusiveTweetControlOptions, - replyToExclusiveTweetControl = request.replyToExclusiveTweetControl, - trustedFriendsControlOptions = request.trustedFriendsControlOptions, - inReplyToTrustedFriendsControl = request.inReplyToTrustedFriendsControl, - quotedTweetOpt = request.quotedTweetOpt, - actorContext = ActorContext(request.userId), - inReplyToTweetId = request.inReplyToTweetId, - inReplyToEditControl = request.inReplyToEditControl, - editOptions = request.editOptions - ), - enableExclusiveTweetControlValidation = enableExclusiveTweetControlValidation, - enableTrustedFriendsControlValidation = enableTrustedFriendsControlValidation, - enableStaleTweetValidation = enableStaleTweetValidation - ) - _ <- result match { - case Allow => - Stitch.Done - case Deny => - Stitch.exception(TweetCreateFailure.State(TweetCreateState.ReplyTweetNotAllowed)) - case DenyExclusiveTweetReply => - Stitch.exception( - TweetCreateFailure.State(TweetCreateState.ExclusiveTweetEngagementNotAllowed)) - case DenySuperFollowsCreate => - Stitch.exception( - TweetCreateFailure.State(TweetCreateState.SuperFollowsCreateNotAuthorized)) - case DenyTrustedFriendsReply => - Stitch.exception( - TweetCreateFailure.State(TweetCreateState.TrustedFriendsEngagementNotAllowed)) - case DenyTrustedFriendsCreate => - Stitch.exception( - TweetCreateFailure.State(TweetCreateState.TrustedFriendsCreateNotAllowed)) - case DenyTrustedFriendsQuoteTweet => - Stitch.exception( - TweetCreateFailure.State(TweetCreateState.TrustedFriendsQuoteTweetNotAllowed)) - case DenyStaleTweetReply => - Stitch.exception( - TweetCreateFailure.State(TweetCreateState.StaleTweetEngagementNotAllowed)) - case DenyStaleTweetQuoteTweet => - Stitch.exception( - TweetCreateFailure.State(TweetCreateState.StaleTweetQuoteTweetNotAllowed)) - } - } yield () - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/U13ValidationUtil.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/U13ValidationUtil.docx new file mode 100644 index 000000000..e0db89d95 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/U13ValidationUtil.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/U13ValidationUtil.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/U13ValidationUtil.scala deleted file mode 100644 index 1b4d46de1..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/U13ValidationUtil.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.tweetypie.handler - -import com.twitter.compliance.userconsent.compliance.birthdate.GlobalBirthdateUtil -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.tweetypie.thriftscala.DeletedTweet -import org.joda.time.DateTime - -/* - * As part of GDPR U13 work, we want to block tweets created from when a user - * was < 13 from being restored. - */ - -private[handler] object U13ValidationUtil { - def wasTweetCreatedBeforeUserTurned13(user: User, deletedTweet: DeletedTweet): Boolean = - deletedTweet.createdAtSecs match { - case None => - throw NoCreatedAtTimeException - case Some(createdAtSecs) => - GlobalBirthdateUtil.isUnderSomeAge(13, new DateTime(createdAtSecs * 1000L), user) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UndeleteTweetHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UndeleteTweetHandler.docx new file mode 100644 index 000000000..7d2b3f643 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UndeleteTweetHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UndeleteTweetHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UndeleteTweetHandler.scala deleted file mode 100644 index c24590298..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UndeleteTweetHandler.scala +++ /dev/null @@ -1,215 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.servo.util.FutureArrow -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.FilteredState -import com.twitter.tweetypie.core.TweetHydrationError -import com.twitter.tweetypie.repository.ParentUserIdRepository -import com.twitter.tweetypie.storage.TweetStorageClient.Undelete -import com.twitter.tweetypie.storage.DeleteState -import com.twitter.tweetypie.storage.DeletedTweetResponse -import com.twitter.tweetypie.storage.TweetStorageClient -import com.twitter.tweetypie.store.UndeleteTweet -import com.twitter.tweetypie.thriftscala.UndeleteTweetState.{Success => TweetypieSuccess, _} -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.thriftscala.entities.EntityExtractor -import scala.util.control.NoStackTrace - -trait UndeleteException extends Exception with NoStackTrace - -/** - * Exceptions we return to the user, things that we don't expect to ever happen unless there is a - * problem with the underlying data in Manhattan or a bug in [[com.twitter.tweetypie.storage.TweetStorageClient]] - */ -object NoDeletedAtTimeException extends UndeleteException -object NoCreatedAtTimeException extends UndeleteException -object NoStatusWithSuccessException extends UndeleteException -object NoUserIdWithTweetException extends UndeleteException -object NoDeletedTweetException extends UndeleteException -object SoftDeleteUserIdNotFoundException extends UndeleteException - -/** - * represents a problem that we choose to return to the user as a response state - * rather than as an exception. - */ -case class ResponseException(state: UndeleteTweetState) extends Exception with NoStackTrace { - def toResponse: UndeleteTweetResponse = UndeleteTweetResponse(state = state) -} - -private[this] object SoftDeleteExpiredException extends ResponseException(SoftDeleteExpired) -private[this] object BounceDeleteException extends ResponseException(TweetIsBounceDeleted) -private[this] object SourceTweetNotFoundException extends ResponseException(SourceTweetNotFound) -private[this] object SourceUserNotFoundException extends ResponseException(SourceUserNotFound) -private[this] object TweetExistsException extends ResponseException(TweetAlreadyExists) -private[this] object TweetNotFoundException extends ResponseException(TweetNotFound) -private[this] object U13TweetException extends ResponseException(TweetIsU13Tweet) -private[this] object UserNotFoundException extends ResponseException(UserNotFound) - -/** - * Undelete Notes: - * - * If request.force is set to true, then the undelete will take place even if the undeleted tweet - * is already present in Manhattan. This is useful if a tweet was recently restored to the backend, - * but the async actions portion of the undelete failed and you want to retry them. - * - * Before undeleting the tweet we check if it's a retweet, in which case we require that the sourceTweet - * and sourceUser exist. - * - * Tweets can only be undeleted for N days where N is the number of days before tweets marked with - * the soft_delete_state flag are deleted permanently by the cleanup job - * - */ -object UndeleteTweetHandler { - - type Type = FutureArrow[UndeleteTweetRequest, UndeleteTweetResponse] - - /** Extract an optional value inside a future or throw if it's missing. */ - def required[T](option: Future[Option[T]], ex: => Exception): Future[T] = - option.flatMap { - case None => Future.exception(ex) - case Some(i) => Future.value(i) - } - - def apply( - undelete: TweetStorageClient.Undelete, - tweetExists: FutureArrow[TweetId, Boolean], - getUser: FutureArrow[UserId, Option[User]], - getDeletedTweets: TweetStorageClient.GetDeletedTweets, - parentUserIdRepo: ParentUserIdRepository.Type, - save: FutureArrow[UndeleteTweet.Event, Tweet] - ): Type = { - - def getParentUserId(tweet: Tweet): Future[Option[UserId]] = - Stitch.run { - parentUserIdRepo(tweet) - .handle { - case ParentUserIdRepository.ParentTweetNotFound(id) => None - } - } - - val entityExtractor = EntityExtractor.mutationAll.endo - - val getDeletedTweet: Long => Future[DeletedTweetResponse] = - id => Stitch.run(getDeletedTweets(Seq(id)).map(_.head)) - - def getRequiredUser(userId: Option[UserId]): Future[User] = - userId match { - case None => Future.exception(SoftDeleteUserIdNotFoundException) - case Some(id) => required(getUser(id), UserNotFoundException) - } - - def getValidatedDeletedTweet( - tweetId: TweetId, - allowNotDeleted: Boolean - ): Future[DeletedTweet] = { - import DeleteState._ - val deletedTweet = getDeletedTweet(tweetId).map { response => - response.deleteState match { - case SoftDeleted => response.tweet - // BounceDeleted tweets violated Twitter Rules and may not be undeleted - case BounceDeleted => throw BounceDeleteException - case HardDeleted => throw SoftDeleteExpiredException - case NotDeleted => if (allowNotDeleted) response.tweet else throw TweetExistsException - case NotFound => throw TweetNotFoundException - } - } - - required(deletedTweet, NoDeletedTweetException) - } - - /** - * Fetch the source tweet's user for a deleted share - */ - def getSourceUser(share: Option[DeletedTweetShare]): Future[Option[User]] = - share match { - case None => Future.value(None) - case Some(s) => required(getUser(s.sourceUserId), SourceUserNotFoundException).map(Some(_)) - } - - /** - * Ensure that the undelete response contains all the required information to continue with - * the tweetypie undelete. - */ - def validateUndeleteResponse(response: Undelete.Response, force: Boolean): Future[Tweet] = - Future { - (response.code, response.tweet) match { - case (Undelete.UndeleteResponseCode.NotCreated, _) => throw TweetNotFoundException - case (Undelete.UndeleteResponseCode.BackupNotFound, _) => throw SoftDeleteExpiredException - case (Undelete.UndeleteResponseCode.Success, None) => throw NoStatusWithSuccessException - case (Undelete.UndeleteResponseCode.Success, Some(tweet)) => - // archivedAtMillis is required on the response unless force is present - // or the tweet is a retweet. retweets have no favs or retweets to clean up - // of their own so the original deleted at time is not needed - if (response.archivedAtMillis.isEmpty && !force && !isRetweet(tweet)) - throw NoDeletedAtTimeException - else - tweet - case (code, _) => throw new Exception(s"Unknown UndeleteResponseCode $code") - } - } - - def enforceU13Compliance(user: User, deletedTweet: DeletedTweet): Future[Unit] = - Future.when(U13ValidationUtil.wasTweetCreatedBeforeUserTurned13(user, deletedTweet)) { - throw U13TweetException - } - - /** - * Fetch required data and perform before/after validations for undelete. - * If everything looks good with the undelete, kick off the tweetypie undelete - * event. - */ - FutureArrow { request => - val hydrationOptions = request.hydrationOptions.getOrElse(WritePathHydrationOptions()) - val force = request.force.getOrElse(false) - val tweetId = request.tweetId - - (for { - // we must be able to query the tweet from the soft delete table - deletedTweet <- getValidatedDeletedTweet(tweetId, allowNotDeleted = force) - - // we always require the user - user <- getRequiredUser(deletedTweet.userId) - - // Make sure we're not restoring any u13 tweets. - () <- enforceU13Compliance(user, deletedTweet) - - // if a retweet, then sourceUser is required; sourceTweet will be hydrated in save() - sourceUser <- getSourceUser(deletedTweet.share) - - // validations passed, perform the undelete. - undeleteResponse <- Stitch.run(undelete(tweetId)) - - // validate the response - tweet <- validateUndeleteResponse(undeleteResponse, force) - - // Extract entities from tweet text - tweetWithEntities = entityExtractor(tweet) - - // If a retweet, get user id of parent retweet - parentUserId <- getParentUserId(tweet) - - // undeletion was successful, hydrate the tweet and - // kick off tweetypie async undelete actions - hydratedTweet <- save( - UndeleteTweet.Event( - tweet = tweetWithEntities, - user = user, - timestamp = Time.now, - hydrateOptions = hydrationOptions, - deletedAt = undeleteResponse.archivedAtMillis.map(Time.fromMilliseconds), - sourceUser = sourceUser, - parentUserId = parentUserId - ) - ) - } yield { - UndeleteTweetResponse(TweetypieSuccess, Some(hydratedTweet)) - }).handle { - case TweetHydrationError(_, Some(FilteredState.Unavailable.SourceTweetNotFound(_))) => - SourceTweetNotFoundException.toResponse - case ex: ResponseException => - ex.toResponse - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UnretweetHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UnretweetHandler.docx new file mode 100644 index 000000000..9b440c454 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UnretweetHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UnretweetHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UnretweetHandler.scala deleted file mode 100644 index 4747ff0ea..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UnretweetHandler.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.Future -import com.twitter.tweetypie.core.FilteredState -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.repository.TweetRepository -import com.twitter.tweetypie.thriftscala._ -import com.twitter.timelineservice.{thriftscala => tls} -import com.twitter.tweetypie.backends.TimelineService.GetPerspectives - -object UnretweetHandler { - - type Type = UnretweetRequest => Future[UnretweetResult] - - def apply( - deleteTweets: TweetDeletePathHandler.DeleteTweets, - getPerspectives: GetPerspectives, - unretweetEdits: TweetDeletePathHandler.UnretweetEdits, - tweetRepo: TweetRepository.Type, - ): Type = { request: UnretweetRequest => - val handleEdits = getSourceTweet(request.sourceTweetId, tweetRepo).liftToTry.flatMap { - case Return(sourceTweet) => - // If we're able to fetch the source Tweet, unretweet all its other versions - unretweetEdits(sourceTweet.editControl, request.sourceTweetId, request.userId) - case Throw(_) => Future.Done - } - - handleEdits.flatMap(_ => unretweetSourceTweet(request, deleteTweets, getPerspectives)) - } - - def unretweetSourceTweet( - request: UnretweetRequest, - deleteTweets: TweetDeletePathHandler.DeleteTweets, - getPerspectives: GetPerspectives, - ): Future[UnretweetResult] = - getPerspectives( - Seq(tls.PerspectiveQuery(request.userId, Seq(request.sourceTweetId))) - ).map { results => results.head.perspectives.headOption.flatMap(_.retweetId) } - .flatMap { - case Some(id) => - deleteTweets( - DeleteTweetsRequest(tweetIds = Seq(id), byUserId = Some(request.userId)), - false - ).map(_.head).map { deleteTweetResult => - UnretweetResult(Some(deleteTweetResult.tweetId), deleteTweetResult.state) - } - case None => Future.value(UnretweetResult(None, TweetDeleteState.Ok)) - } - - def getSourceTweet( - sourceTweetId: TweetId, - tweetRepo: TweetRepository.Type - ): Future[Tweet] = { - val options: TweetQuery.Options = TweetQuery - .Options(include = TweetQuery.Include(tweetFields = Set(Tweet.EditControlField.id))) - - Stitch.run { - tweetRepo(sourceTweetId, options).rescue { - case _: FilteredState => Stitch.NotFound - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UpdatePossiblySensitiveTweetHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UpdatePossiblySensitiveTweetHandler.docx new file mode 100644 index 000000000..8408c576f Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UpdatePossiblySensitiveTweetHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UpdatePossiblySensitiveTweetHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UpdatePossiblySensitiveTweetHandler.scala deleted file mode 100644 index 875edb63c..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UpdatePossiblySensitiveTweetHandler.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.tweetypie.store.UpdatePossiblySensitiveTweet -import com.twitter.tweetypie.thriftscala.UpdatePossiblySensitiveTweetRequest -import com.twitter.tweetypie.util.TweetLenses - -object UpdatePossiblySensitiveTweetHandler { - type Type = FutureArrow[UpdatePossiblySensitiveTweetRequest, Unit] - - def apply( - tweetGetter: FutureArrow[TweetId, Tweet], - userGetter: FutureArrow[UserId, User], - updatePossiblySensitiveTweetStore: FutureEffect[UpdatePossiblySensitiveTweet.Event] - ): Type = - FutureArrow { request => - val nsfwAdminMutation = Mutation[Boolean](_ => request.nsfwAdmin).checkEq - val nsfwUserMutation = Mutation[Boolean](_ => request.nsfwUser).checkEq - val tweetMutation = - TweetLenses.nsfwAdmin - .mutation(nsfwAdminMutation) - .also(TweetLenses.nsfwUser.mutation(nsfwUserMutation)) - - for { - originalTweet <- tweetGetter(request.tweetId) - _ <- tweetMutation(originalTweet) match { - case None => Future.Unit - case Some(mutatedTweet) => - userGetter(getUserId(originalTweet)) - .map { user => - UpdatePossiblySensitiveTweet.Event( - tweet = mutatedTweet, - user = user, - timestamp = Time.now, - byUserId = request.byUserId, - nsfwAdminChange = nsfwAdminMutation(TweetLenses.nsfwAdmin.get(originalTweet)), - nsfwUserChange = nsfwUserMutation(TweetLenses.nsfwUser.get(originalTweet)), - note = request.note, - host = request.host - ) - } - .flatMap(updatePossiblySensitiveTweetStore) - } - } yield () - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlEntityBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlEntityBuilder.docx new file mode 100644 index 000000000..add172cb9 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlEntityBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlEntityBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlEntityBuilder.scala deleted file mode 100644 index 5de0fa625..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlEntityBuilder.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.tco_util.TcoUrl -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.thriftscala.entities.EntityExtractor -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.tweettext.IndexConverter -import com.twitter.tweetypie.tweettext.Offset -import com.twitter.tweetypie.tweettext.Preprocessor._ - -object UrlEntityBuilder { - import UpstreamFailure.UrlShorteningFailure - import UrlShortener.Context - - /** - * Extracts URLs from the given tweet text, shortens them, and returns an updated tweet - * text that contains the shortened URLs, along with the generated `UrlEntity`s. - */ - type Type = FutureArrow[(String, Context), (String, Seq[UrlEntity])] - - def fromShortener(shortener: UrlShortener.Type): Type = - FutureArrow { - case (text, ctx) => - Future - .collect(EntityExtractor.extractAllUrls(text).map(shortenEntity(shortener, _, ctx))) - .map(_.flatMap(_.toSeq)) - .map(updateTextAndUrls(text, _)(replaceInvisiblesWithWhitespace)) - } - - /** - * Update a url entity with tco-ed url - * - * @param urlEntity an url entity with long url in the `url` field - * @param ctx additional data needed to build the shortener request - * @return an updated url entity with tco-ed url in the `url` field, - * and long url in the `expanded` field - */ - private def shortenEntity( - shortener: UrlShortener.Type, - entity: UrlEntity, - ctx: Context - ): Future[Option[UrlEntity]] = - shortener((TcoUrl.normalizeProtocol(entity.url), ctx)) - .map { urlData => - Some( - entity.copy( - url = urlData.shortUrl, - expanded = Some(urlData.longUrl), - display = Some(urlData.displayText) - ) - ) - } - .rescue { - // fail tweets with invalid urls - case UrlShortener.InvalidUrlError => - Future.exception(TweetCreateFailure.State(TweetCreateState.InvalidUrl)) - // fail tweets with malware urls - case UrlShortener.MalwareUrlError => - Future.exception(TweetCreateFailure.State(TweetCreateState.MalwareUrl)) - // propagate OverCapacity - case e @ OverCapacity(_) => Future.exception(e) - // convert any other failure into UrlShorteningFailure - case e => Future.exception(UrlShorteningFailure(e)) - } - - /** - * Applies a text-modification function to all parts of the text not found within a UrlEntity, - * and then updates all the UrlEntity indices as necessary. - */ - def updateTextAndUrls( - text: String, - urlEntities: Seq[UrlEntity] - )( - textMod: String => String - ): (String, Seq[UrlEntity]) = { - var offsetInText = Offset.CodePoint(0) - var offsetInNewText = Offset.CodePoint(0) - val newText = new StringBuilder - val newUrlEntities = Seq.newBuilder[UrlEntity] - val indexConverter = new IndexConverter(text) - - urlEntities.foreach { e => - val nonUrl = textMod(indexConverter.substringByCodePoints(offsetInText.toInt, e.fromIndex)) - newText.append(nonUrl) - newText.append(e.url) - offsetInText = Offset.CodePoint(e.toIndex.toInt) - - val urlFrom = offsetInNewText + Offset.CodePoint.length(nonUrl) - val urlTo = urlFrom + Offset.CodePoint.length(e.url) - val newEntity = - e.copy(fromIndex = urlFrom.toShort, toIndex = urlTo.toShort) - - newUrlEntities += newEntity - offsetInNewText = urlTo - } - - newText.append(textMod(indexConverter.substringByCodePoints(offsetInText.toInt))) - - (newText.toString, newUrlEntities.result()) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlShortener.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlShortener.docx new file mode 100644 index 000000000..686804e96 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlShortener.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlShortener.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlShortener.scala deleted file mode 100644 index bdf939da7..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UrlShortener.scala +++ /dev/null @@ -1,106 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.service.talon.thriftscala._ -import com.twitter.servo.util.FutureArrow -import com.twitter.tco_util.DisplayUrl -import com.twitter.tco_util.TcoUrl -import com.twitter.tweetypie.backends.Talon -import com.twitter.tweetypie.core.OverCapacity -import com.twitter.tweetypie.store.Guano -import com.twitter.tweetypie.thriftscala.ShortenedUrl -import scala.util.control.NoStackTrace - -object UrlShortener { - type Type = FutureArrow[(String, Context), ShortenedUrl] - - case class Context( - tweetId: TweetId, - userId: UserId, - createdAt: Time, - userProtected: Boolean, - clientAppId: Option[Long] = None, - remoteHost: Option[String] = None, - dark: Boolean = false) - - object MalwareUrlError extends Exception with NoStackTrace - object InvalidUrlError extends Exception with NoStackTrace - - /** - * Returns a new UrlShortener that checks the response from the underlying shortner - * and, if the request is not dark but fails with a MalwareUrlError, scribes request - * info to guano. - */ - def scribeMalware(guano: Guano)(underlying: Type): Type = - FutureArrow { - case (longUrl, ctx) => - underlying((longUrl, ctx)).onFailure { - case MalwareUrlError if !ctx.dark => - guano.scribeMalwareAttempt( - Guano.MalwareAttempt( - longUrl, - ctx.userId, - ctx.clientAppId, - ctx.remoteHost - ) - ) - case _ => - } - } - - def fromTalon(talonShorten: Talon.Shorten): Type = { - val log = Logger(getClass) - - FutureArrow { - case (longUrl, ctx) => - val request = - ShortenRequest( - userId = ctx.userId, - longUrl = longUrl, - auditMsg = "tweetypie", - directMessage = Some(false), - protectedAccount = Some(ctx.userProtected), - maxShortUrlLength = None, - tweetData = Some(TweetData(ctx.tweetId, ctx.createdAt.inMilliseconds)), - trafficType = - if (ctx.dark) ShortenTrafficType.Testing - else ShortenTrafficType.Production - ) - - talonShorten(request).flatMap { res => - res.responseCode match { - case ResponseCode.Ok => - if (res.malwareStatus == MalwareStatus.UrlBlocked) { - Future.exception(MalwareUrlError) - } else { - val shortUrl = - res.fullShortUrl.getOrElse { - // fall back to fromSlug if talon response does not have the full short url - // Could be replaced with an exception once the initial integration on production - // is done - TcoUrl.fromSlug(res.shortUrl, TcoUrl.isHttps(res.longUrl)) - } - - Future.value( - ShortenedUrl( - shortUrl = shortUrl, - longUrl = res.longUrl, - displayText = DisplayUrl(shortUrl, Some(res.longUrl), true) - ) - ) - } - - case ResponseCode.BadInput => - log.warn(s"Talon rejected URL that Extractor thought was fine: $longUrl") - Future.exception(InvalidUrlError) - - // we shouldn't see other ResponseCodes, because Talon.Shorten translates them to - // exceptions, but we have this catch-all just in case. - case resCode => - log.warn(s"Unexpected response code $resCode for '$longUrl'") - Future.exception(OverCapacity("talon")) - } - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UserTakedownHandler.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UserTakedownHandler.docx new file mode 100644 index 000000000..3d9033035 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UserTakedownHandler.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UserTakedownHandler.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UserTakedownHandler.scala deleted file mode 100644 index 1410525d5..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/UserTakedownHandler.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.tweetypie -package handler - -import com.twitter.servo.util.FutureArrow -import com.twitter.tweetypie.store.Takedown -import com.twitter.tweetypie.thriftscala.DataError -import com.twitter.tweetypie.thriftscala.DataErrorCause -import com.twitter.tweetypie.thriftscala.SetTweetUserTakedownRequest - -trait UserTakedownHandler { - val setTweetUserTakedownRequest: FutureArrow[SetTweetUserTakedownRequest, Unit] -} - -/** - * This handler processes SetTweetUserTakedownRequest objects sent to Tweetypie's - * setTweetUserTakedown endpoint. These requests originate from tweetypie daemon and the - * request object specifies the user ID of the user who is being modified, and a boolean value - * to indicate whether takedown is being added or removed. - * - * If takedown is being added, the hasTakedown bit is set on all of the user's tweets. - * If takedown is being removed, we can't automatically unset the hasTakedown bit on all tweets - * since some of the tweets might have tweet-specific takedowns, in which case the hasTakedown bit - * needs to remain set. Instead, we flush the user's tweets from cache, and let the repairer - * unset the bit when hydrating tweets where the bit is set but no user or tweet - * takedown country codes are present. - */ -object UserTakedownHandler { - type Type = FutureArrow[SetTweetUserTakedownRequest, Unit] - - def takedownEvent(userHasTakedown: Boolean): Tweet => Option[Takedown.Event] = - tweet => { - val tweetHasTakedown = - TweetLenses.tweetypieOnlyTakedownCountryCodes(tweet).exists(_.nonEmpty) || - TweetLenses.tweetypieOnlyTakedownReasons(tweet).exists(_.nonEmpty) - val updatedHasTakedown = userHasTakedown || tweetHasTakedown - if (updatedHasTakedown == TweetLenses.hasTakedown(tweet)) - None - else - Some( - Takedown.Event( - tweet = TweetLenses.hasTakedown.set(tweet, updatedHasTakedown), - timestamp = Time.now, - eventbusEnqueue = false, - scribeForAudit = false, - updateCodesAndReasons = false - ) - ) - } - - def setHasTakedown( - tweetTakedown: FutureEffect[Takedown.Event], - userHasTakedown: Boolean - ): FutureEffect[Seq[Tweet]] = - tweetTakedown.contramapOption(takedownEvent(userHasTakedown)).liftSeq - - def verifyTweetUserId(expectedUserId: Option[UserId], tweet: Tweet): Unit = { - val tweetUserId: UserId = getUserId(tweet) - val tweetId: Long = tweet.id - expectedUserId.filter(_ != tweetUserId).foreach { u => - throw DataError( - message = - s"SetTweetUserTakedownRequest userId $u does not match userId $tweetUserId for Tweet: $tweetId", - errorCause = Some(DataErrorCause.UserTweetRelationship), - ) - } - } - - def apply( - getTweet: FutureArrow[TweetId, Option[Tweet]], - tweetTakedown: FutureEffect[Takedown.Event], - ): Type = - FutureArrow { request => - for { - tweet <- getTweet(request.tweetId) - _ = tweet.foreach(t => verifyTweetUserId(request.userId, t)) - _ <- setHasTakedown(tweetTakedown, request.hasTakedown)(tweet.toSeq) - } yield () - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/WritePathQueryOptions.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/WritePathQueryOptions.docx new file mode 100644 index 000000000..42e22803e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/WritePathQueryOptions.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/WritePathQueryOptions.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/WritePathQueryOptions.scala deleted file mode 100644 index 5ef7573f2..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/WritePathQueryOptions.scala +++ /dev/null @@ -1,153 +0,0 @@ -package com.twitter.tweetypie.handler - -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.tweetypie.repository.CacheControl -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.thriftscala.MediaEntity -import com.twitter.tweetypie.thriftscala.StatusCounts -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.tweetypie.thriftscala.WritePathHydrationOptions - -object WritePathQueryOptions { - - /** - * Base TweetQuery.Include for all hydration options. - */ - val BaseInclude: TweetQuery.Include = - GetTweetsHandler.BaseInclude.also( - tweetFields = Set( - Tweet.CardReferenceField.id, - Tweet.MediaTagsField.id, - Tweet.SelfPermalinkField.id, - Tweet.ExtendedTweetMetadataField.id, - Tweet.VisibleTextRangeField.id, - Tweet.NsfaHighRecallLabelField.id, - Tweet.CommunitiesField.id, - Tweet.ExclusiveTweetControlField.id, - Tweet.TrustedFriendsControlField.id, - Tweet.CollabControlField.id, - Tweet.EditControlField.id, - Tweet.EditPerspectiveField.id, - Tweet.NoteTweetField.id - ) - ) - - /** - * Base TweetQuery.Include for all creation-related hydrations. - */ - val BaseCreateInclude: TweetQuery.Include = - BaseInclude - .also( - tweetFields = Set( - Tweet.PlaceField.id, - Tweet.ProfileGeoEnrichmentField.id, - Tweet.SelfThreadMetadataField.id - ), - mediaFields = Set(MediaEntity.AdditionalMetadataField.id), - quotedTweet = Some(true), - pastedMedia = Some(true) - ) - - /** - * Base TweetQuery.Include for all deletion-related hydrations. - */ - val BaseDeleteInclude: TweetQuery.Include = BaseInclude - .also(tweetFields = - Set(Tweet.BounceLabelField.id, Tweet.ConversationControlField.id, Tweet.EditControlField.id)) - - val AllCounts: Set[Short] = StatusCounts.fieldInfos.map(_.tfield.id).toSet - - def insert( - cause: TweetQuery.Cause, - user: User, - options: WritePathHydrationOptions, - isEditControlEdit: Boolean - ): TweetQuery.Options = - createOptions( - writePathHydrationOptions = options, - includePerspective = false, - // include counts if tweet edit, otherwise false - includeCounts = isEditControlEdit, - cause = cause, - forUser = user, - // Do not perform any filtering when we are hydrating the tweet we are creating - safetyLevel = SafetyLevel.FilterNone - ) - - def retweetSourceTweet(user: User, options: WritePathHydrationOptions): TweetQuery.Options = - createOptions( - writePathHydrationOptions = options, - includePerspective = true, - includeCounts = true, - cause = TweetQuery.Cause.Read, - forUser = user, - // If Scarecrow is down, we may proceed with creating a RT. The safetyLevel is necessary - // to prevent so that the inner tweet's count is not sent in the TweetCreateEvent we send - // to EventBus. If this were emitted, live pipeline would publish counts to the clients. - safetyLevel = SafetyLevel.TweetWritesApi - ) - - def quotedTweet(user: User, options: WritePathHydrationOptions): TweetQuery.Options = - createOptions( - writePathHydrationOptions = options, - includePerspective = true, - includeCounts = true, - cause = TweetQuery.Cause.Read, - forUser = user, - // We pass in the safetyLevel so that the inner tweet's are excluded - // from the TweetCreateEvent we send to EventBus. If this were emitted, - // live pipeline would publish counts to the clients. - safetyLevel = SafetyLevel.TweetWritesApi - ) - - private def condSet[A](cond: Boolean, item: A): Set[A] = - if (cond) Set(item) else Set.empty - - private def createOptions( - writePathHydrationOptions: WritePathHydrationOptions, - includePerspective: Boolean, - includeCounts: Boolean, - cause: TweetQuery.Cause, - forUser: User, - safetyLevel: SafetyLevel, - ): TweetQuery.Options = { - val cardsEnabled: Boolean = writePathHydrationOptions.includeCards - val cardsPlatformKeySpecified: Boolean = writePathHydrationOptions.cardsPlatformKey.nonEmpty - val cardsV1Enabled: Boolean = cardsEnabled && !cardsPlatformKeySpecified - val cardsV2Enabled: Boolean = cardsEnabled && cardsPlatformKeySpecified - - TweetQuery.Options( - include = BaseCreateInclude.also( - tweetFields = - condSet(includePerspective, Tweet.PerspectiveField.id) ++ - condSet(cardsV1Enabled, Tweet.CardsField.id) ++ - condSet(cardsV2Enabled, Tweet.Card2Field.id) ++ - condSet(includeCounts, Tweet.CountsField.id) ++ - // for PreviousCountsField, copy includeCounts state on the write path - condSet(includeCounts, Tweet.PreviousCountsField.id) ++ - // hydrate ConversationControl on Reply Tweet creations so clients can consume - Set(Tweet.ConversationControlField.id), - countsFields = if (includeCounts) AllCounts else Set.empty - ), - cause = cause, - forUserId = Some(forUser.id), - cardsPlatformKey = writePathHydrationOptions.cardsPlatformKey, - languageTag = forUser.account.map(_.language).getOrElse("en"), - extensionsArgs = writePathHydrationOptions.extensionsArgs, - safetyLevel = safetyLevel, - simpleQuotedTweet = writePathHydrationOptions.simpleQuotedTweet - ) - } - - def deleteTweets: TweetQuery.Options = - TweetQuery.Options( - include = BaseDeleteInclude, - cacheControl = CacheControl.ReadOnlyCache, - extensionsArgs = None, - requireSourceTweet = false // retweet should be deletable even if source tweet missing - ) - - def deleteTweetsWithoutEditControl: TweetQuery.Options = - deleteTweets.copy(enableEditControlHydration = false) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/package.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/package.docx new file mode 100644 index 000000000..f461d8f63 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/package.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/package.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/package.scala deleted file mode 100644 index e9d5021a0..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/handler/package.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.tweetypie - -import com.twitter.context.thriftscala.Viewer -import com.twitter.tweetypie.thriftscala._ - -import scala.util.matching.Regex -import com.twitter.context.TwitterContext -import com.twitter.finagle.stats.Stat -import com.twitter.snowflake.id.SnowflakeId - -package object handler { - type PlaceLanguage = String - type TweetIdGenerator = () => Future[TweetId] - type NarrowcastValidator = FutureArrow[Narrowcast, Narrowcast] - type ReverseGeocoder = FutureArrow[(GeoCoordinates, PlaceLanguage), Option[Place]] - type CardUri = String - - // A narrowcast location can be a PlaceId or a US metro code. - type NarrowcastLocation = String - - val PlaceIdRegex: Regex = """(?i)\A[0-9a-fA-F]{16}\Z""".r - - // Bring Tweetypie permitted TwitterContext into scope - val TwitterContext: TwitterContext = - com.twitter.context.TwitterContext(com.twitter.tweetypie.TwitterContextPermit) - - def getContributor(userId: UserId): Option[Contributor] = { - val viewer = TwitterContext().getOrElse(Viewer()) - viewer.authenticatedUserId.filterNot(_ == userId).map(id => Contributor(id)) - } - - def trackLossyReadsAfterWrite(stat: Stat, windowLength: Duration)(tweetId: TweetId): Unit = { - // If the requested Tweet is NotFound, and the tweet age is less than the defined {{windowLength}} duration, - // then we capture the percentiles of when this request was attempted. - // This is being tracked to understand how lossy the reads are directly after tweet creation. - for { - timestamp <- SnowflakeId.timeFromIdOpt(tweetId) - age = Time.now.since(timestamp) - if age.inMillis <= windowLength.inMillis - } yield stat.add(age.inMillis) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/BUILD b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/BUILD deleted file mode 100644 index 0fb53615d..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/BUILD +++ /dev/null @@ -1,58 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "core-app-services/lib:coreservices", - "featureswitches/featureswitches-core:v2", - "featureswitches/featureswitches-core/src/main/scala", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "mediaservices/commons/src/main/thrift:thrift-scala", - "mediaservices/media-util", - "scrooge/scrooge-core", - "tweetypie/servo/repo", - "tweetypie/servo/repo/src/main/thrift:thrift-scala", - "tweetypie/servo/util", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/scala/com/twitter/takedown/util", - "src/thrift/com/twitter/context:twitter-context-scala", - "src/thrift/com/twitter/dataproducts:enrichments_profilegeo-scala", - "src/thrift/com/twitter/escherbird:media-annotation-structs-scala", - "src/thrift/com/twitter/escherbird:tweet-annotation-scala", - "src/thrift/com/twitter/escherbird/common:common-scala", - "src/thrift/com/twitter/expandodo:cards-scala", - "src/thrift/com/twitter/expandodo:only-scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-label-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:media-entity-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:service-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core", - "stitch/stitch-timelineservice/src/main/scala", - "strato/src/main/scala/com/twitter/strato/access", - "strato/src/main/scala/com/twitter/strato/callcontext", - "tco-util", - "tweet-util", - "tweetypie/server/src/main/scala/com/twitter/tweetypie", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/core", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/media", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/repository", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil", - "tweetypie/server/src/main/thrift:compiled-scala", - "tweetypie/common/src/scala/com/twitter/tweetypie/additionalfields", - "tweetypie/common/src/scala/com/twitter/tweetypie/client_id", - "tweetypie/common/src/scala/com/twitter/tweetypie/media", - "tweetypie/common/src/scala/com/twitter/tweetypie/thriftscala/entities", - "tweetypie/common/src/scala/com/twitter/tweetypie/tweettext", - "tweetypie/common/src/scala/com/twitter/tweetypie/util", - "twitter-context", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - "util/util-stats/src/main/scala", - "visibility/common/src/main/thrift/com/twitter/visibility:action-scala", - "visibility/results/src/main/scala/com/twitter/visibility/results/counts", - ], -) diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/BUILD.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/BUILD.docx new file mode 100644 index 000000000..cc4c1fd01 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/BUILD.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/Card2Hydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/Card2Hydrator.docx new file mode 100644 index 000000000..38579828e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/Card2Hydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/Card2Hydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/Card2Hydrator.scala deleted file mode 100644 index 08ad91bc8..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/Card2Hydrator.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.expandodo.thriftscala.Card2 -import com.twitter.expandodo.thriftscala.Card2RequestOptions -import com.twitter.featureswitches.v2.FeatureSwitchResults -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.CardReferenceUriExtractor -import com.twitter.tweetypie.core.NonTombstone -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object Card2Hydrator { - type Type = ValueHydrator[Option[Card2], Ctx] - - case class Ctx( - urlEntities: Seq[UrlEntity], - mediaEntities: Seq[MediaEntity], - cardReference: Option[CardReference], - underlyingTweetCtx: TweetCtx, - featureSwitchResults: Option[FeatureSwitchResults]) - extends TweetCtx.Proxy - - val hydratedField: FieldByPath = fieldByPath(Tweet.Card2Field) - val hydrationUrlBlockListKey = "card_hydration_blocklist" - - def apply(repo: Card2Repository.Type): ValueHydrator[Option[Card2], Ctx] = - ValueHydrator[Option[Card2], Ctx] { (_, ctx) => - val repoCtx = requestOptions(ctx) - val filterURLs = ctx.featureSwitchResults - .flatMap(_.getStringArray(hydrationUrlBlockListKey, false)) - .getOrElse(Seq()) - - val requests = - ctx.cardReference match { - case Some(CardReferenceUriExtractor(cardUri)) => - cardUri match { - case NonTombstone(uri) if !filterURLs.contains(uri) => - Seq((UrlCard2Key(uri), repoCtx)) - case _ => Nil - } - case _ => - ctx.urlEntities - .filterNot(e => e.expanded.exists(filterURLs.contains)) - .map(e => (UrlCard2Key(e.url), repoCtx)) - } - - Stitch - .traverse(requests) { - case (key, opts) => repo(key, opts).liftNotFoundToOption - }.liftToTry.map { - case Return(results) => - results.flatten.lastOption match { - case None => ValueState.UnmodifiedNone - case res => ValueState.modified(res) - } - case Throw(_) => ValueState.partial(None, hydratedField) - } - }.onlyIf { (curr, ctx) => - curr.isEmpty && - ctx.tweetFieldRequested(Tweet.Card2Field) && - ctx.opts.cardsPlatformKey.nonEmpty && - !ctx.isRetweet && - ctx.mediaEntities.isEmpty && - (ctx.cardReference.nonEmpty || ctx.urlEntities.nonEmpty) - } - - private[this] def requestOptions(ctx: Ctx) = - Card2RequestOptions( - platformKey = ctx.opts.cardsPlatformKey.get, - perspectiveUserId = ctx.opts.forUserId, - allowNonTcoUrls = ctx.cardReference.nonEmpty, - languageTag = Some(ctx.opts.languageTag) - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CardHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CardHydrator.docx new file mode 100644 index 000000000..8d2831410 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CardHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CardHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CardHydrator.scala deleted file mode 100644 index 4a267bfb6..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CardHydrator.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.expandodo.thriftscala.Card -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object CardHydrator { - type Type = ValueHydrator[Option[Seq[Card]], Ctx] - - case class Ctx( - urlEntities: Seq[UrlEntity], - mediaEntities: Seq[MediaEntity], - underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - val hydratedField: FieldByPath = fieldByPath(Tweet.CardsField) - - private[this] val partialResult = ValueState.partial(None, hydratedField) - - def apply(repo: CardRepository.Type): Type = { - def getCards(url: String): Stitch[Seq[Card]] = - repo(url).handle { case NotFound => Nil } - - ValueHydrator[Option[Seq[Card]], Ctx] { (_, ctx) => - val urls = ctx.urlEntities.map(_.url) - - Stitch.traverse(urls)(getCards _).liftToTry.map { - case Return(cards) => - // even though we are hydrating a type of Option[Seq[Card]], we only - // ever return at most one card, and always the last one. - val res = cards.flatten.lastOption.toSeq - if (res.isEmpty) ValueState.UnmodifiedNone - else ValueState.modified(Some(res)) - case _ => partialResult - } - }.onlyIf { (curr, ctx) => - curr.isEmpty && - ctx.tweetFieldRequested(Tweet.CardsField) && - !ctx.isRetweet && - ctx.mediaEntities.isEmpty - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorHydrator.docx new file mode 100644 index 000000000..82acc989b Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorHydrator.scala deleted file mode 100644 index 8adee73b3..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorHydrator.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object ContributorHydrator { - type Type = ValueHydrator[Option[Contributor], TweetCtx] - - val hydratedField: FieldByPath = fieldByPath(Tweet.ContributorField, Contributor.ScreenNameField) - - def once(h: Type): Type = - TweetHydration.completeOnlyOnce( - hydrationType = HydrationType.Contributor, - hydrator = h - ) - - def apply(repo: UserIdentityRepository.Type): Type = - ValueHydrator[Contributor, TweetCtx] { (curr, _) => - repo(UserKey(curr.userId)).liftToTry.map { - case Return(userIdent) => ValueState.delta(curr, update(curr, userIdent)) - case Throw(NotFound) => ValueState.unmodified(curr) - case Throw(_) => ValueState.partial(curr, hydratedField) - } - }.onlyIf((curr, _) => curr.screenName.isEmpty).liftOption - - /** - * Updates a Contributor using the given user data. - */ - private def update(curr: Contributor, userIdent: UserIdentity): Contributor = - curr.copy( - screenName = Some(userIdent.screenName) - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorVisibilityFilter.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorVisibilityFilter.docx new file mode 100644 index 000000000..abbf563e6 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorVisibilityFilter.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorVisibilityFilter.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorVisibilityFilter.scala deleted file mode 100644 index 079b90f78..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ContributorVisibilityFilter.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.thriftscala._ - -/** - * Remove contributor data from tweet if it should not be available to the - * caller. The contributor field is populated in the cached - * [[ContributorHydrator]]. - * - * Contributor data is always available on the write path. It is available on - * the read path for the tweet author (or user authenticated as the tweet - * author in the case of contributors/teams), or if the caller has disabled - * visibility filtering. - * - * The condition for running this filtering hydrator (onlyIf) has been a - * source of confusion. Keep in mind that the condition expresses when to - * *remove* data, not when to return it. - * - * In short, keep data when: - * !reading || requested by author || !(enforce visibility filtering) - * - * Remove data when none of these conditions apply: - * reading && !(requested by author) && enforce visibility filtering - * - */ -object ContributorVisibilityFilter { - type Type = ValueHydrator[Option[Contributor], TweetCtx] - - def apply(): Type = - ValueHydrator - .map[Option[Contributor], TweetCtx] { - case (Some(_), _) => ValueState.modified(None) - case (None, _) => ValueState.unmodified(None) - } - .onlyIf { (_, ctx) => - ctx.opts.cause.reading(ctx.tweetId) && - !ctx.opts.forUserId.contains(ctx.userId) && - ctx.opts.enforceVisibilityFiltering - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationControlHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationControlHydrator.docx new file mode 100644 index 000000000..f007e6f87 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationControlHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationControlHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationControlHydrator.scala deleted file mode 100644 index 55df7e8a7..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationControlHydrator.scala +++ /dev/null @@ -1,108 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.repository.ConversationControlRepository -import com.twitter.tweetypie.serverutil.ExceptionCounter -import com.twitter.tweetypie.thriftscala.ConversationControl - -private object ReplyTweetConversationControlHydrator { - type Type = ConversationControlHydrator.Type - type Ctx = ConversationControlHydrator.Ctx - - // The conversation control thrift field was added Feb 17th, 2020. - // No conversation before this will have a conversation control field to hydrate. - // We explicitly short circuit to save resources from querying for tweets we - // know do not have conversation control fields set. - val FirstValidDate: Time = Time.fromMilliseconds(1554076800000L) // 2020-02-17 - - def apply( - repo: ConversationControlRepository.Type, - stats: StatsReceiver - ): Type = { - val exceptionCounter = ExceptionCounter(stats) - - ValueHydrator[Option[ConversationControl], Ctx] { (curr, ctx) => - repo(ctx.conversationId.get, ctx.opts.cacheControl).liftToTry.map { - case Return(conversationControl) => - ValueState.delta(curr, conversationControl) - case Throw(exception) => { - // In the case where we get an exception, we want to count the - // exception but fail open. - exceptionCounter(exception) - - // Reply Tweet Tweet.ConversationControlField hydration should fail open. - // Ideally we would return ValueState.partial here to notify Tweetypie the caller - // that requested the Tweet.ConversationControlField field was not hydrated. - // We cannot do so because GetTweetFields will return TweetFieldsResultFailed - // for partial results which would fail closed. - ValueState.unmodified(curr) - } - } - }.onlyIf { (_, ctx) => - // This hydrator is specifically for replies so only run when Tweet is a reply - ctx.inReplyToTweetId.isDefined && - // See comment for FirstValidDate - ctx.createdAt > FirstValidDate && - // We need conversation id to get ConversationControl - ctx.conversationId.isDefined && - // Only run if the ConversationControl was requested - ctx.tweetFieldRequested(Tweet.ConversationControlField) - } - } -} - -/** - * ConversationControlHydrator is used to hydrate the conversationControl field. - * For root Tweets, this hydrator just passes through the existing conversationControl. - * For reply Tweets, it loads the conversationControl from the root Tweet of the conversation. - * Only root Tweets in a conversation (i.e. the Tweet pointed to by conversationId) have - * a persisted conversationControl, so we have to hydrate that field for all replies in order - * to know if a Tweet in a conversation can be replied to. - */ -object ConversationControlHydrator { - type Type = ValueHydrator[Option[ConversationControl], Ctx] - - case class Ctx(conversationId: Option[ConversationId], underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - private def scrubInviteViaMention( - ccOpt: Option[ConversationControl] - ): Option[ConversationControl] = { - ccOpt collect { - case ConversationControl.ByInvitation(byInvitation) => - ConversationControl.ByInvitation(byInvitation.copy(inviteViaMention = None)) - case ConversationControl.Community(community) => - ConversationControl.Community(community.copy(inviteViaMention = None)) - case ConversationControl.Followers(followers) => - ConversationControl.Followers(followers.copy(inviteViaMention = None)) - } - } - - def apply( - repo: ConversationControlRepository.Type, - disableInviteViaMention: Gate[Unit], - stats: StatsReceiver - ): Type = { - val replyTweetConversationControlHydrator = ReplyTweetConversationControlHydrator( - repo, - stats - ) - - ValueHydrator[Option[ConversationControl], Ctx] { (curr, ctx) => - val ccUpdated = if (disableInviteViaMention()) { - scrubInviteViaMention(curr) - } else { - curr - } - - if (ctx.inReplyToTweetId.isEmpty) { - // For non-reply tweets, pass through the existing conversation control - Stitch.value(ValueState.delta(curr, ccUpdated)) - } else { - replyTweetConversationControlHydrator(ccUpdated, ctx) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationIdHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationIdHydrator.docx new file mode 100644 index 000000000..b84e0f90f Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationIdHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationIdHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationIdHydrator.scala deleted file mode 100644 index 172ff1746..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationIdHydrator.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -/** - * Hydrates the conversationId field for any tweet that is a reply to another tweet. - * It uses that other tweet's conversationId. - */ -object ConversationIdHydrator { - type Type = ValueHydrator[Option[ConversationId], TweetCtx] - - val hydratedField: FieldByPath = - fieldByPath(Tweet.CoreDataField, TweetCoreData.ConversationIdField) - - def apply(repo: ConversationIdRepository.Type): Type = - ValueHydrator[Option[ConversationId], TweetCtx] { (_, ctx) => - ctx.inReplyToTweetId match { - case None => - // Not a reply to another tweet, use tweet id as conversation root - Stitch.value(ValueState.modified(Some(ctx.tweetId))) - case Some(parentId) => - // Lookup conversation id from in-reply-to tweet - repo(ConversationIdKey(ctx.tweetId, parentId)).liftToTry.map { - case Return(rootId) => ValueState.modified(Some(rootId)) - case Throw(_) => ValueState.partial(None, hydratedField) - } - } - }.onlyIf((curr, _) => curr.isEmpty) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationMutedHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationMutedHydrator.docx new file mode 100644 index 000000000..a07acfd68 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationMutedHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationMutedHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationMutedHydrator.scala deleted file mode 100644 index 3f6e6ad7e..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ConversationMutedHydrator.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala.FieldByPath - -/** - * Hydrates the `conversationMuted` field of Tweet. `conversationMuted` - * will be true if the conversation that this tweet is part of has been - * muted by the user. This field is perspectival, so the result of this - * hydrator should never be cached. - */ -object ConversationMutedHydrator { - type Type = ValueHydrator[Option[Boolean], Ctx] - - case class Ctx(conversationId: Option[TweetId], underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - val hydratedField: FieldByPath = fieldByPath(Tweet.ConversationMutedField) - - private[this] val partialResult = ValueState.partial(None, hydratedField) - private[this] val modifiedTrue = ValueState.modified(Some(true)) - private[this] val modifiedFalse = ValueState.modified(Some(false)) - - def apply(repo: ConversationMutedRepository.Type): Type = { - - ValueHydrator[Option[Boolean], Ctx] { (_, ctx) => - (ctx.opts.forUserId, ctx.conversationId) match { - case (Some(userId), Some(convoId)) => - repo(userId, convoId).liftToTry - .map { - case Return(true) => modifiedTrue - case Return(false) => modifiedFalse - case Throw(_) => partialResult - } - case _ => - ValueState.StitchUnmodifiedNone - } - }.onlyIf { (curr, ctx) => - // It is unlikely that this field will already be set, but if, for - // some reason, this hydrator is run on a tweet that already has - // this value set, we will skip the work to check again. - curr.isEmpty && - // We only hydrate this field if it is explicitly requested. At - // the time of this writing, this field is only used for - // displaying UI for toggling the muted state of the relevant - // conversation. - ctx.tweetFieldRequested(Tweet.ConversationMutedField) && - // Retweets are not part of a conversation, so should not be muted. - !ctx.isRetweet - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CopyFromSourceTweet.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CopyFromSourceTweet.docx new file mode 100644 index 000000000..1cddd6b40 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CopyFromSourceTweet.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CopyFromSourceTweet.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CopyFromSourceTweet.scala deleted file mode 100644 index 0e8a9eada..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CopyFromSourceTweet.scala +++ /dev/null @@ -1,229 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.tweettext.TweetText -import com.twitter.tweetypie.thriftscala._ - -object CopyFromSourceTweet { - - /** - * A `ValueHydrator` that copies and/or merges certain fields from a retweet's source - * tweet into the retweet. - */ - def hydrator: ValueHydrator[TweetData, TweetQuery.Options] = - ValueHydrator.map { (td, _) => - td.sourceTweetResult.map(_.value.tweet) match { - case None => ValueState.unmodified(td) - case Some(src) => ValueState.modified(td.copy(tweet = copy(src, td.tweet))) - } - } - - /** - * Updates `dst` with fields from `src`. This is more complicated than you would think, because: - * - * - the tweet has an extra mention entity due to the "RT @user" prefix; - * - the retweet text may be truncated at the end, and doesn't necessarily contain all of the - * the text from the source tweet. truncation may happen in the middle of entity. - * - the text in the retweet may have a different unicode normalization, which affects - * code point indices. this means entities aren't shifted by a fixed amount equal to - * the RT prefix. - * - url entities, when hydrated, may be converted to media entities; url entities may not - * be hydrated in the retweet, so the source tweet may have a media entity that corresponds - * to an unhydrated url entity in the retweet. - * - there may be multiple media entities that map to a single url entity, because the tweet - * may have multiple photos. - */ - def copy(src: Tweet, dst: Tweet): Tweet = { - val srcCoreData = src.coreData.get - val dstCoreData = dst.coreData.get - - // get the code point index of the end of the text - val max = getText(dst).codePointCount(0, getText(dst).length).toShort - - // get all entities from the source tweet, merged into a single list sorted by fromIndex. - val srcEntities = getWrappedEntities(src) - - // same for the retweet, but drop first @mention, add back later - val dstEntities = getWrappedEntities(dst).drop(1) - - // merge indices from dst into srcEntities. at the end, resort entities back - // to their original ordering. for media entities, order matters to clients. - val mergedEntities = merge(srcEntities, dstEntities, max).sortBy(_.position) - - // extract entities back out by type - val mentions = mergedEntities.collect { case WrappedMentionEntity(e, _) => e } - val hashtags = mergedEntities.collect { case WrappedHashtagEntity(e, _) => e } - val cashtags = mergedEntities.collect { case WrappedCashtagEntity(e, _) => e } - val urls = mergedEntities.collect { case WrappedUrlEntity(e, _) => e } - val media = mergedEntities.collect { case WrappedMediaEntity(e, _) => e } - - // merge the updated entities back into the retweet, adding the RT @mention back in - dst.copy( - coreData = Some( - dstCoreData.copy( - hasMedia = srcCoreData.hasMedia, - hasTakedown = dstCoreData.hasTakedown || srcCoreData.hasTakedown - ) - ), - mentions = Some(getMentions(dst).take(1) ++ mentions), - hashtags = Some(hashtags), - cashtags = Some(cashtags), - urls = Some(urls), - media = Some(media.map(updateSourceStatusId(src.id, getUserId(src)))), - quotedTweet = src.quotedTweet, - card2 = src.card2, - cards = src.cards, - language = src.language, - mediaTags = src.mediaTags, - spamLabel = src.spamLabel, - takedownCountryCodes = - mergeTakedowns(Seq(src, dst).map(TweetLenses.takedownCountryCodes.get): _*), - conversationControl = src.conversationControl, - exclusiveTweetControl = src.exclusiveTweetControl - ) - } - - /** - * Merges one or more optional lists of takedowns. If no lists are defined, returns None. - */ - private def mergeTakedowns(takedowns: Option[Seq[CountryCode]]*): Option[Seq[CountryCode]] = - if (takedowns.exists(_.isDefined)) { - Some(takedowns.flatten.flatten.distinct.sorted) - } else { - None - } - - /** - * A retweet should never have media without a source_status_id or source_user_id - */ - private def updateSourceStatusId( - srcTweetId: TweetId, - srcUserId: UserId - ): MediaEntity => MediaEntity = - mediaEntity => - if (mediaEntity.sourceStatusId.nonEmpty) { - // when sourceStatusId is set this indicates the media is "pasted media" so the values - // should already be correct (retweeting won't change sourceStatusId / sourceUserId) - mediaEntity - } else { - mediaEntity.copy( - sourceStatusId = Some(srcTweetId), - sourceUserId = Some(mediaEntity.sourceUserId.getOrElse(srcUserId)) - ) - } - - /** - * Attempts to match up entities from the source tweet with entities from the retweet, - * and to use the source tweet entities but shifted to the retweet entity indices. If an entity - * got truncated at the end of the retweet text, we drop it and any following entities. - */ - private def merge( - srcEntities: List[WrappedEntity], - rtEntities: List[WrappedEntity], - maxIndex: Short - ): List[WrappedEntity] = { - (srcEntities, rtEntities) match { - case (Nil, Nil) => - // successfully matched all entities! - Nil - - case (Nil, _) => - // no more source tweet entities, but we still have remaining retweet entities. - // this can happen if a a text truncation turns something invalid like #tag1#tag2 or - // @mention1@mention2 into a valid entity. just drop all the remaining retweet entities. - Nil - - case (_, Nil) => - // no more retweet entities, which means the remaining entities have been truncated. - Nil - - case (srcHead :: srcTail, rtHead :: rtTail) => - // we have more entities from the source tweet and the retweet. typically, we can - // match these entities because they have the same normalized text, but the retweet - // entity might be truncated, so we allow for a prefix match if the retweet entity - // ends at the end of the tweet. - val possiblyTruncated = rtHead.toIndex == maxIndex - 1 - val exactMatch = srcHead.normalizedText == rtHead.normalizedText - - if (exactMatch) { - // there could be multiple media entities for the same t.co url, so we need to find - // contiguous groupings of entities that share the same fromIndex. - val rtTail = rtEntities.dropWhile(_.fromIndex == rtHead.fromIndex) - val srcGroup = - srcEntities - .takeWhile(_.fromIndex == srcHead.fromIndex) - .map(_.shift(rtHead.fromIndex, rtHead.toIndex)) - val srcTail = srcEntities.drop(srcGroup.size) - - srcGroup ++ merge(srcTail, rtTail, maxIndex) - } else { - // if we encounter a mismatch, it is most likely because of truncation, - // so we stop here. - Nil - } - } - } - - /** - * Wraps all the entities with the appropriate WrappedEntity subclasses, merges them into - * a single list, and sorts by fromIndex. - */ - private def getWrappedEntities(tweet: Tweet): List[WrappedEntity] = - (getUrls(tweet).zipWithIndex.map { case (e, p) => WrappedUrlEntity(e, p) } ++ - getMedia(tweet).zipWithIndex.map { case (e, p) => WrappedMediaEntity(e, p) } ++ - getMentions(tweet).zipWithIndex.map { case (e, p) => WrappedMentionEntity(e, p) } ++ - getHashtags(tweet).zipWithIndex.map { case (e, p) => WrappedHashtagEntity(e, p) } ++ - getCashtags(tweet).zipWithIndex.map { case (e, p) => WrappedCashtagEntity(e, p) }) - .sortBy(_.fromIndex) - .toList - - /** - * The thrift-entity classes don't share a common entity parent class, so we wrap - * them with a class that allows us to mix entities together into a single list, and - * to provide a generic interface for shifting indicies. - */ - private sealed abstract class WrappedEntity( - val fromIndex: Short, - val toIndex: Short, - val rawText: String) { - - /** the original position of the entity within the entity group */ - val position: Int - - val normalizedText: String = TweetText.nfcNormalize(rawText).toLowerCase - - def shift(fromIndex: Short, toIndex: Short): WrappedEntity - } - - private case class WrappedUrlEntity(entity: UrlEntity, position: Int) - extends WrappedEntity(entity.fromIndex, entity.toIndex, entity.url) { - override def shift(fromIndex: Short, toIndex: Short): WrappedUrlEntity = - copy(entity.copy(fromIndex = fromIndex, toIndex = toIndex)) - } - - private case class WrappedMediaEntity(entity: MediaEntity, position: Int) - extends WrappedEntity(entity.fromIndex, entity.toIndex, entity.url) { - override def shift(fromIndex: Short, toIndex: Short): WrappedMediaEntity = - copy(entity.copy(fromIndex = fromIndex, toIndex = toIndex)) - } - - private case class WrappedMentionEntity(entity: MentionEntity, position: Int) - extends WrappedEntity(entity.fromIndex, entity.toIndex, entity.screenName) { - override def shift(fromIndex: Short, toIndex: Short): WrappedMentionEntity = - copy(entity.copy(fromIndex = fromIndex, toIndex = toIndex)) - } - - private case class WrappedHashtagEntity(entity: HashtagEntity, position: Int) - extends WrappedEntity(entity.fromIndex, entity.toIndex, entity.text) { - override def shift(fromIndex: Short, toIndex: Short): WrappedHashtagEntity = - copy(entity.copy(fromIndex = fromIndex, toIndex = toIndex)) - } - - private case class WrappedCashtagEntity(entity: CashtagEntity, position: Int) - extends WrappedEntity(entity.fromIndex, entity.toIndex, entity.text) { - override def shift(fromIndex: Short, toIndex: Short): WrappedCashtagEntity = - copy(entity.copy(fromIndex = fromIndex, toIndex = toIndex)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CreatedAtRepairer.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CreatedAtRepairer.docx new file mode 100644 index 000000000..5ce33f20e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CreatedAtRepairer.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CreatedAtRepairer.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CreatedAtRepairer.scala deleted file mode 100644 index 88d3fca3e..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/CreatedAtRepairer.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.snowflake.id.SnowflakeId - -object CreatedAtRepairer { - // no createdAt value should be less than this - val jan_01_2006 = 1136073600000L - - // no non-snowflake createdAt value should be greater than this - val jan_01_2011 = 1293840000000L - - // allow createdAt timestamp to be up to this amount off from the snowflake id - // before applying the correction. - val varianceThreshold: MediaId = 10.minutes.inMilliseconds -} - -/** - * Detects tweets with bad createdAt timestamps and attempts to fix, if possible - * using the snowflake id. pre-snowflake tweets are left unmodified. - */ -class CreatedAtRepairer(scribe: FutureEffect[String]) extends Mutation[Tweet] { - import CreatedAtRepairer._ - - def apply(tweet: Tweet): Option[Tweet] = { - assert(tweet.coreData.nonEmpty, "tweet core data is missing") - val createdAtMillis = getCreatedAt(tweet) * 1000 - - if (SnowflakeId.isSnowflakeId(tweet.id)) { - val snowflakeMillis = SnowflakeId(tweet.id).unixTimeMillis.asLong - val diff = (snowflakeMillis - createdAtMillis).abs - - if (diff >= varianceThreshold) { - scribe(tweet.id + "\t" + createdAtMillis) - val snowflakeSeconds = snowflakeMillis / 1000 - Some(TweetLenses.createdAt.set(tweet, snowflakeSeconds)) - } else { - None - } - } else { - // not a snowflake id, hard to repair, so just log it - if (createdAtMillis < jan_01_2006 || createdAtMillis > jan_01_2011) { - scribe(tweet.id + "\t" + createdAtMillis) - } - None - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DeviceSourceHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DeviceSourceHydrator.docx new file mode 100644 index 000000000..38d12a344 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DeviceSourceHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DeviceSourceHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DeviceSourceHydrator.scala deleted file mode 100644 index c1a0c5fcd..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DeviceSourceHydrator.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.serverutil.DeviceSourceParser -import com.twitter.tweetypie.thriftscala.DeviceSource -import com.twitter.tweetypie.thriftscala.FieldByPath - -object DeviceSourceHydrator { - type Type = ValueHydrator[Option[DeviceSource], TweetCtx] - - // WebOauthId is the created_via value for Macaw-Swift through Woodstar. - // We need to special-case it to return the same device_source as "web", - // since we can't map multiple created_via strings to one device_source. - val WebOauthId: String = s"oauth:${DeviceSourceParser.Web}" - - val hydratedField: FieldByPath = fieldByPath(Tweet.DeviceSourceField) - - private def convertForWeb(createdVia: String) = - if (createdVia == DeviceSourceHydrator.WebOauthId) "web" else createdVia - - def apply(repo: DeviceSourceRepository.Type): Type = - ValueHydrator[Option[DeviceSource], TweetCtx] { (_, ctx) => - val req = convertForWeb(ctx.createdVia) - repo(req).liftToTry.map { - case Return(deviceSource) => ValueState.modified(Some(deviceSource)) - case Throw(NotFound) => ValueState.UnmodifiedNone - case Throw(_) => ValueState.partial(None, hydratedField) - } - }.onlyIf((curr, ctx) => curr.isEmpty && ctx.tweetFieldRequested(Tweet.DeviceSourceField)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DirectedAtHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DirectedAtHydrator.docx new file mode 100644 index 000000000..c28c47a0e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DirectedAtHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DirectedAtHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DirectedAtHydrator.scala deleted file mode 100644 index a64d91c2e..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/DirectedAtHydrator.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -/** - * Hydrates the "directedAtUser" field on the tweet. This hydrators uses one of two paths depending - * if DirectedAtUserMetadata is present: - * - * 1. If DirectedAtUserMetadata exists, we use metadata.userId. - * 2. If DirectedAtUserMetadata does not exist, we use the User screenName from the mention starting - * at index 0 if the tweet also has a reply. Creation of a "reply to user" for - * leading @mentions is controlled by PostTweetRequest.enableTweetToNarrowcasting - */ -object DirectedAtHydrator { - type Type = ValueHydrator[Option[DirectedAtUser], Ctx] - - case class Ctx( - mentions: Seq[MentionEntity], - metadata: Option[DirectedAtUserMetadata], - underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy { - val directedAtScreenName: Option[String] = - mentions.headOption.filter(_.fromIndex == 0).map(_.screenName) - } - - val hydratedField: FieldByPath = - fieldByPath(Tweet.CoreDataField, TweetCoreData.DirectedAtUserField) - - def once(h: Type): Type = - TweetHydration.completeOnlyOnce( - hydrationType = HydrationType.DirectedAt, - hydrator = h - ) - - private val partial = ValueState.partial(None, hydratedField) - - def apply(repo: UserIdentityRepository.Type, stats: StatsReceiver = NullStatsReceiver): Type = { - val withMetadata = stats.counter("with_metadata") - val noScreenName = stats.counter("no_screen_name") - val withoutMetadata = stats.counter("without_metadata") - - ValueHydrator[Option[DirectedAtUser], Ctx] { (_, ctx) => - ctx.metadata match { - case Some(DirectedAtUserMetadata(Some(uid))) => - // 1a. new approach of relying exclusively on directed-at metadata if it exists and has a user id - withMetadata.incr() - - repo(UserKey.byId(uid)).liftToTry.map { - case Return(u) => - ValueState.modified(Some(DirectedAtUser(u.id, u.screenName))) - case Throw(NotFound) => - // If user is not found, fallback to directedAtScreenName - ctx.directedAtScreenName - .map { screenName => ValueState.modified(Some(DirectedAtUser(uid, screenName))) } - .getOrElse { - // This should never happen, but let's make sure with a counter - noScreenName.incr() - ValueState.UnmodifiedNone - } - case Throw(_) => partial - } - - case Some(DirectedAtUserMetadata(None)) => - withMetadata.incr() - // 1b. new approach of relying exclusively on directed-at metadata if it exists and has no userId - ValueState.StitchUnmodifiedNone - - case None => - // 2. when DirectedAtUserMetadata not present, look for first leading mention when has reply - withoutMetadata.incr() - - val userKey = ctx.directedAtScreenName - .filter(_ => ctx.isReply) - .map(UserKey.byScreenName) - - val results = userKey.map(repo.apply).getOrElse(Stitch.NotFound) - - results.liftToTry.map { - case Return(u) => ValueState.modified(Some(DirectedAtUser(u.id, u.screenName))) - case Throw(NotFound) => ValueState.UnmodifiedNone - case Throw(_) => partial - } - } - }.onlyIf((curr, _) => curr.isEmpty) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditControlHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditControlHydrator.docx new file mode 100644 index 000000000..9a85dc4de Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditControlHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditControlHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditControlHydrator.scala deleted file mode 100644 index 8d3c5d8e2..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditControlHydrator.scala +++ /dev/null @@ -1,132 +0,0 @@ -package com.twitter.tweetypie.hydrator - -import com.twitter.servo.util.Gate -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.StatsReceiver -import com.twitter.tweetypie.Tweet -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.repository.TweetRepository -import com.twitter.tweetypie.util.EditControlUtil -import com.twitter.tweetypie.serverutil.ExceptionCounter -import com.twitter.tweetypie.thriftscala.EditControl -import com.twitter.tweetypie.thriftscala.EditControlInitial -import com.twitter.tweetypie.thriftscala.FieldByPath -import com.twitter.tweetypie.util.TweetEditFailure.TweetEditGetInitialEditControlException -import com.twitter.tweetypie.util.TweetEditFailure.TweetEditInvalidEditControlException - -/** - * EditControlHydrator is used to hydrate the EditControlEdit arm of the editControl field. - * - * For Tweets without edits and for initial Tweets with subsequent edit(s), this hydrator - * passes through the existing editControl (either None or EditControlInitial). - * - * For edit Tweets, it hydrates the initial Tweet's edit control, set as a field on - * the edit control of the edit Tweet and returns the new edit control. - */ -object EditControlHydrator { - type Type = ValueHydrator[Option[EditControl], TweetCtx] - - val hydratedField: FieldByPath = fieldByPath(Tweet.EditControlField) - - def apply( - repo: TweetRepository.Type, - setEditTimeWindowToSixtyMinutes: Gate[Unit], - stats: StatsReceiver - ): Type = { - val exceptionCounter = ExceptionCounter(stats) - - // Count hydration of edit control for tweets that were written before writing edit control initial. - val noEditControlHydration = stats.counter("noEditControlHydration") - // Count hydration of edit control edit tweets - val editControlEditHydration = stats.counter("editControlEditHydration") - // Count edit control edit hydration which successfully found an edit control initial - val editControlEditHydrationSuccessful = stats.counter("editControlEditHydration", "success") - // Count of initial tweets being hydrated. - val editControlInitialHydration = stats.counter("editControlInitialHydration") - // Count of edits loaded where the ID of edit is not present in the initial tweet - val editTweetIdsMissingAnEdit = stats.counter("editTweetIdsMissingAnEdit") - // Count hydrated tweets where edit control is set, but neither initial nor edit - val unknownUnionVariant = stats.counter("unknownEditControlUnionVariant") - - ValueHydrator[Option[EditControl], TweetCtx] { (curr, ctx) => - curr match { - // Tweet was created before we write edit control - hydrate the value at read. - case None => - noEditControlHydration.incr() - val editControl = EditControlUtil.makeEditControlInitial( - ctx.tweetId, - ctx.createdAt, - setEditTimeWindowToSixtyMinutes) - Stitch.value(ValueState.delta(curr, Some(editControl))) - // Tweet is an initial tweet - case Some(EditControl.Initial(_)) => - editControlInitialHydration.incr() - Stitch.value(ValueState.unmodified(curr)) - - // Tweet is an edited version - case Some(EditControl.Edit(edit)) => - editControlEditHydration.incr() - getInitialTweet(repo, edit.initialTweetId, ctx) - .flatMap(getEditControlInitial(ctx)) - .map { initial: Option[EditControlInitial] => - editControlEditHydrationSuccessful.incr() - - initial.foreach { initialTweet => - // We are able to fetch the initial tweet for this edit but this edit tweet is - // not present in the initial's editTweetIds list - if (!initialTweet.editTweetIds.contains(ctx.tweetId)) { - editTweetIdsMissingAnEdit.incr() - } - } - - val updated = edit.copy(editControlInitial = initial) - ValueState.delta(curr, Some(EditControl.Edit(updated))) - } - .onFailure(exceptionCounter(_)) - case Some(_) => // Unknown union variant - unknownUnionVariant.incr() - Stitch.exception(TweetEditInvalidEditControlException) - } - }.onlyIf { (_, ctx) => ctx.opts.enableEditControlHydration } - } - - def getInitialTweet( - repo: TweetRepository.Type, - initialTweetId: Long, - ctx: TweetCtx, - ): Stitch[Tweet] = { - val options = TweetQuery.Options( - include = TweetQuery.Include(Set(Tweet.EditControlField.id)), - cacheControl = ctx.opts.cacheControl, - enforceVisibilityFiltering = false, - safetyLevel = SafetyLevel.FilterNone, - fetchStoredTweets = ctx.opts.fetchStoredTweets - ) - repo(initialTweetId, options) - } - - def getEditControlInitial(ctx: TweetCtx): Tweet => Stitch[Option[EditControlInitial]] = { - initialTweet: Tweet => - initialTweet.editControl match { - case Some(EditControl.Initial(initial)) => - Stitch.value( - if (ctx.opts.cause.writing(ctx.tweetId)) { - // On the write path we hydrate edit control initial - // as if the initial tweet is already updated. - Some(EditControlUtil.plusEdit(initial, ctx.tweetId)) - } else { - Some(initial) - } - ) - case _ if ctx.opts.fetchStoredTweets => - // If the fetchStoredTweets parameter is set to true, it means we're fetching - // and hydrating tweets regardless of state. In this case, if the initial tweet - // doesn't exist, we return None here to ensure we still hydrate and return the - // current edit tweet. - Stitch.None - case _ => Stitch.exception(TweetEditGetInitialEditControlException) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditHydrator.docx new file mode 100644 index 000000000..141f76f80 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditHydrator.scala deleted file mode 100644 index d14dad52c..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditHydrator.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.EditState - -/** - * An EditHydrator hydrates a value of type `A`, with a hydration context of type `C`, - * and produces a function that takes a value and context and returns an EditState[A, C] - * (an EditState encapsulates a function that takes a value and returns a new ValueState). - * - * A series of EditHydrators of the same type may be run in parallel via - * `EditHydrator.inParallel`. - */ -class EditHydrator[A, C] private (val run: (A, C) => Stitch[EditState[A]]) { - - /** - * Apply this hydrator to a value, producing an EditState. - */ - def apply(a: A, ctx: C): Stitch[EditState[A]] = run(a, ctx) - - /** - * Convert this EditHydrator to the equivalent ValueHydrator. - */ - def toValueHydrator: ValueHydrator[A, C] = - ValueHydrator[A, C] { (a, ctx) => this.run(a, ctx).map(editState => editState.run(a)) } - - /** - * Runs two EditHydrators in parallel. - */ - def inParallelWith(next: EditHydrator[A, C]): EditHydrator[A, C] = - EditHydrator[A, C] { (x0, ctx) => - Stitch.joinMap(run(x0, ctx), next.run(x0, ctx)) { - case (r1, r2) => r1.andThen(r2) - } - } -} - -object EditHydrator { - - /** - * Create an EditHydrator from a function that returns Stitch[EditState[A]]. - */ - def apply[A, C](f: (A, C) => Stitch[EditState[A]]): EditHydrator[A, C] = - new EditHydrator[A, C](f) - - /** - * Creates a "passthrough" Edit: - * Leaves A unchanged and produces empty HydrationState. - */ - def unit[A, C]: EditHydrator[A, C] = - EditHydrator { (_, _) => Stitch.value(EditState.unit[A]) } - - /** - * Runs several EditHydrators in parallel. - */ - def inParallel[A, C](bs: EditHydrator[A, C]*): EditHydrator[A, C] = - bs match { - case Seq(b) => b - case Seq(b1, b2) => b1.inParallelWith(b2) - case _ => bs.reduceLeft(_.inParallelWith(_)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditPerspectiveHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditPerspectiveHydrator.docx new file mode 100644 index 000000000..0eb4fd3e8 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditPerspectiveHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditPerspectiveHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditPerspectiveHydrator.scala deleted file mode 100644 index bc6ed36ef..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EditPerspectiveHydrator.scala +++ /dev/null @@ -1,179 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.featureswitches.v2.FeatureSwitchResults -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives.Query -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.repository.PerspectiveRepository -import com.twitter.tweetypie.thriftscala.EditControl -import com.twitter.tweetypie.thriftscala.FieldByPath -import com.twitter.tweetypie.thriftscala.StatusPerspective -import com.twitter.tweetypie.thriftscala.TweetPerspective - -object EditPerspectiveHydrator { - - type Type = ValueHydrator[Option[TweetPerspective], Ctx] - val HydratedField: FieldByPath = fieldByPath(Tweet.EditPerspectiveField) - - case class Ctx( - currentTweetPerspective: Option[StatusPerspective], - editControl: Option[EditControl], - featureSwitchResults: Option[FeatureSwitchResults], - underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - // Timeline safety levels determine some part of high level traffic - // that we might want to turn off with a decider if edits traffic - // is too big for perspectives to handle. The decider allows us - // to turn down the traffic without the impact on tweet detail. - val TimelinesSafetyLevels: Set[SafetyLevel] = Set( - SafetyLevel.TimelineFollowingActivity, - SafetyLevel.TimelineHome, - SafetyLevel.TimelineConversations, - SafetyLevel.DeprecatedTimelineConnect, - SafetyLevel.TimelineMentions, - SafetyLevel.DeprecatedTimelineActivity, - SafetyLevel.TimelineFavorites, - SafetyLevel.TimelineLists, - SafetyLevel.TimelineInjection, - SafetyLevel.StickersTimeline, - SafetyLevel.LiveVideoTimeline, - SafetyLevel.QuoteTweetTimeline, - SafetyLevel.TimelineHomeLatest, - SafetyLevel.TimelineLikedBy, - SafetyLevel.TimelineRetweetedBy, - SafetyLevel.TimelineBookmark, - SafetyLevel.TimelineMedia, - SafetyLevel.TimelineReactiveBlending, - SafetyLevel.TimelineProfile, - SafetyLevel.TimelineFocalTweet, - SafetyLevel.TimelineHomeRecommendations, - SafetyLevel.NotificationsTimelineDeviceFollow, - SafetyLevel.TimelineConversationsDownranking, - SafetyLevel.TimelineHomeTopicFollowRecommendations, - SafetyLevel.TimelineHomeHydration, - SafetyLevel.FollowedTopicsTimeline, - SafetyLevel.ModeratedTweetsTimeline, - SafetyLevel.TimelineModeratedTweetsHydration, - SafetyLevel.ElevatedQuoteTweetTimeline, - SafetyLevel.TimelineConversationsDownrankingMinimal, - SafetyLevel.BirdwatchNoteTweetsTimeline, - SafetyLevel.TimelineSuperLikedBy, - SafetyLevel.UserScopedTimeline, - SafetyLevel.TweetScopedTimeline, - SafetyLevel.TimelineHomePromotedHydration, - SafetyLevel.NearbyTimeline, - SafetyLevel.TimelineProfileAll, - SafetyLevel.TimelineProfileSuperFollows, - SafetyLevel.SpaceTweetAvatarHomeTimeline, - SafetyLevel.SpaceHomeTimelineUpranking, - SafetyLevel.BlockMuteUsersTimeline, - SafetyLevel.RitoActionedTweetTimeline, - SafetyLevel.TimelineScorer, - SafetyLevel.ArticleTweetTimeline, - SafetyLevel.DesQuoteTweetTimeline, - SafetyLevel.EditHistoryTimeline, - SafetyLevel.DirectMessagesConversationTimeline, - SafetyLevel.DesHomeTimeline, - SafetyLevel.TimelineContentControls, - SafetyLevel.TimelineFavoritesSelfView, - SafetyLevel.TimelineProfileSpaces, - ) - val TweetDetailSafetyLevels: Set[SafetyLevel] = Set( - SafetyLevel.TweetDetail, - SafetyLevel.TweetDetailNonToo, - SafetyLevel.TweetDetailWithInjectionsHydration, - SafetyLevel.DesTweetDetail, - ) - - def apply( - repo: PerspectiveRepository.Type, - timelinesGate: Gate[Unit], - tweetDetailsGate: Gate[Unit], - otherSafetyLevelsGate: Gate[Unit], - bookmarksGate: Gate[Long], - stats: StatsReceiver - ): Type = { - - val statsByLevel = - SafetyLevel.list.map { level => - (level, stats.counter("perspective_by_safety_label", level.name, "calls")) - }.toMap - val editsAggregated = stats.counter("edit_perspective", "edits_aggregated") - - ValueHydrator[Option[TweetPerspective], Ctx] { (curr, ctx) => - val safetyLevel = ctx.opts.safetyLevel - val lookupsDecider = - if (TimelinesSafetyLevels.contains(safetyLevel)) timelinesGate - else if (TweetDetailSafetyLevels.contains(safetyLevel)) tweetDetailsGate - else otherSafetyLevelsGate - - val tweetIds: Seq[TweetId] = if (lookupsDecider()) tweetIdsToAggregate(ctx).toSeq else Seq() - statsByLevel - .getOrElse( - safetyLevel, - stats.counter("perspective_by_safety_label", safetyLevel.name, "calls")) - .incr(tweetIds.size) - editsAggregated.incr(tweetIds.size) - - Stitch - .traverse(tweetIds) { id => - repo( - Query( - ctx.opts.forUserId.get, - id, - PerspectiveHydrator.evaluatePerspectiveTypes( - ctx.opts.forUserId.get, - bookmarksGate, - ctx.featureSwitchResults))).liftToTry - }.map { seq => - if (seq.isEmpty) { - val editPerspective = ctx.currentTweetPerspective.map { c => - TweetPerspective( - c.favorited, - c.retweeted, - c.bookmarked - ) - } - ValueState.delta(curr, editPerspective) - } else { - val returns = seq.collect { case Return(r) => r } - val aggregate = Some( - TweetPerspective( - favorited = - returns.exists(_.favorited) || ctx.currentTweetPerspective.exists(_.favorited), - retweeted = - returns.exists(_.retweeted) || ctx.currentTweetPerspective.exists(_.retweeted), - bookmarked = Some( - returns.exists(_.bookmarked.contains(true)) || ctx.currentTweetPerspective.exists( - _.bookmarked.contains(true))) - ) - ) - - if (seq.exists(_.isThrow)) { - ValueState.partial(aggregate, HydratedField) - } else { - ValueState.modified(aggregate) - } - } - } - }.onlyIf { (curr, ctx) => - curr.isEmpty && - ctx.opts.forUserId.isDefined && - ctx.tweetFieldRequested(Tweet.EditPerspectiveField) - } - } - - private def tweetIdsToAggregate(ctx: Ctx): Set[TweetId] = { - ctx.editControl - .flatMap { - case EditControl.Initial(initial) => Some(initial) - case EditControl.Edit(edit) => edit.editControlInitial - case _ => None - } - .map(_.editTweetIds.toSet) - .getOrElse(Set()) - ctx.tweetId - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EscherbirdAnnotationHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EscherbirdAnnotationHydrator.docx new file mode 100644 index 000000000..03cfd39a5 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EscherbirdAnnotationHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EscherbirdAnnotationHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EscherbirdAnnotationHydrator.scala deleted file mode 100644 index 578af57e5..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/EscherbirdAnnotationHydrator.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala.EscherbirdEntityAnnotations -import com.twitter.tweetypie.thriftscala.FieldByPath - -object EscherbirdAnnotationHydrator { - type Type = ValueHydrator[Option[EscherbirdEntityAnnotations], Tweet] - - val hydratedField: FieldByPath = fieldByPath(Tweet.EscherbirdEntityAnnotationsField) - - def apply(repo: EscherbirdAnnotationRepository.Type): Type = - ValueHydrator[Option[EscherbirdEntityAnnotations], Tweet] { (curr, tweet) => - repo(tweet).liftToTry.map { - case Return(Some(anns)) => ValueState.modified(Some(anns)) - case Return(None) => ValueState.unmodified(curr) - case Throw(_) => ValueState.partial(curr, hydratedField) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/FeatureSwitchResultsHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/FeatureSwitchResultsHydrator.docx new file mode 100644 index 000000000..4a4a8b0f3 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/FeatureSwitchResultsHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/FeatureSwitchResultsHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/FeatureSwitchResultsHydrator.scala deleted file mode 100644 index 8931f153c..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/FeatureSwitchResultsHydrator.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.context.thriftscala.Viewer -import com.twitter.featureswitches.FSRecipient -import com.twitter.featureswitches.UserAgent -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.finagle.mtls.authentication.EmptyServiceIdentifier -import com.twitter.strato.callcontext.CallContext -import com.twitter.tweetypie.client_id.ClientIdHelper -import com.twitter.tweetypie.core.ValueState - -/** - * Hydrate Feature Switch results in TweetData. We can do this once at the - * start of the hydration pipeline so that the rest of the hydrators can - * use the Feature Switch values. - */ -object FeatureSwitchResultsHydrator { - - def apply( - featureSwitchesWithoutExperiments: FeatureSwitches, - clientIdHelper: ClientIdHelper - ): TweetDataValueHydrator = ValueHydrator.map { (td, opts) => - val viewer = TwitterContext().getOrElse(Viewer()) - val recipient = - FSRecipient( - userId = viewer.userId, - clientApplicationId = viewer.clientApplicationId, - userAgent = viewer.userAgent.flatMap(UserAgent(_)), - ).withCustomFields( - "thrift_client_id" -> - clientIdHelper.effectiveClientIdRoot.getOrElse(ClientIdHelper.UnknownClientId), - "forwarded_service_id" -> - CallContext.forwardedServiceIdentifier - .map(_.toString).getOrElse(EmptyServiceIdentifier), - "safety_level" -> opts.safetyLevel.toString, - "client_app_id_is_defined" -> viewer.clientApplicationId.isDefined.toString, - ) - val results = featureSwitchesWithoutExperiments.matchRecipient(recipient) - ValueState.unit(td.copy(featureSwitchResults = Some(results))) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/GeoScrubHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/GeoScrubHydrator.docx new file mode 100644 index 000000000..37dbff446 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/GeoScrubHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/GeoScrubHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/GeoScrubHydrator.scala deleted file mode 100644 index b53c24497..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/GeoScrubHydrator.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -/** - * This hydrator, which is really more of a "repairer", scrubs at read-time geo data - * that should have been scrubbed but wasn't. For any tweet with geo data, it checks - * the last geo-scrub timestamp, if any, for the user, and if the tweet was created before - * that timestamp, it removes the geo data. - */ -object GeoScrubHydrator { - type Data = (Option[GeoCoordinates], Option[PlaceId]) - type Type = ValueHydrator[Data, TweetCtx] - - private[this] val modifiedNoneNoneResult = ValueState.modified((None, None)) - - def apply(repo: GeoScrubTimestampRepository.Type, scribeTweetId: FutureEffect[TweetId]): Type = - ValueHydrator[Data, TweetCtx] { (curr, ctx) => - repo(ctx.userId).liftToTry.map { - case Return(geoScrubTime) if ctx.createdAt <= geoScrubTime => - scribeTweetId(ctx.tweetId) - modifiedNoneNoneResult - - // no-op on failure and no result - case _ => ValueState.unmodified(curr) - } - }.onlyIf { case ((coords, place), _) => coords.nonEmpty || place.nonEmpty } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/HasMediaHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/HasMediaHydrator.docx new file mode 100644 index 000000000..a257e0790 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/HasMediaHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/HasMediaHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/HasMediaHydrator.scala deleted file mode 100644 index 486a6ee23..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/HasMediaHydrator.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.thriftscala._ - -object HasMediaHydrator { - type Type = ValueHydrator[Option[Boolean], Tweet] - - def apply(hasMedia: Tweet => Boolean): Type = - ValueHydrator - .map[Option[Boolean], Tweet] { (_, tweet) => ValueState.modified(Some(hasMedia(tweet))) } - .onlyIf((curr, ctx) => curr.isEmpty) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM1837FilterHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM1837FilterHydrator.docx new file mode 100644 index 000000000..0139d563a Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM1837FilterHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM1837FilterHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM1837FilterHydrator.scala deleted file mode 100644 index 951aa40c9..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM1837FilterHydrator.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.coreservices.IM1837 -import com.twitter.tweetypie.core._ -import com.twitter.stitch.Stitch - -object IM1837FilterHydrator { - type Type = ValueHydrator[Unit, TweetCtx] - - private val Drop = - Stitch.exception(FilteredState.Unavailable.DropUnspecified) - private val Success = Stitch.value(ValueState.unmodified(())) - - def apply(): Type = - ValueHydrator[Unit, TweetCtx] { (_, ctx) => - val userAgent = TwitterContext().flatMap(_.userAgent) - val userAgentAffected = userAgent.exists(IM1837.isAffectedClient) - val mightCrash = userAgentAffected && IM1837.textMightCrashIOS(ctx.text) - - if (mightCrash) Drop else Success - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM2884FilterHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM2884FilterHydrator.docx new file mode 100644 index 000000000..2b4e1d69c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM2884FilterHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM2884FilterHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM2884FilterHydrator.scala deleted file mode 100644 index 16222dec4..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM2884FilterHydrator.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.tweetypie.hydrator - -import com.twitter.coreservices.IM2884 -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.tweetypie.core.FilteredState -import com.twitter.tweetypie.core.ValueState -import com.twitter.stitch.Stitch - -object IM2884FilterHydrator { - type Type = ValueHydrator[Unit, TweetCtx] - - private val Drop = - Stitch.exception(FilteredState.Unavailable.DropUnspecified) - private val Success = Stitch.value(ValueState.unmodified(())) - - def apply(stats: StatsReceiver): Type = { - - val im2884 = new IM2884(stats) - - ValueHydrator[Unit, TweetCtx] { (_, ctx) => - val userAgent = TwitterContext().flatMap(_.userAgent) - val userAgentAffected = userAgent.exists(im2884.isAffectedClient) - val mightCrash = userAgentAffected && im2884.textMightCrashIOS(ctx.text) - if (mightCrash) Drop else Success - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM3433FilterHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM3433FilterHydrator.docx new file mode 100644 index 000000000..143f18578 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM3433FilterHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM3433FilterHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM3433FilterHydrator.scala deleted file mode 100644 index 71ee6139d..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/IM3433FilterHydrator.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.tweetypie.hydrator - -import com.twitter.coreservices.IM3433 -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.FilteredState -import com.twitter.tweetypie.core.ValueState - -object IM3433FilterHydrator { - type Type = ValueHydrator[Unit, TweetCtx] - - private val Drop = - Stitch.exception(FilteredState.Unavailable.DropUnspecified) - private val Success = Stitch.value(ValueState.unmodified(())) - - def apply(stats: StatsReceiver): Type = { - - ValueHydrator[Unit, TweetCtx] { (_, ctx) => - val userAgent = TwitterContext().flatMap(_.userAgent) - val userAgentAffected = userAgent.exists(IM3433.isAffectedClient) - val mightCrash = userAgentAffected && IM3433.textMightCrashIOS(ctx.text) - if (mightCrash) Drop else Success - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/LanguageHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/LanguageHydrator.docx new file mode 100644 index 000000000..86f838b57 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/LanguageHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/LanguageHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/LanguageHydrator.scala deleted file mode 100644 index 2a86091b9..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/LanguageHydrator.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object LanguageHydrator { - type Type = ValueHydrator[Option[Language], TweetCtx] - - val hydratedField: FieldByPath = fieldByPath(Tweet.LanguageField) - - private[this] def isApplicable(curr: Option[Language], ctx: TweetCtx) = - ctx.tweetFieldRequested(Tweet.LanguageField) && !ctx.isRetweet && curr.isEmpty - - def apply(repo: LanguageRepository.Type): Type = - ValueHydrator[Option[Language], TweetCtx] { (langOpt, ctx) => - repo(ctx.text).liftToTry.map { - case Return(Some(l)) => ValueState.modified(Some(l)) - case Return(None) => ValueState.unmodified(langOpt) - case Throw(_) => ValueState.partial(None, hydratedField) - } - }.onlyIf((curr, ctx) => isApplicable(curr, ctx)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaEntityHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaEntityHydrator.docx new file mode 100644 index 000000000..c2e760674 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaEntityHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaEntityHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaEntityHydrator.scala deleted file mode 100644 index 3f3e63fe2..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaEntityHydrator.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.mediaservices.commons.thriftscala.MediaKey -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object MediaEntitiesHydrator { - object Cacheable { - type Ctx = MediaEntityHydrator.Cacheable.Ctx - type Type = ValueHydrator[Seq[MediaEntity], Ctx] - - def once(h: MediaEntityHydrator.Cacheable.Type): Type = - TweetHydration.completeOnlyOnce( - queryFilter = MediaEntityHydrator.queryFilter, - hydrationType = HydrationType.CacheableMedia, - dependsOn = Set(HydrationType.Urls), - hydrator = h.liftSeq - ) - } - - object Uncacheable { - type Ctx = MediaEntityHydrator.Uncacheable.Ctx - type Type = ValueHydrator[Seq[MediaEntity], Ctx] - } -} - -object MediaEntityHydrator { - val hydratedField: FieldByPath = fieldByPath(Tweet.MediaField) - - object Cacheable { - type Type = ValueHydrator[MediaEntity, Ctx] - - case class Ctx(urlEntities: Seq[UrlEntity], underlyingTweetCtx: TweetCtx) extends TweetCtx.Proxy - - /** - * Builds a single media-hydrator out of finer-grained hydrators - * only with cacheable information. - */ - def apply(hydrateMediaUrls: Type, hydrateMediaIsProtected: Type): Type = - hydrateMediaUrls.andThen(hydrateMediaIsProtected) - } - - object Uncacheable { - type Type = ValueHydrator[MediaEntity, Ctx] - - case class Ctx(mediaKeys: Option[Seq[MediaKey]], underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy { - - def includeMediaEntities: Boolean = tweetFieldRequested(Tweet.MediaField) - def includeAdditionalMetadata: Boolean = - opts.include.mediaFields.contains(MediaEntity.AdditionalMetadataField.id) - } - - /** - * Builds a single media-hydrator out of finer-grained hydrators - * only with uncacheable information. - */ - def apply(hydrateMediaKey: Type, hydrateMediaInfo: Type): Type = - (hydrateMediaKey - .andThen(hydrateMediaInfo)) - .onlyIf((_, ctx) => ctx.includeMediaEntities) - } - - def queryFilter(opts: TweetQuery.Options): Boolean = - opts.include.tweetFields.contains(Tweet.MediaField.id) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaInfoHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaInfoHydrator.docx new file mode 100644 index 000000000..bfd3c7ecc Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaInfoHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaInfoHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaInfoHydrator.scala deleted file mode 100644 index 86e7d8e1a..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaInfoHydrator.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.media.MediaKeyUtil -import com.twitter.tweetypie.media.MediaMetadataRequest -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ -import java.nio.ByteBuffer - -object MediaInfoHydrator { - type Ctx = MediaEntityHydrator.Uncacheable.Ctx - type Type = MediaEntityHydrator.Uncacheable.Type - - private[this] val log = Logger(getClass) - - def apply(repo: MediaMetadataRepository.Type, stats: StatsReceiver): Type = { - val attributableUserCounter = stats.counter("attributable_user") - - ValueHydrator[MediaEntity, Ctx] { (curr, ctx) => - val request = - toMediaMetadataRequest( - mediaEntity = curr, - tweetId = ctx.tweetId, - extensionsArgs = ctx.opts.extensionsArgs - ) - - request match { - case None => Stitch.value(ValueState.unmodified(curr)) - - case Some(req) => - repo(req).liftToTry.map { - case Return(metadata) => - if (metadata.attributableUserId.nonEmpty) attributableUserCounter.incr() - - ValueState.delta( - curr, - metadata.updateEntity( - mediaEntity = curr, - tweetUserId = ctx.userId, - includeAdditionalMetadata = ctx.includeAdditionalMetadata - ) - ) - - case Throw(ex) if !PartialEntityCleaner.isPartialMedia(curr) => - log.info("Ignored media info repo failure, media entity already hydrated", ex) - ValueState.unmodified(curr) - - case Throw(ex) => - log.error("Media info hydration failed", ex) - ValueState.partial(curr, MediaEntityHydrator.hydratedField) - } - } - } - } - - def toMediaMetadataRequest( - mediaEntity: MediaEntity, - tweetId: TweetId, - extensionsArgs: Option[ByteBuffer] - ): Option[MediaMetadataRequest] = - mediaEntity.isProtected.map { isProtected => - val mediaKey = MediaKeyUtil.get(mediaEntity) - - MediaMetadataRequest( - tweetId = tweetId, - mediaKey = mediaKey, - isProtected = isProtected, - extensionsArgs = extensionsArgs - ) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaIsProtectedHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaIsProtectedHydrator.docx new file mode 100644 index 000000000..64306401d Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaIsProtectedHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaIsProtectedHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaIsProtectedHydrator.scala deleted file mode 100644 index 9ddfe5851..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaIsProtectedHydrator.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.media.Media -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object MediaIsProtectedHydrator { - type Ctx = MediaEntityHydrator.Cacheable.Ctx - type Type = MediaEntityHydrator.Cacheable.Type - - val hydratedField: FieldByPath = MediaEntityHydrator.hydratedField - - def apply(repo: UserProtectionRepository.Type): Type = - ValueHydrator[MediaEntity, Ctx] { (curr, ctx) => - val request = UserKey(ctx.userId) - - repo(request).liftToTry.map { - case Return(p) => ValueState.modified(curr.copy(isProtected = Some(p))) - case Throw(NotFound) => ValueState.unmodified(curr) - case Throw(_) => ValueState.partial(curr, hydratedField) - } - }.onlyIf { (curr, ctx) => - // We need to update isProtected for media entities that: - // 1. Do not already have it set. - // 2. Did not come from another tweet. - // - // If the entity does not have an expandedUrl, we can't be sure - // whether the media originated with this tweet. - curr.isProtected.isEmpty && - Media.isOwnMedia(ctx.tweetId, curr) && - curr.expandedUrl != null - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaKeyHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaKeyHydrator.docx new file mode 100644 index 000000000..4deac0653 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaKeyHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaKeyHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaKeyHydrator.scala deleted file mode 100644 index a6e491d61..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaKeyHydrator.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.tweetypie.hydrator - -import com.twitter.mediaservices.commons.tweetmedia.thriftscala._ -import com.twitter.mediaservices.commons.thriftscala._ -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.thriftscala._ - -object MediaKeyHydrator { - type Ctx = MediaEntityHydrator.Uncacheable.Ctx - type Type = MediaEntityHydrator.Uncacheable.Type - - def apply(): Type = - ValueHydrator - .map[MediaEntity, Ctx] { (curr, ctx) => - val mediaKey = infer(ctx.mediaKeys, curr) - ValueState.modified(curr.copy(mediaKey = Some(mediaKey))) - } - .onlyIf((curr, ctx) => curr.mediaKey.isEmpty) - - def infer(mediaKeys: Option[Seq[MediaKey]], mediaEntity: MediaEntity): MediaKey = { - - def inferByMediaId = - mediaKeys - .flatMap(_.find(_.mediaId == mediaEntity.mediaId)) - - def contentType = - mediaEntity.sizes.find(_.sizeType == MediaSizeType.Orig).map(_.deprecatedContentType) - - def inferByContentType = - contentType.map { tpe => - val category = - tpe match { - case MediaContentType.VideoMp4 => MediaCategory.TweetGif - case MediaContentType.VideoGeneric => MediaCategory.TweetVideo - case _ => MediaCategory.TweetImage - } - MediaKey(category, mediaEntity.mediaId) - } - - def fail = - throw new IllegalStateException( - s""" - |Can't infer media key. - | mediaKeys:'$mediaKeys' - | mediaEntity:'$mediaEntity' - """.stripMargin - ) - - mediaEntity.mediaKey - .orElse(inferByMediaId) - .orElse(inferByContentType) - .getOrElse(fail) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaRefsHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaRefsHydrator.docx new file mode 100644 index 000000000..8f39e45a9 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaRefsHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaRefsHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaRefsHydrator.scala deleted file mode 100644 index c2408b634..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaRefsHydrator.scala +++ /dev/null @@ -1,124 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.mediaservices.commons.thriftscala.MediaKey -import com.twitter.mediaservices.media_util.GenericMediaKey -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.thriftscala.MediaEntity -import com.twitter.tweetypie.thriftscala.UrlEntity -import com.twitter.tweetypie.media.thriftscala.MediaRef -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.repository.TweetRepository -import com.twitter.tweetypie.thriftscala.FieldByPath - -/** - * MediaRefsHydrator hydrates the Tweet.mediaRefs field based on stored media keys - * and pasted media. Media keys are available in three ways: - * - * 1. (For old Tweets): in the stored MediaEntity - * 2. (For 2016+ Tweets): in the mediaKeys field - * 3. From other Tweets using pasted media - * - * This hydrator combines these three sources into a single field, providing the - * media key and source Tweet information for pasted media. - * - * Long-term we will move this logic to the write path and backfill the field for old Tweets. - */ -object MediaRefsHydrator { - type Type = ValueHydrator[Option[Seq[MediaRef]], Ctx] - - case class Ctx( - media: Seq[MediaEntity], - mediaKeys: Seq[MediaKey], - urlEntities: Seq[UrlEntity], - underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy { - def includePastedMedia: Boolean = opts.include.pastedMedia - } - - val hydratedField: FieldByPath = fieldByPath(Tweet.MediaRefsField) - - def mediaKeyToMediaRef(mediaKey: MediaKey): MediaRef = - MediaRef( - genericMediaKey = GenericMediaKey(mediaKey).toStringKey() - ) - - // Convert a pasted Tweet into a Seq of MediaRef from that Tweet with the correct sourceTweetId and sourceUserId - def pastedTweetToMediaRefs( - tweet: Tweet - ): Seq[MediaRef] = - tweet.mediaRefs.toSeq.flatMap { mediaRefs => - mediaRefs.map( - _.copy( - sourceTweetId = Some(tweet.id), - sourceUserId = Some(getUserId(tweet)) - )) - } - - // Fetch MediaRefs from pasted media Tweet URLs in the Tweet text - def getPastedMediaRefs( - repo: TweetRepository.Optional, - ctx: Ctx, - includePastedMedia: Gate[Unit] - ): Stitch[Seq[MediaRef]] = { - if (includePastedMedia() && ctx.includePastedMedia) { - - // Extract Tweet ids from pasted media permalinks in the Tweet text - val pastedMediaTweetIds: Seq[TweetId] = - PastedMediaHydrator.pastedIdsAndEntities(ctx.tweetId, ctx.urlEntities).map(_._1) - - val opts = TweetQuery.Options( - include = TweetQuery.Include( - tweetFields = Set(Tweet.CoreDataField.id, Tweet.MediaRefsField.id), - pastedMedia = false // don't recursively load pasted media refs - )) - - // Load a Seq of Tweets with pasted media, ignoring any returned with NotFound or a FilteredState - val pastedTweets: Stitch[Seq[Tweet]] = Stitch - .traverse(pastedMediaTweetIds) { id => - repo(id, opts) - }.map(_.flatten) - - pastedTweets.map(_.flatMap(pastedTweetToMediaRefs)) - } else { - Stitch.Nil - } - } - - // Make empty Seq None and non-empty Seq Some(Seq(...)) to comply with the thrift field type - def optionalizeSeq(mediaRefs: Seq[MediaRef]): Option[Seq[MediaRef]] = - Some(mediaRefs).filterNot(_.isEmpty) - - def apply( - repo: TweetRepository.Optional, - includePastedMedia: Gate[Unit] - ): Type = { - ValueHydrator[Option[Seq[MediaRef]], Ctx] { (curr, ctx) => - // Fetch mediaRefs from Tweet media - val storedMediaRefs: Seq[MediaRef] = ctx.media.map { mediaEntity => - // Use MediaKeyHydrator.infer to determine the media key from the media entity - val mediaKey = MediaKeyHydrator.infer(Some(ctx.mediaKeys), mediaEntity) - mediaKeyToMediaRef(mediaKey) - } - - // Fetch mediaRefs from pasted media - getPastedMediaRefs(repo, ctx, includePastedMedia).liftToTry.map { - case Return(pastedMediaRefs) => - // Combine the refs from the Tweet's own media and those from pasted media, then limit - // to MaxMediaEntitiesPerTweet. - val limitedRefs = - (storedMediaRefs ++ pastedMediaRefs).take(PastedMediaHydrator.MaxMediaEntitiesPerTweet) - - ValueState.delta(curr, optionalizeSeq(limitedRefs)) - case Throw(_) => - ValueState.partial(optionalizeSeq(storedMediaRefs), hydratedField) - } - - }.onlyIf { (_, ctx) => - ctx.tweetFieldRequested(Tweet.MediaRefsField) || - ctx.opts.safetyLevel != SafetyLevel.FilterNone - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaTagsHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaTagsHydrator.docx new file mode 100644 index 000000000..b28748efe Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaTagsHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaTagsHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaTagsHydrator.scala deleted file mode 100644 index 4e3f1bc42..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaTagsHydrator.scala +++ /dev/null @@ -1,103 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object MediaTagsHydrator { - type Type = ValueHydrator[Option[TweetMediaTags], TweetCtx] - - /** - * TweetMediaTags contains a map of MediaId to Seq[MediaTag]. - * The outer traverse maps over each MediaId, while the inner - * traverse maps over each MediaTag. - * - * A MediaTag has four fields: - * - * 1: MediaTagType tag_type - * 2: optional i64 user_id - * 3: optional string screen_name - * 4: optional string name - * - * For each MediaTag, if the tag type is MediaTagType.User and the user id is defined - * (see mediaTagToKey) we look up the tagged user, using the tagging user (the tweet - * author) as the viewer id (this means that visibility rules between the tagged user - * and tagging user are applied). - * - * If we get a taggable user back, we fill in the screen name and name fields. If not, - * we drop the tag. - */ - def apply(repo: UserViewRepository.Type): Type = - ValueHydrator[TweetMediaTags, TweetCtx] { (tags, ctx) => - val mediaTagsByMediaId: Seq[(MediaId, Seq[MediaTag])] = tags.tagMap.toSeq - - Stitch - .traverse(mediaTagsByMediaId) { - case (mediaId, mediaTags) => - Stitch.traverse(mediaTags)(tag => hydrateMediaTag(repo, tag, ctx.userId)).map { - ValueState.sequence(_).map(tags => (mediaId, tags.flatten)) - } - } - .map { - // Reconstruct TweetMediaTags(tagMap: Map[MediaId, SeqMediaTag]) - ValueState.sequence(_).map(s => TweetMediaTags(s.toMap)) - } - }.onlyIf { (_, ctx) => - !ctx.isRetweet && ctx.tweetFieldRequested(Tweet.MediaTagsField) - }.liftOption - - /** - * A function to hydrate a single `MediaTag`. The return type is `Option[MediaTag]` - * because we may return `None` to filter out a `MediaTag` if the tagged user doesn't - * exist or isn't taggable. - */ - private[this] def hydrateMediaTag( - repo: UserViewRepository.Type, - mediaTag: MediaTag, - authorId: UserId - ): Stitch[ValueState[Option[MediaTag]]] = - mediaTagToKey(mediaTag) match { - case None => Stitch.value(ValueState.unmodified(Some(mediaTag))) - case Some(key) => - repo(toRepoQuery(key, authorId)) - .map { - case user if user.mediaView.exists(_.canMediaTag) => - ValueState.modified( - Some( - mediaTag.copy( - userId = Some(user.id), - screenName = user.profile.map(_.screenName), - name = user.profile.map(_.name) - ) - ) - ) - - // if `canMediaTag` is false, drop the tag - case _ => ValueState.modified(None) - } - .handle { - // if user is not found, drop the tag - case NotFound => ValueState.modified(None) - } - } - - private[this] val queryFields: Set[UserField] = Set(UserField.Profile, UserField.MediaView) - - def toRepoQuery(userKey: UserKey, forUserId: UserId): UserViewRepository.Query = - UserViewRepository.Query( - userKey = userKey, - // view is based on tagging user, not tweet viewer - forUserId = Some(forUserId), - visibility = UserVisibility.MediaTaggable, - queryFields = queryFields - ) - - private[this] def mediaTagToKey(mediaTag: MediaTag): Option[UserKey] = - mediaTag match { - case MediaTag(MediaTagType.User, Some(taggedUserId), _, _) => Some(UserKey(taggedUserId)) - case _ => None - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaUrlFieldsHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaUrlFieldsHydrator.docx new file mode 100644 index 000000000..430d6c2e3 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaUrlFieldsHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaUrlFieldsHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaUrlFieldsHydrator.scala deleted file mode 100644 index 0cacf3b74..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MediaUrlFieldsHydrator.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.media.Media -import com.twitter.tweetypie.media.MediaUrl -import com.twitter.tweetypie.thriftscala._ - -object MediaUrlFieldsHydrator { - type Ctx = MediaEntityHydrator.Cacheable.Ctx - type Type = MediaEntityHydrator.Cacheable.Type - - def mediaPermalink(ctx: Ctx): Option[UrlEntity] = - ctx.urlEntities.view.reverse.find(MediaUrl.Permalink.hasTweetId(_, ctx.tweetId)) - - def apply(): Type = - ValueHydrator - .map[MediaEntity, Ctx] { (curr, ctx) => - mediaPermalink(ctx) match { - case None => ValueState.unmodified(curr) - case Some(urlEntity) => ValueState.modified(Media.copyFromUrlEntity(curr, urlEntity)) - } - } - .onlyIf((curr, ctx) => curr.url == null && Media.isOwnMedia(ctx.tweetId, curr)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MentionEntityHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MentionEntityHydrator.docx new file mode 100644 index 000000000..8f23a5f5b Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MentionEntityHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MentionEntityHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MentionEntityHydrator.scala deleted file mode 100644 index a1d7c09cd..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/MentionEntityHydrator.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object MentionEntitiesHydrator { - type Type = ValueHydrator[Seq[MentionEntity], TweetCtx] - - def once(h: MentionEntityHydrator.Type): Type = - TweetHydration.completeOnlyOnce( - queryFilter = queryFilter, - hydrationType = HydrationType.Mentions, - hydrator = h.liftSeq - ) - - def queryFilter(opts: TweetQuery.Options): Boolean = - opts.include.tweetFields.contains(Tweet.MentionsField.id) -} - -object MentionEntityHydrator { - type Type = ValueHydrator[MentionEntity, TweetCtx] - - val hydratedField: FieldByPath = fieldByPath(Tweet.MentionsField) - - def apply(repo: UserIdentityRepository.Type): Type = - ValueHydrator[MentionEntity, TweetCtx] { (entity, _) => - repo(UserKey(entity.screenName)).liftToTry.map { - case Return(user) => ValueState.delta(entity, update(entity, user)) - case Throw(NotFound) => ValueState.unmodified(entity) - case Throw(_) => ValueState.partial(entity, hydratedField) - } - // only hydrate mention if userId or name is empty - }.onlyIf((entity, _) => entity.userId.isEmpty || entity.name.isEmpty) - - /** - * Updates a MentionEntity using the given user data. - */ - def update(entity: MentionEntity, userIdent: UserIdentity): MentionEntity = - entity.copy( - screenName = userIdent.screenName, - userId = Some(userIdent.id), - name = Some(userIdent.realName) - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NegativeVisibleTextRangeRepairer.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NegativeVisibleTextRangeRepairer.docx new file mode 100644 index 000000000..1c49cb4a0 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NegativeVisibleTextRangeRepairer.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NegativeVisibleTextRangeRepairer.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NegativeVisibleTextRangeRepairer.scala deleted file mode 100644 index 5babf5b88..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NegativeVisibleTextRangeRepairer.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.thriftscala.TextRange - -/** - * Some tweets with visibleTextRange may have fromIndex > toIndex, in which case set fromIndex - * to toIndex. - */ -object NegativeVisibleTextRangeRepairer { - private val mutation = - Mutation[Option[TextRange]] { - case Some(TextRange(from, to)) if from > to => Some(Some(TextRange(to, to))) - case _ => None - } - - private[tweetypie] val tweetMutation = TweetLenses.visibleTextRange.mutation(mutation) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NoteTweetSuffixHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NoteTweetSuffixHydrator.docx new file mode 100644 index 000000000..09cc20671 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NoteTweetSuffixHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NoteTweetSuffixHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NoteTweetSuffixHydrator.scala deleted file mode 100644 index c7224a8db..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/NoteTweetSuffixHydrator.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.TweetData -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.thriftscala.entities.Implicits._ -import com.twitter.tweetypie.thriftscala.TextRange -import com.twitter.tweetypie.tweettext.Offset -import com.twitter.tweetypie.tweettext.TextModification -import com.twitter.tweetypie.tweettext.TweetText -import com.twitter.tweetypie.util.TweetLenses - -object NoteTweetSuffixHydrator { - - val ELLIPSIS: String = "\u2026" - - private def addTextSuffix(tweet: Tweet): Tweet = { - val originalText = TweetLenses.text(tweet) - val originalTextLength = TweetText.codePointLength(originalText) - - val visibleTextRange: TextRange = - TweetLenses - .visibleTextRange(tweet) - .getOrElse(TextRange(0, originalTextLength)) - - val insertAtCodePoint = Offset.CodePoint(visibleTextRange.toIndex) - - val textModification = TextModification.insertAt( - originalText, - insertAtCodePoint, - ELLIPSIS - ) - - val mediaEntities = TweetLenses.media(tweet) - val urlEntities = TweetLenses.urls(tweet) - - val modifiedText = textModification.updated - val modifiedMediaEntities = textModification.reindexEntities(mediaEntities) - val modifiedUrlEntities = textModification.reindexEntities(urlEntities) - val modifiedVisibleTextRange = visibleTextRange.copy(toIndex = - visibleTextRange.toIndex + TweetText.codePointLength(ELLIPSIS)) - - val updatedTweet = - Lens.setAll( - tweet, - TweetLenses.text -> modifiedText, - TweetLenses.urls -> modifiedUrlEntities.sortBy(_.fromIndex), - TweetLenses.media -> modifiedMediaEntities.sortBy(_.fromIndex), - TweetLenses.visibleTextRange -> Some(modifiedVisibleTextRange) - ) - - updatedTweet - } - - def apply(): TweetDataValueHydrator = { - ValueHydrator[TweetData, TweetQuery.Options] { (td, _) => - val updatedTweet = addTextSuffix(td.tweet) - Stitch.value(ValueState.delta(td, td.copy(tweet = updatedTweet))) - }.onlyIf { (td, _) => - td.tweet.noteTweet.isDefined && - td.tweet.noteTweet.flatMap(_.isExpandable).getOrElse(true) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PartialEntityCleaner.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PartialEntityCleaner.docx new file mode 100644 index 000000000..495b6a7d1 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PartialEntityCleaner.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PartialEntityCleaner.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PartialEntityCleaner.scala deleted file mode 100644 index a15e64383..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PartialEntityCleaner.scala +++ /dev/null @@ -1,80 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.mediaservices.commons.tweetmedia.thriftscala._ -import com.twitter.tweetypie.media._ -import com.twitter.tweetypie.thriftscala._ -import scala.collection.Set - -/** - * Removes partial Url, Media, and Mention entities that were not - * fully hydrated. Rather than returning no value or a value with - * incomplete entities on an entity hydration failure, we gracefully - * degrade to just omitting those entities. This step needs to be - * applied in the post-cache filter, so that we don't cache the value - * with missing entities. - * - * A MediaEntity will first be converted back to a UrlEntity if it is only - * partially hydrated. If the resulting UrlEntity is itself then only partially - * hydrated, it will get dropped also. - */ -object PartialEntityCleaner { - def apply(stats: StatsReceiver): Mutation[Tweet] = { - val scopedStats = stats.scope("partial_entity_cleaner") - Mutation - .all( - Seq( - TweetLenses.urls.mutation(urls.countMutations(scopedStats.counter("urls"))), - TweetLenses.media.mutation(media.countMutations(scopedStats.counter("media"))), - TweetLenses.mentions.mutation(mentions.countMutations(scopedStats.counter("mentions"))) - ) - ) - .onlyIf(!isRetweet(_)) - } - - private[this] def clean[E](isPartial: E => Boolean) = - Mutation[Seq[E]] { items => - items.partition(isPartial) match { - case (Nil, nonPartial) => None - case (partial, nonPartial) => Some(nonPartial) - } - } - - private[this] val mentions = - clean[MentionEntity](e => e.userId.isEmpty || e.name.isEmpty) - - private[this] val urls = - clean[UrlEntity](e => - isNullOrEmpty(e.url) || isNullOrEmpty(e.expanded) || isNullOrEmpty(e.display)) - - private[this] val media = - Mutation[Seq[MediaEntity]] { mediaEntities => - mediaEntities.partition(isPartialMedia) match { - case (Nil, nonPartial) => None - case (partial, nonPartial) => Some(nonPartial) - } - } - - def isPartialMedia(e: MediaEntity): Boolean = - e.fromIndex < 0 || - e.toIndex <= 0 || - isNullOrEmpty(e.url) || - isNullOrEmpty(e.displayUrl) || - isNullOrEmpty(e.mediaUrl) || - isNullOrEmpty(e.mediaUrlHttps) || - isNullOrEmpty(e.expandedUrl) || - e.mediaInfo.isEmpty || - e.mediaKey.isEmpty || - (MediaKeyClassifier.isImage(MediaKeyUtil.get(e)) && containsInvalidSizeVariant(e.sizes)) - - private[this] val userMentions = - clean[UserMention](e => e.screenName.isEmpty || e.name.isEmpty) - - def isNullOrEmpty(optString: Option[String]): Boolean = - optString.isEmpty || optString.exists(isNullOrEmpty(_)) - - def isNullOrEmpty(str: String): Boolean = str == null || str.isEmpty - - def containsInvalidSizeVariant(sizes: Set[MediaSize]): Boolean = - sizes.exists(size => size.height == 0 || size.width == 0) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PastedMediaHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PastedMediaHydrator.docx new file mode 100644 index 000000000..31b8297b3 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PastedMediaHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PastedMediaHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PastedMediaHydrator.scala deleted file mode 100644 index 769c9bead..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PastedMediaHydrator.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.media.MediaUrl -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object PastedMediaHydrator { - type Type = ValueHydrator[PastedMedia, Ctx] - - /** - * Ensure that the final tweet has at most 4 media entities. - */ - val MaxMediaEntitiesPerTweet = 4 - - /** - * Enforce visibility rules when hydrating media for a write. - */ - val writeSafetyLevel = SafetyLevel.TweetWritesApi - - case class Ctx(urlEntities: Seq[UrlEntity], underlyingTweetCtx: TweetCtx) extends TweetCtx.Proxy { - def includePastedMedia: Boolean = opts.include.pastedMedia - def includeMediaEntities: Boolean = tweetFieldRequested(Tweet.MediaField) - def includeAdditionalMetadata: Boolean = - mediaFieldRequested(MediaEntity.AdditionalMetadataField.id) - def includeMediaTags: Boolean = tweetFieldRequested(Tweet.MediaTagsField) - } - - def getPastedMedia(t: Tweet): PastedMedia = PastedMedia(getMedia(t), Map.empty) - - def apply(repo: PastedMediaRepository.Type): Type = { - def hydrateOneReference( - tweetId: TweetId, - urlEntity: UrlEntity, - repoCtx: PastedMediaRepository.Ctx - ): Stitch[PastedMedia] = - repo(tweetId, repoCtx).liftToTry.map { - case Return(pastedMedia) => pastedMedia.updateEntities(urlEntity) - case _ => PastedMedia.empty - } - - ValueHydrator[PastedMedia, Ctx] { (curr, ctx) => - val repoCtx = asRepoCtx(ctx) - val idsAndEntities = pastedIdsAndEntities(ctx.tweetId, ctx.urlEntities) - - val res = Stitch.traverse(idsAndEntities) { - case (tweetId, urlEntity) => - hydrateOneReference(tweetId, urlEntity, repoCtx) - } - - res.liftToTry.map { - case Return(pastedMedias) => - val merged = pastedMedias.foldLeft(curr)(_.merge(_)) - val limited = merged.take(MaxMediaEntitiesPerTweet) - ValueState.delta(curr, limited) - - case Throw(_) => ValueState.unmodified(curr) - } - }.onlyIf { (_, ctx) => - // we only attempt to hydrate pasted media if media is requested - ctx.includePastedMedia && - !ctx.isRetweet && - ctx.includeMediaEntities - } - } - - /** - * Finds url entities for foreign permalinks, and returns a sequence of tuples containing - * the foreign tweet IDs and the associated UrlEntity containing the permalink. If the same - * permalink appears multiple times, only one of the duplicate entities is returned. - */ - def pastedIdsAndEntities( - tweetId: TweetId, - urlEntities: Seq[UrlEntity] - ): Seq[(TweetId, UrlEntity)] = - urlEntities - .foldLeft(Map.empty[TweetId, UrlEntity]) { - case (z, e) => - MediaUrl.Permalink.getTweetId(e).filter(_ != tweetId) match { - case Some(id) if !z.contains(id) => z + (id -> e) - case _ => z - } - } - .toSeq - - def asRepoCtx(ctx: Ctx) = - PastedMediaRepository.Ctx( - ctx.includeMediaEntities, - ctx.includeAdditionalMetadata, - ctx.includeMediaTags, - ctx.opts.extensionsArgs, - if (ctx.opts.cause == TweetQuery.Cause.Insert(ctx.tweetId) || - ctx.opts.cause == TweetQuery.Cause.Undelete(ctx.tweetId)) { - writeSafetyLevel - } else { - ctx.opts.safetyLevel - } - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PerspectiveHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PerspectiveHydrator.docx new file mode 100644 index 000000000..d7a379b1d Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PerspectiveHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PerspectiveHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PerspectiveHydrator.scala deleted file mode 100644 index 4a055f5ec..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PerspectiveHydrator.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.featureswitches.v2.FeatureSwitchResults -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives.Query -import com.twitter.timelineservice.{thriftscala => tls} -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository.PerspectiveRepository -import com.twitter.tweetypie.thriftscala.FieldByPath -import com.twitter.tweetypie.thriftscala.StatusPerspective - -object PerspectiveHydrator { - type Type = ValueHydrator[Option[StatusPerspective], Ctx] - val hydratedField: FieldByPath = fieldByPath(Tweet.PerspectiveField) - - case class Ctx(featureSwitchResults: Option[FeatureSwitchResults], underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - val Types: Set[tls.PerspectiveType] = - Set( - tls.PerspectiveType.Reported, - tls.PerspectiveType.Favorited, - tls.PerspectiveType.Retweeted, - tls.PerspectiveType.Bookmarked - ) - - val TypesWithoutBookmarked: Set[tls.PerspectiveType] = - Set( - tls.PerspectiveType.Reported, - tls.PerspectiveType.Favorited, - tls.PerspectiveType.Retweeted - ) - - private[this] val partialResult = ValueState.partial(None, hydratedField) - - val bookmarksPerspectiveHydrationEnabledKey = "bookmarks_perspective_hydration_enabled" - - def evaluatePerspectiveTypes( - userId: Long, - bookmarksPerspectiveDecider: Gate[Long], - featureSwitchResults: Option[FeatureSwitchResults] - ): Set[tls.PerspectiveType] = { - if (bookmarksPerspectiveDecider(userId) || - featureSwitchResults - .flatMap(_.getBoolean(bookmarksPerspectiveHydrationEnabledKey, false)) - .getOrElse(false)) - Types - else - TypesWithoutBookmarked - } - - def apply( - repo: PerspectiveRepository.Type, - shouldHydrateBookmarksPerspective: Gate[Long], - stats: StatsReceiver - ): Type = { - val statsByLevel = - SafetyLevel.list.map(level => (level, stats.counter(level.name, "calls"))).toMap - - ValueHydrator[Option[StatusPerspective], Ctx] { (_, ctx) => - val res: Stitch[tls.TimelineEntryPerspective] = if (ctx.isRetweet) { - Stitch.value( - tls.TimelineEntryPerspective( - favorited = false, - retweetId = None, - retweeted = false, - reported = false, - bookmarked = None - ) - ) - } else { - statsByLevel - .getOrElse(ctx.opts.safetyLevel, stats.counter(ctx.opts.safetyLevel.name, "calls")) - .incr() - - repo( - Query( - userId = ctx.opts.forUserId.get, - tweetId = ctx.tweetId, - types = evaluatePerspectiveTypes( - ctx.opts.forUserId.get, - shouldHydrateBookmarksPerspective, - ctx.featureSwitchResults) - )) - } - - res.liftToTry.map { - case Return(perspective) => - ValueState.modified( - Some( - StatusPerspective( - userId = ctx.opts.forUserId.get, - favorited = perspective.favorited, - retweeted = perspective.retweeted, - retweetId = perspective.retweetId, - reported = perspective.reported, - bookmarked = perspective.bookmarked - ) - ) - ) - case _ => partialResult - } - - }.onlyIf { (curr, ctx) => - curr.isEmpty && - ctx.opts.forUserId.nonEmpty && - (ctx.tweetFieldRequested(Tweet.PerspectiveField) || ctx.opts.excludeReported) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PlaceHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PlaceHydrator.docx new file mode 100644 index 000000000..c0f4af74e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PlaceHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PlaceHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PlaceHydrator.scala deleted file mode 100644 index 186619df8..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PlaceHydrator.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object PlaceHydrator { - type Type = ValueHydrator[Option[Place], TweetCtx] - - val HydratedField: FieldByPath = fieldByPath(Tweet.PlaceField) - - def apply(repo: PlaceRepository.Type): Type = - ValueHydrator[Option[Place], TweetCtx] { (_, ctx) => - val key = PlaceKey(ctx.placeId.get, ctx.opts.languageTag) - repo(key).liftToTry.map { - case Return(place) => ValueState.modified(Some(place)) - case Throw(NotFound) => ValueState.UnmodifiedNone - case Throw(_) => ValueState.partial(None, HydratedField) - } - }.onlyIf { (curr, ctx) => - curr.isEmpty && - ctx.tweetFieldRequested(Tweet.PlaceField) && - !ctx.isRetweet && - ctx.placeId.nonEmpty - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PreviousTweetCountsHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PreviousTweetCountsHydrator.docx new file mode 100644 index 000000000..f82619d83 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PreviousTweetCountsHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PreviousTweetCountsHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PreviousTweetCountsHydrator.scala deleted file mode 100644 index 5dff256ac..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/PreviousTweetCountsHydrator.scala +++ /dev/null @@ -1,152 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.featureswitches.v2.FeatureSwitchResults -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.FieldId -import com.twitter.tweetypie.TweetId -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.repository.TweetCountKey -import com.twitter.tweetypie.repository.TweetCountsRepository -import com.twitter.tweetypie.thriftscala.EditControl -import com.twitter.tweetypie.thriftscala.StatusCounts -import com.twitter.tweetypie.thriftscala._ - -/* - * A constructor for a ValueHydrator that hydrates `previous_counts` - * information. Previous counts are applied to edit tweets, they - * are the summation of all the status_counts in an edit chain up to - * but not including the tweet being hydrated. - * - */ -object PreviousTweetCountsHydrator { - - case class Ctx( - editControl: Option[EditControl], - featureSwitchResults: Option[FeatureSwitchResults], - underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - type Type = ValueHydrator[Option[StatusCounts], Ctx] - - val hydratedField: FieldByPath = fieldByPath(Tweet.PreviousCountsField) - - /* - * Params: - * tweetId: The tweet being hydrated. - * editTweetIds: The sorted list of all edits in an edit chain. - * - * Returns: tweetIds in an edit chain from the initial tweet up to but not including - * the tweet being hydrated (`tweetId`) - */ - def previousTweetIds(tweetId: TweetId, editTweetIds: Seq[TweetId]): Seq[TweetId] = { - editTweetIds.takeWhile(_ < tweetId) - } - - /* An addition operation for Option[Long] */ - def sumOptions(A: Option[Long], B: Option[Long]): Option[Long] = - (A, B) match { - case (None, None) => None - case (Some(a), None) => Some(a) - case (None, Some(b)) => Some(b) - case (Some(a), Some(b)) => Some(a + b) - } - - /* An addition operation for StatusCounts */ - def sumStatusCounts(A: StatusCounts, B: StatusCounts): StatusCounts = - StatusCounts( - retweetCount = sumOptions(A.retweetCount, B.retweetCount), - replyCount = sumOptions(A.replyCount, B.replyCount), - favoriteCount = sumOptions(A.favoriteCount, B.favoriteCount), - quoteCount = sumOptions(A.quoteCount, B.quoteCount), - bookmarkCount = sumOptions(A.bookmarkCount, B.bookmarkCount) - ) - - def apply(repo: TweetCountsRepository.Type, shouldHydrateBookmarksCount: Gate[Long]): Type = { - - /* - * Get a StatusCount representing the summed engagements of all previous - * StatusCounts in an edit chain. Only `countsFields` that are specifically requested - * are included in the aggregate StatusCount, otherwise those fields are None. - */ - def getPreviousEngagementCounts( - tweetId: TweetId, - editTweetIds: Seq[TweetId], - countsFields: Set[FieldId] - ): Stitch[ValueState[StatusCounts]] = { - val editTweetIdList = previousTweetIds(tweetId, editTweetIds) - - // StatusCounts for each edit tweet revision - val statusCountsPerEditVersion: Stitch[Seq[ValueState[StatusCounts]]] = - Stitch.collect(editTweetIdList.map { tweetId => - // Which tweet count keys to request, as indicated by the tweet options. - val keys: Seq[TweetCountKey] = - TweetCountsHydrator.toKeys(tweetId, countsFields, None) - - // A separate StatusCounts for each count field, for `tweetId` - // e.g. Seq(StatusCounts(retweetCounts=5L), StatusCounts(favCounts=6L)) - val statusCountsPerCountField: Stitch[Seq[ValueState[StatusCounts]]] = - Stitch.collect(keys.map(key => TweetCountsHydrator.statusCountsRepo(key, repo))) - - // Reduce the per-field counts into a single StatusCounts for `tweetId` - statusCountsPerCountField.map { vs => - // NOTE: This StatusCounts reduction uses different logic than - // `sumStatusCounts`. This reduction takes the latest value for a field. - // instead of summing the fields. - ValueState.sequence(vs).map(TweetCountsHydrator.reduceStatusCounts) - } - }) - - // Sum together the StatusCounts for each edit tweet revision into a single Status Count - statusCountsPerEditVersion.map { vs => - ValueState.sequence(vs).map { statusCounts => - // Reduce a list of StatusCounts into a single StatusCount by summing their fields. - statusCounts.reduce { (a, b) => sumStatusCounts(a, b) } - } - } - } - - ValueHydrator[Option[StatusCounts], Ctx] { (inputStatusCounts, ctx) => - val countsFields: Set[FieldId] = TweetCountsHydrator.filterRequestedCounts( - ctx.opts.forUserId.getOrElse(ctx.userId), - ctx.opts.include.countsFields, - shouldHydrateBookmarksCount, - ctx.featureSwitchResults - ) - - ctx.editControl match { - case Some(EditControl.Edit(edit)) => - edit.editControlInitial match { - case Some(initial) => - val previousStatusCounts: Stitch[ValueState[StatusCounts]] = - getPreviousEngagementCounts(ctx.tweetId, initial.editTweetIds, countsFields) - - // Add the new aggregated StatusCount to the TweetData and return it - previousStatusCounts.map { valueState => - valueState.map { statusCounts => Some(statusCounts) } - } - case None => - // EditControlInitial is not hydrated within EditControlEdit - // This means we cannot provide aggregated previous counts, we will - // fail open and return the input data unchanged. - Stitch.value(ValueState.partial(inputStatusCounts, hydratedField)) - } - - case _ => - // If the tweet has an EditControlInitial - it's the first Tweet in the Edit Chain - // or has no EditControl - it could be an old Tweet from when no Edit Controls existed - // then the previous counts are set to be equal to None. - Stitch.value(ValueState.unit(None)) - } - }.onlyIf { (_, ctx: Ctx) => - // only run if the CountsField was requested; note this is ran both on read and write path - TweetCountsHydrator - .filterRequestedCounts( - ctx.opts.forUserId.getOrElse(ctx.userId), - ctx.opts.include.countsFields, - shouldHydrateBookmarksCount, - ctx.featureSwitchResults - ).nonEmpty - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ProfileGeoHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ProfileGeoHydrator.docx new file mode 100644 index 000000000..193e2f7d4 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ProfileGeoHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ProfileGeoHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ProfileGeoHydrator.scala deleted file mode 100644 index ea461bae8..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ProfileGeoHydrator.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.dataproducts.enrichments.thriftscala.ProfileGeoEnrichment -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository.ProfileGeoKey -import com.twitter.tweetypie.repository.ProfileGeoRepository -import com.twitter.tweetypie.thriftscala.FieldByPath - -object ProfileGeoHydrator { - type Type = ValueHydrator[Option[ProfileGeoEnrichment], TweetCtx] - - val hydratedField: FieldByPath = fieldByPath(Tweet.ProfileGeoEnrichmentField) - - private[this] val partialResult = ValueState.partial(None, hydratedField) - - def apply(repo: ProfileGeoRepository.Type): Type = - ValueHydrator[Option[ProfileGeoEnrichment], TweetCtx] { (curr, ctx) => - val key = - ProfileGeoKey( - tweetId = ctx.tweetId, - userId = Some(ctx.userId), - coords = ctx.geoCoordinates - ) - repo(key).liftToTry.map { - case Return(enrichment) => ValueState.modified(Some(enrichment)) - case Throw(_) => partialResult - } - }.onlyIf((curr, ctx) => - curr.isEmpty && ctx.tweetFieldRequested(Tweet.ProfileGeoEnrichmentField)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuoteTweetVisibilityHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuoteTweetVisibilityHydrator.docx new file mode 100644 index 000000000..a939c0f1d Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuoteTweetVisibilityHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuoteTweetVisibilityHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuoteTweetVisibilityHydrator.scala deleted file mode 100644 index f82e9fa0b..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuoteTweetVisibilityHydrator.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala.QuotedTweet - -/** - * Enforce that users are not shown quoted tweets where the author of the - * inner quoted tweet blocks the author of the outer quote tweet or the author - * of the inner quoted tweet is otherwise not visible to the outer author. - * - * In the example below, QuoteTweetVisibilityHydrator checks if @jack - * blocks @trollmaster. - * - * {{{ - * @viewer - * +------------------------------+ - * | @trollmaster | <-- OUTER QUOTE TWEET - * | lol u can't spell twitter | - * | +--------------------------+ | - * | | @jack | <---- INNER QUOTED TWEET - * | | just setting up my twttr | | - * | +--------------------------+ | - * +------------------------------+ - * }}} - * - * In the example below, QuoteTweetVisibilityHydrator checks if @h4x0r can view - * user @protectedUser. - * - * {{{ - * @viewer - * +------------------------------+ - * | @h4x0r | <-- OUTER QUOTE TWEET - * | lol nice password | - * | +--------------------------+ | - * | | @protectedUser | <---- INNER QUOTED TWEET - * | | my password is 1234 | | - * | +--------------------------+ | - * +------------------------------+ - * }}} - * - * - * In the example below, QuoteTweetVisibilityHydrator checks if @viewer blocks @jack: - * - * {{{ - * @viewer - * +------------------------------+ - * | @sometweeter | <-- OUTER QUOTE TWEET - * | This is a historic tweet | - * | +--------------------------+ | - * | | @jack | <---- INNER QUOTED TWEET - * | | just setting up my twttr | | - * | +--------------------------+ | - * +------------------------------+ - * }}} - * - */ -object QuoteTweetVisibilityHydrator { - type Type = ValueHydrator[Option[FilteredState.Unavailable], TweetCtx] - - def apply(repo: QuotedTweetVisibilityRepository.Type): QuoteTweetVisibilityHydrator.Type = - ValueHydrator[Option[FilteredState.Unavailable], TweetCtx] { (_, ctx) => - val innerTweet: QuotedTweet = ctx.quotedTweet.get - val request = QuotedTweetVisibilityRepository.Request( - outerTweetId = ctx.tweetId, - outerAuthorId = ctx.userId, - innerTweetId = innerTweet.tweetId, - innerAuthorId = innerTweet.userId, - viewerId = ctx.opts.forUserId, - safetyLevel = ctx.opts.safetyLevel - ) - - repo(request).liftToTry.map { - case Return(Some(f: FilteredState.Unavailable)) => - ValueState.modified(Some(f)) - - // For tweet::quotedTweet relationships, all other FilteredStates - // allow the quotedTweet to be hydrated and filtered independently - case Return(_) => - ValueState.UnmodifiedNone - - // On VF failure, gracefully degrade to no filtering - case Throw(_) => - ValueState.UnmodifiedNone - } - }.onlyIf { (_, ctx) => - !ctx.isRetweet && - ctx.tweetFieldRequested(Tweet.QuotedTweetField) && - ctx.opts.enforceVisibilityFiltering && - ctx.quotedTweet.isDefined - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetHydrator.docx new file mode 100644 index 000000000..4054e867f Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetHydrator.scala deleted file mode 100644 index e112ef395..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetHydrator.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ - -/** - * Loads the tweet referenced by `Tweet.quotedTweet`. - */ -object QuotedTweetHydrator { - type Type = ValueHydrator[Option[QuotedTweetResult], Ctx] - - case class Ctx( - quotedTweetFilteredState: Option[FilteredState.Unavailable], - underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - def apply(repo: TweetResultRepository.Type): Type = { - ValueHydrator[Option[QuotedTweetResult], Ctx] { (_, ctx) => - (ctx.quotedTweetFilteredState, ctx.quotedTweet) match { - - case (_, None) => - // If there is no quoted tweet ref, leave the value as None, - // indicating undefined - ValueState.StitchUnmodifiedNone - - case (Some(fs), _) => - Stitch.value(ValueState.modified(Some(QuotedTweetResult.Filtered(fs)))) - - case (None, Some(qtRef)) => - val qtQueryOptions = - ctx.opts.copy( - // we don't want to recursively load quoted tweets - include = ctx.opts.include.copy(quotedTweet = false), - // be sure to get a clean version of the tweet - scrubUnrequestedFields = true, - // TweetVisibilityLibrary filters quoted tweets slightly differently from other tweets. - // Specifically, most Interstitial verdicts are converted to Drops. - isInnerQuotedTweet = true - ) - - repo(qtRef.tweetId, qtQueryOptions).transform { t => - Stitch.const { - QuotedTweetResult.fromTry(t).map(r => ValueState.modified(Some(r))) - } - } - } - }.onlyIf((curr, ctx) => curr.isEmpty && ctx.opts.include.quotedTweet) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefHydrator.docx new file mode 100644 index 000000000..bea750279 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefHydrator.scala deleted file mode 100644 index e2556f986..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefHydrator.scala +++ /dev/null @@ -1,129 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetutil.TweetPermalink -import com.twitter.tweetypie.core.FilteredState -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -/** - * Adds QuotedTweet structs to tweets that contain a tweet permalink url at the end of the - * tweet text. After introduction of QT + Media, we stopped storing inner tweet permalinks - * in the outer tweet text. So this hydrator would run only for below cases: - * - * - historical quote tweets which have inner tweet url in the tweet text and url entities. - * - new quote tweets created with pasted tweet permalinks, going forward we want to persist - * quoted_tweet struct in MH for these tweets - */ -object QuotedTweetRefHydrator { - type Type = ValueHydrator[Option[QuotedTweet], Ctx] - - case class Ctx(urlEntities: Seq[UrlEntity], underlyingTweetCtx: TweetCtx) extends TweetCtx.Proxy - - val hydratedField: FieldByPath = fieldByPath(Tweet.QuotedTweetField) - - private val partial = ValueState.partial(None, hydratedField) - - val queryOptions: TweetQuery.Options = - TweetQuery.Options( - include = TweetQuery.Include(Set(Tweet.CoreDataField.id)), - // Don't enforce visibility filtering when loading the QuotedTweet struct because it is - // cacheable. The filtering happens in QuoteTweetVisibilityHydrator. - enforceVisibilityFiltering = false, - forUserId = None - ) - - def once(h: Type): Type = - TweetHydration.completeOnlyOnce( - queryFilter = queryFilter, - hydrationType = HydrationType.QuotedTweetRef, - dependsOn = Set(HydrationType.Urls), - hydrator = h - ) - - case class UrlHydrationFailed(url: String) extends Exception - - /** - * Iterate through UrlEntity objects in reverse to identify a quoted-tweet ID - * to hydrate. Quoted tweets are indicated by a TweetPermalink in the tweet text - * that references an older tweet ID. If a quoted tweet permalink is found, also - * return the corresponding UrlEntity. - * - * @throws UrlHydrationFailed if we encounter a partial URL entity before - * finding a tweet permalink URL. - */ - def quotedTweetId(ctx: Ctx): Option[(UrlEntity, TweetId)] = - ctx.urlEntities.reverseIterator // we want the rightmost tweet permalink - .map { e: UrlEntity => - if (UrlEntityHydrator.hydrationFailed(e)) throw UrlHydrationFailed(e.url) - else (e, e.expanded) - } - .collectFirst { - case (e, Some(TweetPermalink(_, quotedTweetId))) => (e, quotedTweetId) - } - // Prevent tweet-quoting cycles - .filter { case (_, quotedTweetId) => ctx.tweetId > quotedTweetId } - - def buildShortenedUrl(e: UrlEntity): ShortenedUrl = - ShortenedUrl( - shortUrl = e.url, - // Reading from MH will also default the following to "". - // QuotedTweetRefUrlsHydrator will hydrate these cases - longUrl = e.expanded.getOrElse(""), - displayText = e.display.getOrElse("") - ) - - /** - * We run this hydrator only if: - * - * - quoted_tweet struct is empty - * - quoted_tweet is present but permalink is not - * - url entities is present. QT hydration depends on urls - long term goal - * is to entirely rely on persisted quoted_tweet struct in MH - * - requested tweet is not a retweet - * - * Hydration steps: - * - We determine the last tweet permalink from url entities - * - Extract the inner tweet Id from the permalink - * - Query tweet repo with inner tweet Id - * - Construct quoted_tweet struct from hydrated tweet object and last permalink - */ - def apply(repo: TweetRepository.Type): Type = - ValueHydrator[Option[QuotedTweet], Ctx] { (_, ctx) => - // propagate errors from quotedTweetId in Stitch - Stitch(quotedTweetId(ctx)).liftToTry.flatMap { - case Return(Some((lastPermalinkEntity, quotedTweetId))) => - repo(quotedTweetId, queryOptions).liftToTry.map { - case Return(tweet) => - ValueState.modified( - Some(asQuotedTweet(tweet, lastPermalinkEntity)) - ) - case Throw(NotFound | _: FilteredState) => ValueState.UnmodifiedNone - case Throw(_) => partial - } - case Return(None) => Stitch(ValueState.UnmodifiedNone) - case Throw(_) => Stitch(partial) - } - }.onlyIf { (curr, ctx) => - (curr.isEmpty || curr.exists(_.permalink.isEmpty)) && - !ctx.isRetweet && ctx.urlEntities.nonEmpty - } - - def queryFilter(opts: TweetQuery.Options): Boolean = - opts.include.tweetFields(Tweet.QuotedTweetField.id) - - /** - * We construct Tweet.quoted_tweet from hydrated inner tweet. - * Note: if the inner tweet is a Retweet, we populate the quoted_tweet struct from source tweet. - */ - def asQuotedTweet(tweet: Tweet, entity: UrlEntity): QuotedTweet = { - val shortenedUrl = Some(buildShortenedUrl(entity)) - getShare(tweet) match { - case None => QuotedTweet(tweet.id, getUserId(tweet), shortenedUrl) - case Some(share) => QuotedTweet(share.sourceStatusId, share.sourceUserId, shortenedUrl) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefUrlsHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefUrlsHydrator.docx new file mode 100644 index 000000000..75949341d Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefUrlsHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefUrlsHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefUrlsHydrator.scala deleted file mode 100644 index b25acfc2e..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/QuotedTweetRefUrlsHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tco_util.DisplayUrl -import com.twitter.tweetutil.TweetPermalink -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -/** - * This populates expanded URL and display text in ShortenedUrl struct, - * which is part of QuotedTweet metadata. We are using User Identity repo - * to retrieve user's current screen-name to construct expanded url, instead - * of relying on URL hydration. - * - * Expanded urls contain a mutable screen name and an immutable tweetId. - * when visiting the link, you're always redirected to the link with - * correct screen name - therefore, it's okay to have permalinks containing - * old screen names that have since been changed by their user in the cache. - * Keys will be auto-refreshed based on the 14 days TTL, we can also have - * a daemon flush the keys with screen-name change. - * - */ -object QuotedTweetRefUrlsHydrator { - type Type = ValueHydrator[Option[QuotedTweet], TweetCtx] - - /** - * Return true if longUrl is not set or if a prior hydration set it to shortUrl due to - * a partial (to re-attempt hydration). - */ - def needsHydration(s: ShortenedUrl): Boolean = - s.longUrl.isEmpty || s.displayText.isEmpty || s.longUrl == s.shortUrl - - def apply(repo: UserIdentityRepository.Type): Type = { - ValueHydrator[QuotedTweet, TweetCtx] { (curr, _) => - repo(UserKey(curr.userId)).liftToTry.map { r => - // we verify curr.permalink.exists pre-hydration - val shortUrl = curr.permalink.get.shortUrl - val expandedUrl = r match { - case Return(user) => TweetPermalink(user.screenName, curr.tweetId).httpsUrl - case Throw(_) => shortUrl // fall-back to shortUrl as expandedUrl - } - ValueState.delta( - curr, - curr.copy( - permalink = Some( - ShortenedUrl( - shortUrl, - expandedUrl, - DisplayUrl.truncateUrl(expandedUrl, true) - ) - ) - ) - ) - } - } - }.onlyIf { (curr, ctx) => - curr.permalink.exists(needsHydration) && - ctx.tweetFieldRequested(Tweet.QuotedTweetField) && !ctx.isRetweet - }.liftOption -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RepairMutation.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RepairMutation.docx new file mode 100644 index 000000000..50039a178 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RepairMutation.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RepairMutation.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RepairMutation.scala deleted file mode 100644 index f960740b2..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RepairMutation.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -/** - * A Mutation that will note all repairs that took place in the - * supplied StatsReceiver, under the names in repairers. - */ -object RepairMutation { - def apply[T](stats: StatsReceiver, repairers: (String, Mutation[T])*): Mutation[T] = - Mutation.all( - repairers.map { - case (name, mutation) => mutation.countMutations(stats.counter(name)) - } - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReplyScreenNameHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReplyScreenNameHydrator.docx new file mode 100644 index 000000000..6fcd3bb0d Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReplyScreenNameHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReplyScreenNameHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReplyScreenNameHydrator.scala deleted file mode 100644 index 6fa50d572..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReplyScreenNameHydrator.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ - -object ReplyScreenNameHydrator { - import TweetLenses.Reply.{inReplyToScreenName => screenNameLens} - - type Type = ValueHydrator[Option[Reply], TweetCtx] - - val hydratedField: FieldByPath = - fieldByPath(Tweet.CoreDataField, TweetCoreData.ReplyField, Reply.InReplyToScreenNameField) - - def once(h: ValueHydrator[Option[Reply], TweetCtx]): Type = - TweetHydration.completeOnlyOnce( - hydrationType = HydrationType.ReplyScreenName, - hydrator = h - ) - - def apply[C](repo: UserIdentityRepository.Type): ValueHydrator[Option[Reply], C] = - ValueHydrator[Reply, C] { (reply, ctx) => - val key = UserKey(reply.inReplyToUserId) - - repo(key).liftToTry.map { - case Return(user) => ValueState.modified(screenNameLens.set(reply, Some(user.screenName))) - case Throw(NotFound) => ValueState.unmodified(reply) - case Throw(_) => ValueState.partial(reply, hydratedField) - } - }.onlyIf((reply, _) => screenNameLens.get(reply).isEmpty).liftOption -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReportedTweetFilter.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReportedTweetFilter.docx new file mode 100644 index 000000000..8988c8c36 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReportedTweetFilter.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReportedTweetFilter.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReportedTweetFilter.scala deleted file mode 100644 index 6f22c0634..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ReportedTweetFilter.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.thriftscala._ - -object ReportedTweetFilter { - type Type = ValueHydrator[Unit, Ctx] - - object MissingPerspectiveError - extends TweetHydrationError("Cannot determine reported state because perspective is missing") - - case class Ctx(perspective: Option[StatusPerspective], underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - def apply(): Type = - ValueHydrator[Unit, Ctx] { (_, ctx) => - ctx.perspective match { - case Some(p) if !p.reported => ValueState.StitchUnmodifiedUnit - case Some(_) => Stitch.exception(FilteredState.Unavailable.Reported) - case None => Stitch.exception(MissingPerspectiveError) - } - }.onlyIf { (_, ctx) => ctx.opts.excludeReported && ctx.opts.forUserId.isDefined } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetMediaRepairer.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetMediaRepairer.docx new file mode 100644 index 000000000..a88b9f365 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetMediaRepairer.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetMediaRepairer.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetMediaRepairer.scala deleted file mode 100644 index c200c0d75..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetMediaRepairer.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -/** - * Retweets should never have their own media, and should never be cached with a media - * entity. - */ -object RetweetMediaRepairer extends Mutation[Tweet] { - def apply(tweet: Tweet): Option[Tweet] = { - if (isRetweet(tweet) && getMedia(tweet).nonEmpty) - Some(TweetLenses.media.set(tweet, Nil)) - else - None - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetParentStatusIdRepairer.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetParentStatusIdRepairer.docx new file mode 100644 index 000000000..fab48d4c4 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetParentStatusIdRepairer.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetParentStatusIdRepairer.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetParentStatusIdRepairer.scala deleted file mode 100644 index 5206d39f1..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/RetweetParentStatusIdRepairer.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.thriftscala.Share - -/** - * When creating a retweet, we set parent_status_id to the tweet id that the user sent (the tweet they're retweeting). - * Old tweets have parent_status_id set to zero. - * When loading the old tweets, we should set parent_status_id to source_status_id if it's zero. - */ -object RetweetParentStatusIdRepairer { - private val shareMutation = - Mutation.fromPartial[Option[Share]] { - case Some(share) if share.parentStatusId == 0L => - Some(share.copy(parentStatusId = share.sourceStatusId)) - } - - private[tweetypie] val tweetMutation = TweetLenses.share.mutation(shareMutation) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubEngagementHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubEngagementHydrator.docx new file mode 100644 index 000000000..5122c9d67 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubEngagementHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubEngagementHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubEngagementHydrator.scala deleted file mode 100644 index 068a283ca..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubEngagementHydrator.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.spam.rtf.thriftscala.FilteredReason -import com.twitter.tweetypie.core.FilteredState -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.thriftscala._ -import com.twitter.visibility.results.counts.EngagementCounts - -/** - * Redact Tweet.counts (StatusCounts) for some visibility results - */ -object ScrubEngagementHydrator { - type Type = ValueHydrator[Option[StatusCounts], Ctx] - - case class Ctx(filteredState: Option[FilteredState.Suppress]) - - def apply(): Type = - ValueHydrator.map[Option[StatusCounts], Ctx] { (curr: Option[StatusCounts], ctx: Ctx) => - ctx.filteredState match { - case Some(FilteredState.Suppress(FilteredReason.SafetyResult(result))) if curr.nonEmpty => - ValueState.delta(curr, EngagementCounts.scrubEngagementCounts(result.action, curr)) - case _ => - ValueState.unmodified(curr) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubUncacheableTweetRepairer.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubUncacheableTweetRepairer.docx new file mode 100644 index 000000000..6746ee8c8 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubUncacheableTweetRepairer.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubUncacheableTweetRepairer.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubUncacheableTweetRepairer.scala deleted file mode 100644 index ef76e9e76..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ScrubUncacheableTweetRepairer.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.thriftscala._ - -object ScrubUncacheable { - - // A mutation to use for scrubbing tweets for cache - val tweetMutation: Mutation[Tweet] = - Mutation { tweet => - if (tweet.place != None || - tweet.counts != None || - tweet.deviceSource != None || - tweet.perspective != None || - tweet.cards != None || - tweet.card2 != None || - tweet.spamLabels != None || - tweet.conversationMuted != None) - Some( - tweet.copy( - place = None, - counts = None, - deviceSource = None, - perspective = None, - cards = None, - card2 = None, - spamLabels = None, - conversationMuted = None - ) - ) - else - None - } - - // throws an AssertionError if a tweet when a tweet is scrubbed - def assertNotScrubbed(message: String): Mutation[Tweet] = - tweetMutation.withEffect(Effect(update => assert(update.isEmpty, message))) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SourceTweetHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SourceTweetHydrator.docx new file mode 100644 index 000000000..d4791e6b1 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SourceTweetHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SourceTweetHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SourceTweetHydrator.scala deleted file mode 100644 index 7309b016c..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SourceTweetHydrator.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.FilteredState.Unavailable._ -import com.twitter.tweetypie.core.TweetResult -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.repository.TweetResultRepository -import com.twitter.tweetypie.thriftscala.DetachedRetweet - -/** - * Loads the source tweet for a retweet - */ -object SourceTweetHydrator { - type Type = ValueHydrator[Option[TweetResult], TweetCtx] - - def configureOptions(opts: TweetQuery.Options): TweetQuery.Options = { - // set scrubUnrequestedFields to false so that we will have access to - // additional fields, which will be copied into the retweet. - // set fetchStoredTweets to false because we don't want to fetch and hydrate - // the source tweet if it is deleted. - opts.copy(scrubUnrequestedFields = false, fetchStoredTweets = false, isSourceTweet = true) - } - - private object NotFoundException { - def unapply(t: Throwable): Option[Boolean] = - t match { - case NotFound => Some(false) - case TweetDeleted | BounceDeleted => Some(true) - case _ => None - } - } - - def apply( - repo: TweetResultRepository.Type, - stats: StatsReceiver, - scribeDetachedRetweets: FutureEffect[DetachedRetweet] = FutureEffect.unit - ): Type = { - val notFoundCounter = stats.counter("not_found") - - ValueHydrator[Option[TweetResult], TweetCtx] { (_, ctx) => - ctx.sourceTweetId match { - case None => - ValueState.StitchUnmodifiedNone - case Some(srcTweetId) => - repo(srcTweetId, configureOptions(ctx.opts)).liftToTry.flatMap { - case Throw(NotFoundException(isDeleted)) => - notFoundCounter.incr() - scribeDetachedRetweets(detachedRetweet(srcTweetId, ctx)) - if (ctx.opts.requireSourceTweet) { - Stitch.exception(SourceTweetNotFound(isDeleted)) - } else { - ValueState.StitchUnmodifiedNone - } - - case Return(r) => Stitch.value(ValueState.modified(Some(r))) - case Throw(t) => Stitch.exception(t) - } - } - }.onlyIf((curr, _) => curr.isEmpty) - } - - def detachedRetweet(srcTweetId: TweetId, ctx: TweetCtx): DetachedRetweet = - DetachedRetweet(ctx.tweetId, ctx.userId, srcTweetId) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/StripHiddenGeoCoordinates.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/StripHiddenGeoCoordinates.docx new file mode 100644 index 000000000..006142fc3 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/StripHiddenGeoCoordinates.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/StripHiddenGeoCoordinates.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/StripHiddenGeoCoordinates.scala deleted file mode 100644 index 3727c8779..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/StripHiddenGeoCoordinates.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -object StripHiddenGeoCoordinates extends Mutation[Tweet] { - def apply(tweet: Tweet): Option[Tweet] = - for { - coreData <- tweet.coreData - coords <- coreData.coordinates - if !coords.display - coreData2 = coreData.copy(coordinates = None) - } yield tweet.copy(coreData = Some(coreData2)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SuperfluousUrlEntityScrubber.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SuperfluousUrlEntityScrubber.docx new file mode 100644 index 000000000..2fda6f700 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SuperfluousUrlEntityScrubber.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SuperfluousUrlEntityScrubber.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SuperfluousUrlEntityScrubber.scala deleted file mode 100644 index d49b2c17a..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/SuperfluousUrlEntityScrubber.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.thriftscala._ - -/** - * Removes superfluous urls entities when there is a corresponding MediaEntity for the same - * url. - */ -object SuperfluousUrlEntityScrubber { - case class RawEntity(fromIndex: Short, toIndex: Short, url: String) - - object RawEntity { - def from(e: UrlEntity): RawEntity = RawEntity(e.fromIndex, e.toIndex, e.url) - def fromUrls(es: Seq[UrlEntity]): Set[RawEntity] = es.map(from(_)).toSet - def from(e: MediaEntity): RawEntity = RawEntity(e.fromIndex, e.toIndex, e.url) - def fromMedia(es: Seq[MediaEntity]): Set[RawEntity] = es.map(from(_)).toSet - } - - val mutation: Mutation[Tweet] = - Mutation[Tweet] { tweet => - val mediaEntities = getMedia(tweet) - val urlEntities = getUrls(tweet) - - if (mediaEntities.isEmpty || urlEntities.isEmpty) { - None - } else { - val mediaUrls = mediaEntities.map(RawEntity.from(_)).toSet - val scrubbedUrls = urlEntities.filterNot(e => mediaUrls.contains(RawEntity.from(e))) - - if (scrubbedUrls.size == urlEntities.size) - None - else - Some(TweetLenses.urls.set(tweet, scrubbedUrls)) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TakedownHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TakedownHydrator.docx new file mode 100644 index 000000000..7d88fe746 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TakedownHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TakedownHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TakedownHydrator.scala deleted file mode 100644 index f5a510047..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TakedownHydrator.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala.FieldByPath -import com.twitter.tweetypie.util.Takedowns - -/** - * Hydrates per-country takedowns which is a union of: - * 1. per-tweet takedowns, from tweetypieOnlyTakedown{CountryCode|Reasons} fields - * 2. user takedowns, read from gizmoduck. - * - * Note that this hydrator performs backwards compatibility by converting to and from - * [[com.twitter.tseng.withholding.thriftscala.TakedownReason]]. This is possible because a taken - * down country code can always be represented as a - * [[com.twitter.tseng.withholding.thriftscala.UnspecifiedReason]]. - */ -object TakedownHydrator { - type Type = ValueHydrator[Option[Takedowns], Ctx] - - case class Ctx(tweetTakedowns: Takedowns, underlyingTweetCtx: TweetCtx) extends TweetCtx.Proxy - - val hydratedFields: Set[FieldByPath] = - Set( - fieldByPath(Tweet.TakedownCountryCodesField), - fieldByPath(Tweet.TakedownReasonsField) - ) - - def apply(repo: UserTakedownRepository.Type): Type = - ValueHydrator[Option[Takedowns], Ctx] { (curr, ctx) => - repo(ctx.userId).liftToTry.map { - case Return(userReasons) => - val reasons = Seq.concat(ctx.tweetTakedowns.reasons, userReasons).toSet - ValueState.delta(curr, Some(Takedowns(reasons))) - case Throw(_) => - ValueState.partial(curr, hydratedFields) - } - }.onlyIf { (_, ctx) => - ( - ctx.tweetFieldRequested(Tweet.TakedownCountryCodesField) || - ctx.tweetFieldRequested(Tweet.TakedownReasonsField) - ) && ctx.hasTakedown - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TextRepairer.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TextRepairer.docx new file mode 100644 index 000000000..cedaaaef8 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TextRepairer.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TextRepairer.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TextRepairer.scala deleted file mode 100644 index 5a5e62c3d..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TextRepairer.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.serverutil.ExtendedTweetMetadataBuilder -import com.twitter.tweetypie.tweettext.Preprocessor._ -import com.twitter.tweetypie.tweettext.TextModification -import com.twitter.tweetypie.thriftscala.entities.Implicits._ - -object TextRepairer { - def apply(replace: String => Option[TextModification]): Mutation[Tweet] = - Mutation { tweet => - replace(getText(tweet)).map { mod => - val repairedTweet = tweet.copy( - coreData = tweet.coreData.map(c => c.copy(text = mod.updated)), - urls = Some(getUrls(tweet).flatMap(mod.reindexEntity(_))), - mentions = Some(getMentions(tweet).flatMap(mod.reindexEntity(_))), - hashtags = Some(getHashtags(tweet).flatMap(mod.reindexEntity(_))), - cashtags = Some(getCashtags(tweet).flatMap(mod.reindexEntity(_))), - media = Some(getMedia(tweet).flatMap(mod.reindexEntity(_))), - visibleTextRange = tweet.visibleTextRange.flatMap(mod.reindexEntity(_)) - ) - - val repairedExtendedTweetMetadata = repairedTweet.selfPermalink.flatMap { permalink => - val extendedTweetMetadata = ExtendedTweetMetadataBuilder(repairedTweet, permalink) - val repairedTextLength = getText(repairedTweet).length - if (extendedTweetMetadata.apiCompatibleTruncationIndex == repairedTextLength) { - None - } else { - Some(extendedTweetMetadata) - } - } - - repairedTweet.copy(extendedTweetMetadata = repairedExtendedTweetMetadata) - } - } - - /** - * Removes whitespace from the tweet, and updates all entity indices. - */ - val BlankLineCollapser: Mutation[Tweet] = TextRepairer(collapseBlankLinesModification _) - - /** - * Replace a special unicode string that crashes ios app with '\ufffd' - */ - val CoreTextBugPatcher: Mutation[Tweet] = TextRepairer(replaceCoreTextBugModification _) - -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetAuthorVisibilityHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetAuthorVisibilityHydrator.docx new file mode 100644 index 000000000..6e4873697 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetAuthorVisibilityHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetAuthorVisibilityHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetAuthorVisibilityHydrator.scala deleted file mode 100644 index c9c5c71f9..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetAuthorVisibilityHydrator.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ - -/** - * Ensures that the tweet's author and source tweet's author (if retweet) are visible to the - * viewing user - ctx.opts.forUserId - when enforceVisibilityFiltering is true. - * If either of these users is not visible then a FilteredState.Suppress will be returned. - * - * Note: blocking relationship is NOT checked here, this means if viewing user `forUserId` is blocked - * by either the tweet's author or source tweet's author, this will not filter out the tweet. - */ -object TweetAuthorVisibilityHydrator { - type Type = ValueHydrator[Unit, TweetCtx] - - def apply(repo: UserVisibilityRepository.Type): Type = - ValueHydrator[Unit, TweetCtx] { (_, ctx) => - val ids = Seq(ctx.userId) ++ ctx.sourceUserId - val keys = ids.map(id => toRepoQuery(id, ctx)) - - Stitch - .traverse(keys)(repo.apply).flatMap { responses => - val fs: Option[FilteredState.Unavailable] = responses.flatten.headOption - - fs match { - case Some(fs: FilteredState.Unavailable) => Stitch.exception(fs) - case None => ValueState.StitchUnmodifiedUnit - } - } - }.onlyIf((_, ctx) => ctx.opts.enforceVisibilityFiltering) - - private def toRepoQuery(userId: UserId, ctx: TweetCtx) = - UserVisibilityRepository.Query( - UserKey(userId), - ctx.opts.forUserId, - ctx.tweetId, - ctx.isRetweet, - ctx.opts.isInnerQuotedTweet, - Some(ctx.opts.safetyLevel)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCountsHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCountsHydrator.docx new file mode 100644 index 000000000..3d94b9034 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCountsHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCountsHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCountsHydrator.scala deleted file mode 100644 index 17462081a..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCountsHydrator.scala +++ /dev/null @@ -1,189 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.featureswitches.v2.FeatureSwitchResults -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ -import scala.collection.mutable - -object TweetCountsHydrator { - type Type = ValueHydrator[Option[StatusCounts], Ctx] - - case class Ctx(featureSwitchResults: Option[FeatureSwitchResults], underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - val retweetCountField: FieldByPath = - fieldByPath(Tweet.CountsField, StatusCounts.RetweetCountField) - val replyCountField: FieldByPath = fieldByPath(Tweet.CountsField, StatusCounts.ReplyCountField) - val favoriteCountField: FieldByPath = - fieldByPath(Tweet.CountsField, StatusCounts.FavoriteCountField) - val quoteCountField: FieldByPath = fieldByPath(Tweet.CountsField, StatusCounts.QuoteCountField) - val bookmarkCountField: FieldByPath = - fieldByPath(Tweet.CountsField, StatusCounts.BookmarkCountField) - - val emptyCounts = StatusCounts() - - val retweetCountPartial = ValueState.partial(emptyCounts, retweetCountField) - val replyCountPartial = ValueState.partial(emptyCounts, replyCountField) - val favoriteCountPartial = ValueState.partial(emptyCounts, favoriteCountField) - val quoteCountPartial = ValueState.partial(emptyCounts, quoteCountField) - val bookmarkCountPartial = ValueState.partial(emptyCounts, bookmarkCountField) - - val bookmarksCountHydrationEnabledKey = "bookmarks_count_hydration_enabled" - - /** - * Take a Seq of StatusCounts and reduce down to a single StatusCounts. - * Note: `reduce` here is safe because we are guaranteed to always have at least - * one value. - */ - def reduceStatusCounts(counts: Seq[StatusCounts]): StatusCounts = - counts.reduce { (a, b) => - StatusCounts( - retweetCount = b.retweetCount.orElse(a.retweetCount), - replyCount = b.replyCount.orElse(a.replyCount), - favoriteCount = b.favoriteCount.orElse(a.favoriteCount), - quoteCount = b.quoteCount.orElse(a.quoteCount), - bookmarkCount = b.bookmarkCount.orElse(a.bookmarkCount) - ) - } - - def toKeys( - tweetId: TweetId, - countsFields: Set[FieldId], - curr: Option[StatusCounts] - ): Seq[TweetCountKey] = { - val keys = new mutable.ArrayBuffer[TweetCountKey](4) - - countsFields.foreach { - case StatusCounts.RetweetCountField.id => - if (curr.flatMap(_.retweetCount).isEmpty) - keys += RetweetsKey(tweetId) - - case StatusCounts.ReplyCountField.id => - if (curr.flatMap(_.replyCount).isEmpty) - keys += RepliesKey(tweetId) - - case StatusCounts.FavoriteCountField.id => - if (curr.flatMap(_.favoriteCount).isEmpty) - keys += FavsKey(tweetId) - - case StatusCounts.QuoteCountField.id => - if (curr.flatMap(_.quoteCount).isEmpty) - keys += QuotesKey(tweetId) - - case StatusCounts.BookmarkCountField.id => - if (curr.flatMap(_.bookmarkCount).isEmpty) - keys += BookmarksKey(tweetId) - - case _ => - } - - keys - } - - /* - * Get a StatusCounts object for a specific tweet and specific field (e.g. only fav, or reply etc). - * StatusCounts returned from here can be combined with other StatusCounts using `sumStatusCount` - */ - def statusCountsRepo( - key: TweetCountKey, - repo: TweetCountsRepository.Type - ): Stitch[ValueState[StatusCounts]] = - repo(key).liftToTry.map { - case Return(count) => - ValueState.modified( - key match { - case _: RetweetsKey => StatusCounts(retweetCount = Some(count)) - case _: RepliesKey => StatusCounts(replyCount = Some(count)) - case _: FavsKey => StatusCounts(favoriteCount = Some(count)) - case _: QuotesKey => StatusCounts(quoteCount = Some(count)) - case _: BookmarksKey => StatusCounts(bookmarkCount = Some(count)) - } - ) - - case Throw(_) => - key match { - case _: RetweetsKey => retweetCountPartial - case _: RepliesKey => replyCountPartial - case _: FavsKey => favoriteCountPartial - case _: QuotesKey => quoteCountPartial - case _: BookmarksKey => bookmarkCountPartial - } - } - - def filterRequestedCounts( - userId: UserId, - requestedCounts: Set[FieldId], - bookmarkCountsDecider: Gate[Long], - featureSwitchResults: Option[FeatureSwitchResults] - ): Set[FieldId] = { - if (requestedCounts.contains(StatusCounts.BookmarkCountField.id)) - if (bookmarkCountsDecider(userId) || - featureSwitchResults - .flatMap(_.getBoolean(bookmarksCountHydrationEnabledKey, false)) - .getOrElse(false)) - requestedCounts - else - requestedCounts.filter(_ != StatusCounts.BookmarkCountField.id) - else - requestedCounts - } - - def apply(repo: TweetCountsRepository.Type, shouldHydrateBookmarksCount: Gate[Long]): Type = { - - val all: Set[FieldId] = StatusCounts.fieldInfos.map(_.tfield.id).toSet - - val modifiedZero: Map[Set[FieldId], ValueState[Some[StatusCounts]]] = { - for (set <- all.subsets) yield { - @inline - def zeroOrNone(fieldId: FieldId) = - if (set.contains(fieldId)) Some(0L) else None - - val statusCounts = - StatusCounts( - retweetCount = zeroOrNone(StatusCounts.RetweetCountField.id), - replyCount = zeroOrNone(StatusCounts.ReplyCountField.id), - favoriteCount = zeroOrNone(StatusCounts.FavoriteCountField.id), - quoteCount = zeroOrNone(StatusCounts.QuoteCountField.id), - bookmarkCount = zeroOrNone(StatusCounts.BookmarkCountField.id) - ) - - set -> ValueState.modified(Some(statusCounts)) - } - }.toMap - - ValueHydrator[Option[StatusCounts], Ctx] { (curr, ctx) => - val countsFields: Set[FieldId] = filterRequestedCounts( - ctx.opts.forUserId.getOrElse(ctx.userId), - ctx.opts.include.countsFields, - shouldHydrateBookmarksCount, - ctx.featureSwitchResults - ) - if (ctx.isRetweet) { - // To avoid a reflection-induced key error where the countsFields can contain a fieldId - // that is not in the thrift schema loaded at start, we strip unknown field_ids using - // `intersect` - Stitch.value(modifiedZero(countsFields.intersect(all))) - } else { - val keys = toKeys(ctx.tweetId, countsFields, curr) - - Stitch.traverse(keys)(key => statusCountsRepo(key, repo)).map { results => - // always flag modified if starting from None - val vs0 = ValueState.success(curr.getOrElse(emptyCounts), curr.isEmpty) - val vs = vs0 +: results - - ValueState.sequence(vs).map(reduceStatusCounts).map(Some(_)) - } - } - }.onlyIf { (_, ctx) => - filterRequestedCounts( - ctx.opts.forUserId.getOrElse(ctx.userId), - ctx.opts.include.countsFields, - shouldHydrateBookmarksCount, - ctx.featureSwitchResults - ).nonEmpty - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCtx.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCtx.docx new file mode 100644 index 000000000..d770b318d Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCtx.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCtx.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCtx.scala deleted file mode 100644 index 5540dc8dc..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetCtx.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie -import com.twitter.tweetypie.core.TweetData -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ -import org.apache.thrift.protocol.TField - -/** - * Encapsulates basic, immutable details about a tweet to be hydrated, along with the - * `TweetQuery.Options`. Only tweet data that are not affected by hydration should be - * exposed here, as a single `TweetCtx` instance should be usable for the entire hydration - * of a tweet. - */ -trait TweetCtx { - def opts: TweetQuery.Options - - def tweetId: TweetId - def userId: UserId - def text: String - def createdAt: Time - def createdVia: String - def isRetweet: Boolean - def isReply: Boolean - def isSelfReply: Boolean - def sourceUserId: Option[UserId] - def sourceTweetId: Option[TweetId] - def inReplyToTweetId: Option[TweetId] - def geoCoordinates: Option[GeoCoordinates] - def placeId: Option[String] - def hasTakedown: Boolean - def quotedTweet: Option[QuotedTweet] - - def completedHydrations: Set[HydrationType] - - def isInitialInsert: Boolean = opts.cause.initialInsert(tweetId) - - def tweetFieldRequested(field: TField): Boolean = tweetFieldRequested(field.id) - def tweetFieldRequested(fieldId: FieldId): Boolean = opts.include.tweetFields.contains(fieldId) - - def mediaFieldRequested(field: TField): Boolean = mediaFieldRequested(field.id) - def mediaFieldRequested(fieldId: FieldId): Boolean = opts.include.mediaFields.contains(fieldId) -} - -object TweetCtx { - def from(td: TweetData, opts: TweetQuery.Options): TweetCtx = FromTweetData(td, opts) - - trait Proxy extends TweetCtx { - protected def underlyingTweetCtx: TweetCtx - - def opts: TweetQuery.Options = underlyingTweetCtx.opts - def tweetId: TweetId = underlyingTweetCtx.tweetId - def userId: UserId = underlyingTweetCtx.userId - def text: String = underlyingTweetCtx.text - def createdAt: Time = underlyingTweetCtx.createdAt - def createdVia: String = underlyingTweetCtx.createdVia - def isRetweet: Boolean = underlyingTweetCtx.isRetweet - def isReply: Boolean = underlyingTweetCtx.isReply - def isSelfReply: Boolean = underlyingTweetCtx.isSelfReply - def sourceUserId: Option[UserId] = underlyingTweetCtx.sourceUserId - def sourceTweetId: Option[TweetId] = underlyingTweetCtx.sourceTweetId - def inReplyToTweetId: Option[TweetId] = underlyingTweetCtx.inReplyToTweetId - def geoCoordinates: Option[GeoCoordinates] = underlyingTweetCtx.geoCoordinates - def placeId: Option[String] = underlyingTweetCtx.placeId - def hasTakedown: Boolean = underlyingTweetCtx.hasTakedown - def completedHydrations: Set[HydrationType] = underlyingTweetCtx.completedHydrations - def quotedTweet: Option[QuotedTweet] = underlyingTweetCtx.quotedTweet - } - - private case class FromTweetData(td: TweetData, opts: TweetQuery.Options) extends TweetCtx { - private val tweet = td.tweet - def tweetId: MediaId = tweet.id - def userId: UserId = getUserId(tweet) - def text: String = getText(tweet) - def createdAt: Time = getTimestamp(tweet) - def createdVia: String = TweetLenses.createdVia.get(tweet) - def isRetweet: Boolean = getShare(tweet).isDefined - def isSelfReply: Boolean = tweetypie.isSelfReply(tweet) - def isReply: Boolean = getReply(tweet).isDefined - def sourceUserId: Option[MediaId] = getShare(tweet).map(_.sourceUserId) - def sourceTweetId: Option[MediaId] = getShare(tweet).map(_.sourceStatusId) - def inReplyToTweetId: Option[MediaId] = getReply(tweet).flatMap(_.inReplyToStatusId) - def geoCoordinates: Option[GeoCoordinates] = TweetLenses.geoCoordinates.get(tweet) - def placeId: Option[String] = TweetLenses.placeId.get(tweet) - def hasTakedown: Boolean = TweetLenses.hasTakedown(tweet) - def completedHydrations: Set[HydrationType] = td.completedHydrations - def quotedTweet: Option[QuotedTweet] = getQuotedTweet(tweet) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetHydration.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetHydration.docx new file mode 100644 index 000000000..4bc424c17 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetHydration.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetHydration.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetHydration.scala deleted file mode 100644 index a12295322..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetHydration.scala +++ /dev/null @@ -1,848 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.expandodo.thriftscala.Card -import com.twitter.expandodo.thriftscala.Card2 -import com.twitter.servo.cache.Cached -import com.twitter.servo.cache.CachedValueStatus -import com.twitter.servo.cache.LockingCache -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.media.thriftscala.MediaRef -import com.twitter.tweetypie.repository.PastedMedia -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.repository.TweetRepoCachePicker -import com.twitter.tweetypie.repository.TweetResultRepository -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.util.Takedowns -import com.twitter.util.Return -import com.twitter.util.Throw - -object TweetHydration { - - /** - * Wires up a set of hydrators that include those whose results are cached on the tweet, - * and some whose results are not cached but depend upon the results of the former. - */ - def apply( - hydratorStats: StatsReceiver, - hydrateFeatureSwitchResults: TweetDataValueHydrator, - hydrateMentions: MentionEntitiesHydrator.Type, - hydrateLanguage: LanguageHydrator.Type, - hydrateUrls: UrlEntitiesHydrator.Type, - hydrateQuotedTweetRef: QuotedTweetRefHydrator.Type, - hydrateQuotedTweetRefUrls: QuotedTweetRefUrlsHydrator.Type, - hydrateMediaCacheable: MediaEntitiesHydrator.Cacheable.Type, - hydrateReplyScreenName: ReplyScreenNameHydrator.Type, - hydrateConvoId: ConversationIdHydrator.Type, - hydratePerspective: PerspectiveHydrator.Type, - hydrateEditPerspective: EditPerspectiveHydrator.Type, - hydrateConversationMuted: ConversationMutedHydrator.Type, - hydrateContributor: ContributorHydrator.Type, - hydrateTakedowns: TakedownHydrator.Type, - hydrateDirectedAt: DirectedAtHydrator.Type, - hydrateGeoScrub: GeoScrubHydrator.Type, - hydrateCacheableRepairs: TweetDataValueHydrator, - hydrateMediaUncacheable: MediaEntitiesHydrator.Uncacheable.Type, - hydratePostCacheRepairs: TweetDataValueHydrator, - hydrateTweetLegacyFormat: TweetDataValueHydrator, - hydrateQuoteTweetVisibility: QuoteTweetVisibilityHydrator.Type, - hydrateQuotedTweet: QuotedTweetHydrator.Type, - hydratePastedMedia: PastedMediaHydrator.Type, - hydrateMediaRefs: MediaRefsHydrator.Type, - hydrateMediaTags: MediaTagsHydrator.Type, - hydrateClassicCards: CardHydrator.Type, - hydrateCard2: Card2Hydrator.Type, - hydrateContributorVisibility: ContributorVisibilityFilter.Type, - hydrateHasMedia: HasMediaHydrator.Type, - hydrateTweetCounts: TweetCountsHydrator.Type, - hydratePreviousTweetCounts: PreviousTweetCountsHydrator.Type, - hydratePlace: PlaceHydrator.Type, - hydrateDeviceSource: DeviceSourceHydrator.Type, - hydrateProfileGeo: ProfileGeoHydrator.Type, - hydrateSourceTweet: SourceTweetHydrator.Type, - hydrateIM1837State: IM1837FilterHydrator.Type, - hydrateIM2884State: IM2884FilterHydrator.Type, - hydrateIM3433State: IM3433FilterHydrator.Type, - hydrateTweetAuthorVisibility: TweetAuthorVisibilityHydrator.Type, - hydrateReportedTweetVisibility: ReportedTweetFilter.Type, - scrubSuperfluousUrlEntities: TweetDataValueHydrator, - copyFromSourceTweet: TweetDataValueHydrator, - hydrateTweetVisibility: TweetVisibilityHydrator.Type, - hydrateEscherbirdAnnotations: EscherbirdAnnotationHydrator.Type, - hydrateScrubEngagements: ScrubEngagementHydrator.Type, - hydrateConversationControl: ConversationControlHydrator.Type, - hydrateEditControl: EditControlHydrator.Type, - hydrateUnmentionData: UnmentionDataHydrator.Type, - hydrateNoteTweetSuffix: TweetDataValueHydrator - ): TweetDataValueHydrator = { - val scrubCachedTweet: TweetDataValueHydrator = - ValueHydrator - .fromMutation[Tweet, TweetQuery.Options]( - ScrubUncacheable.tweetMutation.countMutations(hydratorStats.counter("scrub_cached_tweet")) - ) - .lensed(TweetData.Lenses.tweet) - .onlyIf((td, opts) => opts.cause.reading(td.tweet.id)) - - // We perform independent hydrations of individual bits of - // data and pack the results into tuples instead of updating - // the tweet for each one in order to avoid making lots of - // copies of the tweet. - - val hydratePrimaryCacheableFields: TweetDataValueHydrator = - ValueHydrator[TweetData, TweetQuery.Options] { (td, opts) => - val ctx = TweetCtx.from(td, opts) - val tweet = td.tweet - - val urlsMediaQuoteTweet: Stitch[ - ValueState[(Seq[UrlEntity], Seq[MediaEntity], Option[QuotedTweet])] - ] = - for { - urls <- hydrateUrls(getUrls(tweet), ctx) - (media, quotedTweet) <- Stitch.join( - hydrateMediaCacheable( - getMedia(tweet), - MediaEntityHydrator.Cacheable.Ctx(urls.value, ctx) - ), - for { - qtRef <- hydrateQuotedTweetRef( - tweet.quotedTweet, - QuotedTweetRefHydrator.Ctx(urls.value, ctx) - ) - qtRefWithUrls <- hydrateQuotedTweetRefUrls(qtRef.value, ctx) - } yield { - ValueState(qtRefWithUrls.value, qtRef.state ++ qtRefWithUrls.state) - } - ) - } yield { - ValueState.join(urls, media, quotedTweet) - } - - val conversationId: Stitch[ValueState[Option[ConversationId]]] = - hydrateConvoId(getConversationId(tweet), ctx) - - val mentions: Stitch[ValueState[Seq[MentionEntity]]] = - hydrateMentions(getMentions(tweet), ctx) - - val replyScreenName: Stitch[ValueState[Option[Reply]]] = - hydrateReplyScreenName(getReply(tweet), ctx) - - val directedAt: Stitch[ValueState[Option[DirectedAtUser]]] = - hydrateDirectedAt( - getDirectedAtUser(tweet), - DirectedAtHydrator.Ctx( - mentions = getMentions(tweet), - metadata = tweet.directedAtUserMetadata, - underlyingTweetCtx = ctx - ) - ) - - val language: Stitch[ValueState[Option[Language]]] = - hydrateLanguage(tweet.language, ctx) - - val contributor: Stitch[ValueState[Option[Contributor]]] = - hydrateContributor(tweet.contributor, ctx) - - val geoScrub: Stitch[ValueState[(Option[GeoCoordinates], Option[PlaceId])]] = - hydrateGeoScrub( - (TweetLenses.geoCoordinates(tweet), TweetLenses.placeId(tweet)), - ctx - ) - - Stitch - .joinMap( - urlsMediaQuoteTweet, - conversationId, - mentions, - replyScreenName, - directedAt, - language, - contributor, - geoScrub - )(ValueState.join(_, _, _, _, _, _, _, _)) - .map { values => - if (values.state.isEmpty) { - ValueState.unmodified(td) - } else { - values.map { - case ( - (urls, media, quotedTweet), - conversationId, - mentions, - reply, - directedAt, - language, - contributor, - coreGeo - ) => - val (coordinates, placeId) = coreGeo - td.copy( - tweet = tweet.copy( - coreData = tweet.coreData.map( - _.copy( - reply = reply, - conversationId = conversationId, - directedAtUser = directedAt, - coordinates = coordinates, - placeId = placeId - ) - ), - urls = Some(urls), - media = Some(media), - mentions = Some(mentions), - language = language, - quotedTweet = quotedTweet, - contributor = contributor - ) - ) - } - } - } - } - - val assertNotScrubbed: TweetDataValueHydrator = - ValueHydrator.fromMutation[TweetData, TweetQuery.Options]( - ScrubUncacheable - .assertNotScrubbed( - "output of the cacheable tweet hydrator should not require scrubbing" - ) - .lensed(TweetData.Lenses.tweet) - ) - - val hydrateDependentUncacheableFields: TweetDataValueHydrator = - ValueHydrator[TweetData, TweetQuery.Options] { (td, opts) => - val ctx = TweetCtx.from(td, opts) - val tweet = td.tweet - - val quotedTweetResult: Stitch[ValueState[Option[QuotedTweetResult]]] = - for { - qtFilterState <- hydrateQuoteTweetVisibility(None, ctx) - quotedTweet <- hydrateQuotedTweet( - td.quotedTweetResult, - QuotedTweetHydrator.Ctx(qtFilterState.value, ctx) - ) - } yield { - ValueState.join(qtFilterState, quotedTweet).map(_._2) - } - - val pastedMedia: Stitch[ValueState[PastedMedia]] = - hydratePastedMedia( - PastedMediaHydrator.getPastedMedia(tweet), - PastedMediaHydrator.Ctx(getUrls(tweet), ctx) - ) - - val mediaTags: Stitch[ValueState[Option[TweetMediaTags]]] = - hydrateMediaTags(tweet.mediaTags, ctx) - - val classicCards: Stitch[ValueState[Option[Seq[Card]]]] = - hydrateClassicCards( - tweet.cards, - CardHydrator.Ctx(getUrls(tweet), getMedia(tweet), ctx) - ) - - val card2: Stitch[ValueState[Option[Card2]]] = - hydrateCard2( - tweet.card2, - Card2Hydrator.Ctx( - getUrls(tweet), - getMedia(tweet), - getCardReference(tweet), - ctx, - td.featureSwitchResults - ) - ) - - val contributorVisibility: Stitch[ValueState[Option[Contributor]]] = - hydrateContributorVisibility(tweet.contributor, ctx) - - val takedowns: Stitch[ValueState[Option[Takedowns]]] = - hydrateTakedowns( - None, // None because uncacheable hydrator doesn't depend on previous value - TakedownHydrator.Ctx(Takedowns.fromTweet(tweet), ctx) - ) - - val conversationControl: Stitch[ValueState[Option[ConversationControl]]] = - hydrateConversationControl( - tweet.conversationControl, - ConversationControlHydrator.Ctx(getConversationId(tweet), ctx) - ) - - // PreviousTweetCounts and Perspective hydration depends on tweet.editControl.edit_control_initial - // having been hydrated in EditControlHydrator; thus we are chaining them together. - val editControlWithDependencies: Stitch[ - ValueState[ - ( - Option[EditControl], - Option[StatusPerspective], - Option[StatusCounts], - Option[TweetPerspective] - ) - ] - ] = - for { - (edit, perspective) <- Stitch.join( - hydrateEditControl(tweet.editControl, ctx), - hydratePerspective( - tweet.perspective, - PerspectiveHydrator.Ctx(td.featureSwitchResults, ctx)) - ) - (counts, editPerspective) <- Stitch.join( - hydratePreviousTweetCounts( - tweet.previousCounts, - PreviousTweetCountsHydrator.Ctx(edit.value, td.featureSwitchResults, ctx)), - hydrateEditPerspective( - tweet.editPerspective, - EditPerspectiveHydrator - .Ctx(perspective.value, edit.value, td.featureSwitchResults, ctx)) - ) - } yield { - ValueState.join(edit, perspective, counts, editPerspective) - } - - Stitch - .joinMap( - quotedTweetResult, - pastedMedia, - mediaTags, - classicCards, - card2, - contributorVisibility, - takedowns, - conversationControl, - editControlWithDependencies - )(ValueState.join(_, _, _, _, _, _, _, _, _)) - .map { values => - if (values.state.isEmpty) { - ValueState.unmodified(td) - } else { - values.map { - case ( - quotedTweetResult, - pastedMedia, - ownedMediaTags, - cards, - card2, - contributor, - takedowns, - conversationControl, - (editControl, perspective, previousCounts, editPerspective) - ) => - td.copy( - tweet = tweet.copy( - media = Some(pastedMedia.mediaEntities), - mediaTags = pastedMedia.mergeTweetMediaTags(ownedMediaTags), - cards = cards, - card2 = card2, - contributor = contributor, - takedownCountryCodes = takedowns.map(_.countryCodes.toSeq), - takedownReasons = takedowns.map(_.reasons.toSeq), - conversationControl = conversationControl, - editControl = editControl, - previousCounts = previousCounts, - perspective = perspective, - editPerspective = editPerspective, - ), - quotedTweetResult = quotedTweetResult - ) - } - } - } - } - - val hydrateIndependentUncacheableFields: TweetDataEditHydrator = - EditHydrator[TweetData, TweetQuery.Options] { (td, opts) => - val ctx = TweetCtx.from(td, opts) - val tweet = td.tweet - - // Group together the results of hydrators that don't perform - // filtering, because we don't care about the precedence of - // exceptions from these hydrators, because the exceptions all - // indicate failures, and picking any failure will be - // fine. (All of the other hydrators might throw filtering - // exceptions, so we need to make sure that we give precedence - // to their failures.) - val hydratorsWithoutFiltering = - Stitch.joinMap( - hydrateTweetCounts(tweet.counts, TweetCountsHydrator.Ctx(td.featureSwitchResults, ctx)), - // Note: Place is cached in memcache, it is just not cached on the Tweet. - hydratePlace(tweet.place, ctx), - hydrateDeviceSource(tweet.deviceSource, ctx), - hydrateProfileGeo(tweet.profileGeoEnrichment, ctx) - )(ValueState.join(_, _, _, _)) - - /** - * Multiple hydrators throw visibility filtering exceptions so specify an order to achieve - * a deterministic hydration result while ensuring that any retweet has a source tweet: - * 1. hydrateSourceTweet throws SourceTweetNotFound, this is a detached-retweet so treat - * the retweet hydration as if it were not found - * 2. hydrateTweetAuthorVisibility - * 3. hydrateSourceTweet (other than SourceTweetNotFound already handled above) - * 4. hydrateIM1837State - * 5. hydrateIM2884State - * 6. hydrateIM3433State - * 7. hydratorsWithoutFiltering miscellaneous exceptions (any visibility filtering - * exceptions should win over failure of a hydrator) - */ - val sourceTweetAndTweetAuthorResult = - Stitch - .joinMap( - hydrateSourceTweet(td.sourceTweetResult, ctx).liftToTry, - hydrateTweetAuthorVisibility((), ctx).liftToTry, - hydrateIM1837State((), ctx).liftToTry, - hydrateIM2884State((), ctx).liftToTry, - hydrateIM3433State((), ctx).liftToTry - ) { - case (Throw(t @ FilteredState.Unavailable.SourceTweetNotFound(_)), _, _, _, _) => - Throw(t) - case (_, Throw(t), _, _, _) => Throw(t) // TweetAuthorVisibility - case (Throw(t), _, _, _, _) => Throw(t) // SourceTweet - case (_, _, Throw(t), _, _) => Throw(t) // IM1837State - case (_, _, _, Throw(t), _) => Throw(t) // IM2884State - case (_, _, _, _, Throw(t)) => Throw(t) // IM3433State - case ( - Return(sourceTweetResultValue), - Return(authorVisibilityValue), - Return(im1837Value), - Return(im2884Value), - Return(im3433Value) - ) => - Return( - ValueState - .join( - sourceTweetResultValue, - authorVisibilityValue, - im1837Value, - im2884Value, - im3433Value - ) - ) - }.lowerFromTry - - StitchExceptionPrecedence(sourceTweetAndTweetAuthorResult) - .joinWith(hydratorsWithoutFiltering)(ValueState.join(_, _)) - .toStitch - .map { values => - if (values.state.isEmpty) { - EditState.unit[TweetData] - } else { - EditState[TweetData] { tweetData => - val tweet = tweetData.tweet - values.map { - case ( - (sourceTweetResult, _, _, _, _), - (counts, place, deviceSource, profileGeo) - ) => - tweetData.copy( - tweet = tweet.copy( - counts = counts, - place = place, - deviceSource = deviceSource, - profileGeoEnrichment = profileGeo - ), - sourceTweetResult = sourceTweetResult - ) - } - } - } - } - } - - val hydrateUnmentionDataToTweetData: TweetDataValueHydrator = - TweetHydration.setOnTweetData( - TweetData.Lenses.tweet.andThen(TweetLenses.unmentionData), - (td: TweetData, opts: TweetQuery.Options) => - UnmentionDataHydrator - .Ctx(getConversationId(td.tweet), getMentions(td.tweet), TweetCtx.from(td, opts)), - hydrateUnmentionData - ) - - val hydrateCacheableFields: TweetDataValueHydrator = - ValueHydrator.inSequence( - scrubCachedTweet, - hydratePrimaryCacheableFields, - // Relies on mentions being hydrated in hydratePrimaryCacheableFields - hydrateUnmentionDataToTweetData, - assertNotScrubbed, - hydrateCacheableRepairs - ) - - // The conversation muted hydrator needs the conversation id, - // which comes from the primary cacheable fields, and the media hydrator - // needs the cacheable media entities. - val hydrateUncacheableMedia: TweetDataValueHydrator = - ValueHydrator[TweetData, TweetQuery.Options] { (td, opts) => - val ctx = TweetCtx.from(td, opts) - val tweet = td.tweet - - val mediaCtx = - MediaEntityHydrator.Uncacheable.Ctx(td.tweet.mediaKeys, ctx) - - val media: Stitch[ValueState[Option[Seq[MediaEntity]]]] = - hydrateMediaUncacheable.liftOption.apply(td.tweet.media, mediaCtx) - - val conversationMuted: Stitch[ValueState[Option[Boolean]]] = - hydrateConversationMuted( - tweet.conversationMuted, - ConversationMutedHydrator.Ctx(getConversationId(tweet), ctx) - ) - - // MediaRefs need to be hydrated at this phase because they rely on the media field - // on the Tweet, which can get unset by later hydrators. - val mediaRefs: Stitch[ValueState[Option[Seq[MediaRef]]]] = - hydrateMediaRefs( - tweet.mediaRefs, - MediaRefsHydrator.Ctx(getMedia(tweet), getMediaKeys(tweet), getUrls(tweet), ctx) - ) - - Stitch - .joinMap( - media, - conversationMuted, - mediaRefs - )(ValueState.join(_, _, _)) - .map { values => - if (values.state.isEmpty) { - ValueState.unmodified(td) - } else { - val tweet = td.tweet - values.map { - case (media, conversationMuted, mediaRefs) => - td.copy( - tweet = tweet.copy( - media = media, - conversationMuted = conversationMuted, - mediaRefs = mediaRefs - ) - ) - } - } - } - } - - val hydrateHasMediaToTweetData: TweetDataValueHydrator = - TweetHydration.setOnTweetData( - TweetData.Lenses.tweet.andThen(TweetLenses.hasMedia), - (td: TweetData, opts: TweetQuery.Options) => td.tweet, - hydrateHasMedia - ) - - val hydrateReportedTweetVisibilityToTweetData: TweetDataValueHydrator = { - // Create a TweetDataValueHydrator that calls hydrateReportedTweetVisibility, which - // either throws a FilteredState.Unavailable or returns Unit. - ValueHydrator[TweetData, TweetQuery.Options] { (td, opts) => - val ctx = ReportedTweetFilter.Ctx(td.tweet.perspective, TweetCtx.from(td, opts)) - hydrateReportedTweetVisibility((), ctx).map { _ => - ValueState.unmodified(td) - } - } - } - - val hydrateTweetVisibilityToTweetData: TweetDataValueHydrator = - TweetHydration.setOnTweetData( - TweetData.Lenses.suppress, - (td: TweetData, opts: TweetQuery.Options) => - TweetVisibilityHydrator.Ctx(td.tweet, TweetCtx.from(td, opts)), - hydrateTweetVisibility - ) - - val hydrateEscherbirdAnnotationsToTweetAndCachedTweet: TweetDataValueHydrator = - TweetHydration.setOnTweetAndCachedTweet( - TweetLenses.escherbirdEntityAnnotations, - (td: TweetData, _: TweetQuery.Options) => td.tweet, - hydrateEscherbirdAnnotations - ) - - val scrubEngagements: TweetDataValueHydrator = - TweetHydration.setOnTweetData( - TweetData.Lenses.tweetCounts, - (td: TweetData, _: TweetQuery.Options) => ScrubEngagementHydrator.Ctx(td.suppress), - hydrateScrubEngagements - ) - - /** - * This is where we wire up all the separate hydrators into a single [[TweetDataValueHydrator]]. - * - * Each hydrator here is either a [[TweetDataValueHydrator]] or a [[TweetDataEditHydrator]]. - * We use [[EditHydrator]]s for anything that needs to run in parallel ([[ValueHydrator]]s can - * only be run in sequence). - */ - ValueHydrator.inSequence( - // Hydrate FeatureSwitchResults first, so they can be used by other hydrators if needed - hydrateFeatureSwitchResults, - EditHydrator - .inParallel( - ValueHydrator - .inSequence( - // The result of running these hydrators is saved as `cacheableTweetResult` and - // written back to cache via `cacheChangesEffect` in `hydrateRepo` - TweetHydration.captureCacheableTweetResult( - hydrateCacheableFields - ), - // Uncacheable hydrators that depend only on the cacheable fields - hydrateUncacheableMedia, - // clean-up partially hydrated entities before any of the hydrators that look at - // url and media entities run, so that they never see bad entities. - hydratePostCacheRepairs, - // These hydrators are all dependent on each other and/or the previous hydrators - hydrateDependentUncacheableFields, - // Sets `hasMedia`. Comes after PastedMediaHydrator in order to include pasted - // pics as well as other media & urls. - hydrateHasMediaToTweetData - ) - .toEditHydrator, - // These hydrators do not rely on any other hydrators and so can be run in parallel - // with the above hydrators (and with each other) - hydrateIndependentUncacheableFields - ) - .toValueHydrator, - // Depends on reported perspectival having been hydrated in PerspectiveHydrator - hydrateReportedTweetVisibilityToTweetData, - // Remove superfluous urls entities when there is a corresponding MediaEntity for the same url - scrubSuperfluousUrlEntities, - // The copyFromSourceTweet hydrator needs to be located after the hydrators that produce the - // fields to copy. It must be located after PartialEntityCleaner (part of postCacheRepairs), - // which removes failed MediaEntities. It also depends on takedownCountryCodes having been - // hydrated in TakedownHydrator. - copyFromSourceTweet, - // depends on AdditionalFieldsHydrator and CopyFromSourceTweet to copy safety labels - hydrateTweetVisibilityToTweetData, - // for IPI'd tweets, we want to disable tweet engagement counts from being returned - // StatusCounts for replyCount, retweetCount. - // scrubEngagements hydrator must come after tweet visibility hydrator. - // tweet visibility hydrator emits the suppressed FilteredState needed for scrubbing. - scrubEngagements, - // this hydrator runs when writing the current tweet - // Escherbird comes last in order to consume a tweet that's as close as possible - // to the tweet written to tweet_events - hydrateEscherbirdAnnotationsToTweetAndCachedTweet - .onlyIf((td, opts) => opts.cause.writing(td.tweet.id)), - // Add an ellipsis to the end of the text for a Tweet that has a NoteTweet associated. - // This is so that the Tweet is displayed on the home timeline with an ellipsis, letting - // the User know that there's more to see. - hydrateNoteTweetSuffix, - /** - * Post-cache repair of QT text and entities to support rendering on all clients - * Moving this to end of the pipeline to avoid/minimize chance of following hydrators - * depending on modified tweet text or entities. - * When we start persisting shortUrl in MH - permalink won't be empty. therefore, - * we won't run QuotedTweetRefHydrator and just hydrate expanded and display - * using QuotedTweetRefUrlsHydrator. We will use hydrated permalink to repair - * QT text and entities for non-upgraded clients in this step. - * */ - hydrateTweetLegacyFormat - ) - } - - /** - * Returns a new hydrator that takes the produced result, and captures the result value - * in the `cacheableTweetResult` field of the enclosed `TweetData`. - */ - def captureCacheableTweetResult(h: TweetDataValueHydrator): TweetDataValueHydrator = - ValueHydrator[TweetData, TweetQuery.Options] { (td, opts) => - h(td, opts).map { v => - // In addition to saving off a copy of ValueState, make sure that the TweetData inside - // the ValueState has its "completedHydrations" set to the ValueState.HydrationStates's - // completedHydrations. This is used when converting to a CachedTweet. - v.map { td => - td.copy( - cacheableTweetResult = Some(v.map(_.addHydrated(v.state.completedHydrations))) - ) - } - } - } - - /** - * Takes a ValueHydrator and a Lens and returns a `TweetDataValueHydrator` that does three things: - * - * 1. Runs the ValueHydrator on the lensed value - * 2. Saves the result back to the main tweet using the lens - * 3. Saves the result back to the tweet in cacheableTweetResult using the lens - */ - def setOnTweetAndCachedTweet[A, C]( - l: Lens[Tweet, A], - mkCtx: (TweetData, TweetQuery.Options) => C, - h: ValueHydrator[A, C] - ): TweetDataValueHydrator = { - // A lens that goes from TweetData -> tweet -> l - val tweetDataLens = TweetData.Lenses.tweet.andThen(l) - - // A lens that goes from TweetData -> cacheableTweetResult -> tweet -> l - val cachedTweetLens = - TweetLenses - .requireSome(TweetData.Lenses.cacheableTweetResult) - .andThen(TweetResult.Lenses.tweet) - .andThen(l) - - ValueHydrator[TweetData, TweetQuery.Options] { (td, opts) => - h.run(tweetDataLens.get(td), mkCtx(td, opts)).map { r => - if (r.state.isEmpty) { - ValueState.unmodified(td) - } else { - r.map { v => Lens.setAll(td, tweetDataLens -> v, cachedTweetLens -> v) } - } - } - } - } - - /** - * Creates a `TweetDataValueHydrator` that hydrates a lensed value, overwriting - * the existing value. - */ - def setOnTweetData[A, C]( - lens: Lens[TweetData, A], - mkCtx: (TweetData, TweetQuery.Options) => C, - h: ValueHydrator[A, C] - ): TweetDataValueHydrator = - ValueHydrator[TweetData, TweetQuery.Options] { (td, opts) => - h.run(lens.get(td), mkCtx(td, opts)).map { r => - if (r.state.isEmpty) ValueState.unmodified(td) else r.map(lens.set(td, _)) - } - } - - /** - * Produces an [[Effect]] that can be applied to a [[TweetDataValueHydrator]] to write updated - * values back to cache. - */ - def cacheChanges( - cache: LockingCache[TweetId, Cached[TweetData]], - stats: StatsReceiver - ): Effect[ValueState[TweetData]] = { - val updatedCounter = stats.counter("updated") - val unchangedCounter = stats.counter("unchanged") - val picker = new TweetRepoCachePicker[TweetData](_.cachedAt) - val cacheErrorCounter = stats.counter("cache_error") - val missingCacheableResultCounter = stats.counter("missing_cacheable_result") - - Effect[TweetResult] { result => - // cacheErrorEncountered will never be set on `cacheableTweetResult`, so we need to - // look at the outer tweet state. - val cacheErrorEncountered = result.state.cacheErrorEncountered - - result.value.cacheableTweetResult match { - case Some(ValueState(td, state)) if state.modified && !cacheErrorEncountered => - val tweetData = td.addHydrated(state.completedHydrations) - val now = Time.now - val cached = Cached(Some(tweetData), CachedValueStatus.Found, now, Some(now)) - val handler = LockingCache.PickingHandler(cached, picker) - - updatedCounter.incr() - cache.lockAndSet(tweetData.tweet.id, handler) - - case Some(ValueState(_, _)) if cacheErrorEncountered => - cacheErrorCounter.incr() - - case None => - missingCacheableResultCounter.incr() - - case _ => - unchangedCounter.incr() - } - } - } - - /** - * Wraps a hydrator with a check such that it only executes the hydrator if `queryFilter` - * returns true for the `TweetQuery.Option` in the `Ctx` value, and the specified - * `HydrationType` is not already marked as having been completed in - * `ctx.tweetData.completedHydrations`. If these conditions pass, and the underlying - * hydrator is executed, and the result does not contain a field-level or total failure, - * then the resulting `HydrationState` is updated to indicate that the specified - * `HydrationType` has been completed. - */ - def completeOnlyOnce[A, C <: TweetCtx]( - queryFilter: TweetQuery.Options => Boolean = _ => true, - hydrationType: HydrationType, - dependsOn: Set[HydrationType] = Set.empty, - hydrator: ValueHydrator[A, C] - ): ValueHydrator[A, C] = { - val completedState = HydrationState.modified(hydrationType) - - ValueHydrator[A, C] { (a, ctx) => - hydrator(a, ctx).map { res => - if (res.state.failedFields.isEmpty && - dependsOn.forall(ctx.completedHydrations.contains)) { - // successful result! - if (!ctx.completedHydrations.contains(hydrationType)) { - res.copy(state = res.state ++ completedState) - } else { - // forced rehydration - don't add hydrationType or change modified flag - res - } - } else { - // hydration failed or not all dependencies satisfied so don't mark as complete - res - } - } - }.onlyIf { (a, ctx) => - queryFilter(ctx.opts) && - (!ctx.completedHydrations.contains(hydrationType)) - } - } - - /** - * Applies a `TweetDataValueHydrator` to a `TweetRepository.Type`-typed repository. - * The incoming `TweetQuery.Options` are first expanded using `optionsExpander`, and the - * resulting options passed to `repo` and `hydrator`. The resulting tweet result - * objects are passed to `cacheChangesEffect` for possible write-back to cache. Finally, - * the tweets are scrubbed according to the original input `TweetQuery.Options`. - */ - def hydrateRepo( - hydrator: TweetDataValueHydrator, - cacheChangesEffect: Effect[TweetResult], - optionsExpander: TweetQueryOptionsExpander.Type - )( - repo: TweetResultRepository.Type - ): TweetResultRepository.Type = - (tweetId: TweetId, originalOpts: TweetQuery.Options) => { - val expandedOpts = optionsExpander(originalOpts) - - for { - repoResult <- repo(tweetId, expandedOpts) - hydratorResult <- hydrator(repoResult.value, expandedOpts) - } yield { - val hydratingRepoResult = - TweetResult(hydratorResult.value, repoResult.state ++ hydratorResult.state) - - if (originalOpts.cacheControl.writeToCache) { - cacheChangesEffect(hydratingRepoResult) - } - - UnrequestedFieldScrubber(originalOpts).scrub(hydratingRepoResult) - } - } - - /** - * A trivial wrapper around a Stitch[_] to provide a `joinWith` - * method that lets us choose the precedence of exceptions. - * - * This wrapper is useful for the case in which it's important that - * we specify which of the two exceptions wins (such as visibility - * filtering). - * - * Since this is an [[AnyVal]], using this is no more expensive than - * inlining the joinWith method. - */ - // exposed for testing - case class StitchExceptionPrecedence[A](toStitch: Stitch[A]) extends AnyVal { - - /** - * Concurrently evaluate two Stitch[_] values. This is different - * from Stitch.join in that any exception from the expression on - * the left hand side will take precedence over an exception on - * the right hand side. This means that an exception from the - * right-hand side will not short-circuit evaluation, but an - * exception on the left-hand side *will* short-circuit. This is - * desirable because it allows us to return the failure with as - * little latency as possible. (Compare to lifting *both* to Try, - * which would force us to wait for both computations to complete - * before returning, even if the one with the higher precedence is - * already known to be an exception.) - */ - def joinWith[B, C](rhs: Stitch[B])(f: (A, B) => C): StitchExceptionPrecedence[C] = - StitchExceptionPrecedence { - Stitch - .joinMap(toStitch, rhs.liftToTry) { (a, tryB) => tryB.map(b => f(a, b)) } - .lowerFromTry - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetLegacyFormatter.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetLegacyFormatter.docx new file mode 100644 index 000000000..a845426a6 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetLegacyFormatter.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetLegacyFormatter.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetLegacyFormatter.scala deleted file mode 100644 index adadcefd0..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetLegacyFormatter.scala +++ /dev/null @@ -1,330 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.media.Media -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.serverutil.ExtendedTweetMetadataBuilder -import com.twitter.tweetypie.thriftscala.UrlEntity -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.thriftscala.entities.Implicits._ -import com.twitter.tweetypie.tweettext.Offset -import com.twitter.tweetypie.tweettext.TextModification -import com.twitter.tweetypie.tweettext.TweetText -import com.twitter.tweetypie.util.EditControlUtil -import com.twitter.tweetypie.util.TweetLenses - -/** - * This hydrator is the backwards-compatibility layer to support QT, Edit Tweets & Mixed Media - * Tweets rendering on legacy non-updated clients. Legacy rendering provides a way for every client - * to consume these Tweets until the client is upgraded. For Edit and Mixed Media Tweets, the - * Tweet's self-permalink is appended to the visible text. For Quoting Tweets, the Quoted Tweet's - * permalink is appended to the text. For Tweets that meet multiple criteria for legacy rendering - * (e.g. QT containing Mixed Media), only one permalink is appended and the self-permalink takes - * precedence. - */ -object TweetLegacyFormatter { - - private[this] val log = Logger(getClass) - - import TweetText._ - - def legacyQtPermalink( - td: TweetData, - opts: TweetQuery.Options - ): Option[ShortenedUrl] = { - val tweet = td.tweet - val tweetText = TweetLenses.text(tweet) - val urls = TweetLenses.urls(tweet) - val ctx = TweetCtx.from(td, opts) - val qtPermalink: Option[ShortenedUrl] = tweet.quotedTweet.flatMap(_.permalink) - val qtShortUrl = qtPermalink.map(_.shortUrl) - - def urlsContains(url: String): Boolean = - urls.exists(_.url == url) - - val doLegacyQtFormatting = - !opts.simpleQuotedTweet && !ctx.isRetweet && - qtPermalink.isDefined && qtShortUrl.isDefined && - !qtShortUrl.exists(tweetText.contains) && - !qtShortUrl.exists(urlsContains) - - if (doLegacyQtFormatting) qtPermalink else None - } - - def legacySelfPermalink( - td: TweetData - ): Option[ShortenedUrl] = { - val tweet = td.tweet - val selfPermalink = tweet.selfPermalink - val tweetText = TweetLenses.text(tweet) - val urls = TweetLenses.urls(tweet) - val selfShortUrl = selfPermalink.map(_.shortUrl) - - def urlsContains(url: String): Boolean = - urls.exists(_.url == url) - - val doLegacyFormatting = - selfPermalink.isDefined && selfShortUrl.isDefined && - !selfShortUrl.exists(tweetText.contains) && - !selfShortUrl.exists(urlsContains) && - needsLegacyFormatting(td) - - if (doLegacyFormatting) selfPermalink else None - } - - def isMixedMediaTweet(tweet: Tweet): Boolean = - tweet.media.exists(Media.isMixedMedia) - - def buildUrlEntity(from: Short, to: Short, permalink: ShortenedUrl): UrlEntity = - UrlEntity( - fromIndex = from, - toIndex = to, - url = permalink.shortUrl, - expanded = Some(permalink.longUrl), - display = Some(permalink.displayText) - ) - - private[this] def isValidVisibleRange( - tweetIdForLogging: TweetId, - textRange: TextRange, - textLength: Int - ) = { - val isValid = textRange.fromIndex <= textRange.toIndex && textRange.toIndex <= textLength - if (!isValid) { - log.warn(s"Tweet $tweetIdForLogging has invalid visibleTextRange: $textRange") - } - isValid - } - - // This Function checks if legacy formatting is required for Edit & Mixed Media Tweets. - // Calls FeatureSwitches.matchRecipient which is an expensive call, - // so caution is taken to call it only once and only when needed. - def needsLegacyFormatting( - td: TweetData - ): Boolean = { - val isEdit = EditControlUtil.isEditTweet(td.tweet) - val isMixedMedia = isMixedMediaTweet(td.tweet) - val isNoteTweet = td.tweet.noteTweet.isDefined - - if (isEdit || isMixedMedia || isNoteTweet) { - - // These feature switches are disabled unless greater than certain android, ios versions - // & all versions of RWEB. - val TweetEditConsumptionEnabledKey = "tweet_edit_consumption_enabled" - val MixedMediaEnabledKey = "mixed_media_enabled" - val NoteTweetConsumptionEnabledKey = "note_tweet_consumption_enabled" - - def fsEnabled(fsKey: String): Boolean = { - td.featureSwitchResults - .flatMap(_.getBoolean(fsKey, shouldLogImpression = false)) - .getOrElse(false) - } - - val tweetEditConsumptionEnabled = fsEnabled(TweetEditConsumptionEnabledKey) - val mixedMediaEnabled = fsEnabled(MixedMediaEnabledKey) - val noteTweetConsumptionEnabled = fsEnabled(NoteTweetConsumptionEnabledKey) - - (isEdit && !tweetEditConsumptionEnabled) || - (isMixedMedia && !mixedMediaEnabled) || - (isNoteTweet && !noteTweetConsumptionEnabled) - } else { - false - } - } - - //given a permalink, the tweet text gets updated - def updateTextAndURLsAndMedia( - permalink: ShortenedUrl, - tweet: Tweet, - statsReceiver: StatsReceiver - ): Tweet = { - - val originalText = TweetLenses.text(tweet) - val originalTextLength = codePointLength(originalText) - - // Default the visible range to the whole tweet if the existing visible range is invalid. - val visibleRange: TextRange = - TweetLenses - .visibleTextRange(tweet) - .filter((r: TextRange) => isValidVisibleRange(tweet.id, r, originalTextLength)) - .getOrElse(TextRange(0, originalTextLength)) - - val permalinkShortUrl = permalink.shortUrl - val insertAtCodePoint = Offset.CodePoint(visibleRange.toIndex) - - /* - * Insertion at position 0 implies that the original tweet text has no - * visible text, so the resulting text should be only the url without - * leading padding. - */ - val padLeft = if (insertAtCodePoint.toInt > 0) " " else "" - - /* - * Empty visible text at position 0 implies that the original tweet text - * only contains a URL in the hidden suffix area, which would not already - * be padded. - */ - val padRight = if (visibleRange == TextRange(0, 0)) " " else "" - val paddedShortUrl = s"$padLeft$permalinkShortUrl$padRight" - - val tweetTextModification = TextModification.insertAt( - originalText, - insertAtCodePoint, - paddedShortUrl - ) - - /* - * As we modified tweet text and appended tweet permalink above - * we have to correct the url and media entities accordingly as they are - * expected to be present in the hidden suffix of text. - * - * - we compute the new (from, to) indices for the url entity - * - build new url entity for quoted tweet permalink or self permalink for Edit/ MM Tweets - * - shift url entities which are after visible range end - * - shift media entities associated with above url entities - */ - val shortUrlLength = codePointLength(permalinkShortUrl) - val fromIndex = insertAtCodePoint.toInt + codePointLength(padLeft) - val toIndex = fromIndex + shortUrlLength - - val tweetUrlEntity = buildUrlEntity( - from = fromIndex.toShort, - to = toIndex.toShort, - permalink = permalink - ) - - val tweetMedia = if (isMixedMediaTweet(tweet)) { - TweetLenses.media(tweet).take(1) - } else { - TweetLenses.media(tweet) - } - - val modifiedMedia = tweetTextModification.reindexEntities(tweetMedia) - val modifiedUrls = - tweetTextModification.reindexEntities(TweetLenses.urls(tweet)) :+ tweetUrlEntity - val modifiedText = tweetTextModification.updated - - /* - * Visible Text Range computation differs by scenario - * == Any Tweet with Media == - * Tweet text has a media url *after* the visible text range - * original text: [visible text] https://t.co/mediaUrl - * original range: ^START END^ - * - * Append the permalink URL to the *visible text* so non-upgraded clients can see it - * modified text: [visible text https://t.co/permalink] https://t.co/mediaUrl - * modified range: ^START END^ - * visible range expanded, permalink is visible - * - * == Non-QT Tweet w/o Media == - * original text: [visible text] - * original range: None (default: whole text is visible) - * - * modified text: [visible text https://t.co/selfPermalink] - * modified range: None (default: whole text is visible) - * trailing self permalink will be visible - * - * == QT w/o Media == - * original text: [visible text] - * original range: None (default: whole text is visible) - * - * modified text: [visible text] https://t.co/qtPermalink - * modified range: ^START END^ - * trailing QT permalink is *hidden* because legacy clients that process the visible text range know how to display QTs - * - * == Non-QT Replies w/o media == - * original text: @user [visible text] - * original range: ^START END^ - * - * modified text: @user [visible text https://t.co/selfPermalink] - * modified range: ^START END^ - * visible range expanded, self permalink is visible - * - * == QT Replies w/o media == - * original text: @user [visible text] - * original range: ^START END^ - * - * modified text: @user [visible text] https://t.co/qtPermalink - * modified range: ^START END^ - * visible range remains the same, trailing QT permalink is hidden - * - */ - - val modifiedVisibleTextRange = - if (modifiedMedia.nonEmpty || - EditControlUtil.isEditTweet(tweet) || - tweet.noteTweet.isDefined) { - Some( - visibleRange.copy( - toIndex = visibleRange.toIndex + codePointLength(padLeft) + shortUrlLength - ) - ) - } else { - Some(visibleRange) - } - - val updatedTweet = - Lens.setAll( - tweet, - TweetLenses.text -> modifiedText, - TweetLenses.urls -> modifiedUrls.sortBy(_.fromIndex), - TweetLenses.media -> modifiedMedia.sortBy(_.fromIndex), - TweetLenses.visibleTextRange -> modifiedVisibleTextRange - ) - - /** - * compute extended tweet metadata when text length > 140 - * and apply the final lens to return a modified tweet - */ - val totalDisplayLength = displayLength(modifiedText) - if (totalDisplayLength > OriginalMaxDisplayLength) { - updatedTweet.selfPermalink match { - case Some(permalink) => - val extendedTweetMetadata = ExtendedTweetMetadataBuilder(updatedTweet, permalink) - updatedTweet.copy( - extendedTweetMetadata = Some(extendedTweetMetadata) - ) - case None => - /** - * This case shouldn't happen as TweetBuilder currently populates - * selfPermalink for extended tweets. In QT + Media, we will - * use AttachmentBuilder to store selfPermalink during writes, - * if text display length is going to exceed 140 after QT url append. - */ - log.error( - s"Failed to compute extended metadata for tweet: ${tweet.id} with " + - s"display length: ${totalDisplayLength}, as self-permalink is empty." - ) - statsReceiver.counter("self_permalink_not_found").incr() - tweet - } - } else { - updatedTweet - } - } - - def apply( - statsReceiver: StatsReceiver - ): TweetDataValueHydrator = { - ValueHydrator[TweetData, TweetQuery.Options] { (td, opts) => - // Prefer any required self permalink rendering over QT permalink rendering because a - // client that doesn't understand the attributes of the Tweet (i.e. Edit, Mixed - // Media) won't be able to render the Tweet properly at all, regardless of whether - // it's a QT. By preferring a visible self-permalink, the viewer is linked to an - // RWeb view of the Tweet which can fully display all of its features. - val permalink: Option[ShortenedUrl] = - legacySelfPermalink(td) - .orElse(legacyQtPermalink(td, opts)) - - permalink match { - case Some(permalink) => - val updatedTweet = updateTextAndURLsAndMedia(permalink, td.tweet, statsReceiver) - Stitch(ValueState.delta(td, td.copy(tweet = updatedTweet))) - case _ => - Stitch(ValueState.unmodified(td)) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetQueryOptionsExpander.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetQueryOptionsExpander.docx new file mode 100644 index 000000000..06aa26d71 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetQueryOptionsExpander.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetQueryOptionsExpander.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetQueryOptionsExpander.scala deleted file mode 100644 index 732b9c752..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetQueryOptionsExpander.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.repository.TweetQuery - -/** - * An instance of `TweetQueryOptionsExpander.Type` can be used to take a `TweetQuery.Options` - * instance provided by a user, and expand the set of options included to take into account - * dependencies between fields and options. - */ -object TweetQueryOptionsExpander { - import TweetQuery._ - - /** - * Used by AdditionalFieldsHydrator, this function type can filter out or inject fieldIds to - * request from Manhattan per tweet. - */ - type Type = Options => Options - - /** - * The identity TweetQueryOptionsExpander, which passes through fieldIds unchanged. - */ - val unit: TweetQueryOptionsExpander.Type = identity - - case class Selector(f: Include => Boolean) { - def apply(i: Include): Boolean = f(i) - - def ||(other: Selector) = Selector(i => this(i) || other(i)) - } - - private def selectTweetField(fieldId: FieldId): Selector = - Selector(_.tweetFields.contains(fieldId)) - - private val firstOrderDependencies: Seq[(Selector, Include)] = - Seq( - selectTweetField(Tweet.MediaField.id) -> - Include(tweetFields = Set(Tweet.UrlsField.id, Tweet.MediaKeysField.id)), - selectTweetField(Tweet.QuotedTweetField.id) -> - Include(tweetFields = Set(Tweet.UrlsField.id)), - selectTweetField(Tweet.MediaRefsField.id) -> - Include(tweetFields = Set(Tweet.UrlsField.id, Tweet.MediaKeysField.id)), - selectTweetField(Tweet.CardsField.id) -> - Include(tweetFields = Set(Tweet.UrlsField.id)), - selectTweetField(Tweet.Card2Field.id) -> - Include(tweetFields = Set(Tweet.UrlsField.id, Tweet.CardReferenceField.id)), - selectTweetField(Tweet.CoreDataField.id) -> - Include(tweetFields = Set(Tweet.DirectedAtUserMetadataField.id)), - selectTweetField(Tweet.SelfThreadInfoField.id) -> - Include(tweetFields = Set(Tweet.CoreDataField.id)), - (selectTweetField(Tweet.TakedownCountryCodesField.id) || - selectTweetField(Tweet.TakedownReasonsField.id)) -> - Include( - tweetFields = Set( - Tweet.TweetypieOnlyTakedownCountryCodesField.id, - Tweet.TweetypieOnlyTakedownReasonsField.id - ) - ), - selectTweetField(Tweet.EditPerspectiveField.id) -> - Include(tweetFields = Set(Tweet.PerspectiveField.id)), - Selector(_.quotedTweet) -> - Include(tweetFields = Set(Tweet.QuotedTweetField.id)), - // asking for any count implies getting the Tweet.counts field - Selector(_.countsFields.nonEmpty) -> - Include(tweetFields = Set(Tweet.CountsField.id)), - // asking for any media field implies getting the Tweet.media field - Selector(_.mediaFields.nonEmpty) -> - Include(tweetFields = Set(Tweet.MediaField.id)), - selectTweetField(Tweet.UnmentionDataField.id) -> - Include(tweetFields = Set(Tweet.MentionsField.id)), - ) - - private val allDependencies = - firstOrderDependencies.map { - case (sel, inc) => sel -> transitiveExpand(inc) - } - - private def transitiveExpand(inc: Include): Include = - firstOrderDependencies.foldLeft(inc) { - case (z, (selector, include)) => - if (!selector(z)) z - else z ++ include ++ transitiveExpand(include) - } - - /** - * Sequentially composes multiple TweetQueryOptionsExpander into a new TweetQueryOptionsExpander - */ - def sequentially(updaters: TweetQueryOptionsExpander.Type*): TweetQueryOptionsExpander.Type = - options => - updaters.foldLeft(options) { - case (options, updater) => updater(options) - } - - /** - * For requested fields that depend on other fields being present for correct hydration, - * returns an updated `TweetQuery.Options` with those dependee fields included. - */ - def expandDependencies: TweetQueryOptionsExpander.Type = - options => - options.copy( - include = allDependencies.foldLeft(options.include) { - case (z, (selector, include)) => - if (!selector(options.include)) z - else z ++ include - } - ) - - /** - * If the gate is true, add 'fields' to the list of tweetFields to load. - */ - def gatedTweetFieldUpdater( - gate: Gate[Unit], - fields: Seq[FieldId] - ): TweetQueryOptionsExpander.Type = - options => - if (gate()) { - options.copy( - include = options.include.also(tweetFields = fields) - ) - } else { - options - } - - /** - * Uses a `ThreadLocal` to remember the last expansion performed, and to reuse the - * previous result if the input value is the same. This is useful to avoid repeatedly - * computing the expansion of the same input when multiple tweets are queried together - * with the same options. - */ - def threadLocalMemoize(expander: Type): Type = { - val memo: ThreadLocal[Option[(Options, Options)]] = - new ThreadLocal[Option[(Options, Options)]] { - override def initialValue(): None.type = None - } - - options => - memo.get() match { - case Some((`options`, res)) => res - case _ => - val res = expander(options) - memo.set(Some((options, res))) - res - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetVisibilityHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetVisibilityHydrator.docx new file mode 100644 index 000000000..b1b6c1da3 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetVisibilityHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetVisibilityHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetVisibilityHydrator.scala deleted file mode 100644 index 9d05fbf8e..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/TweetVisibilityHydrator.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.util.CommunityUtil - -object TweetVisibilityHydrator { - type Type = ValueHydrator[Option[FilteredState.Suppress], Ctx] - - case class Ctx(tweet: Tweet, underlyingTweetCtx: TweetCtx) extends TweetCtx.Proxy - - def apply( - repo: TweetVisibilityRepository.Type, - failClosedInVF: Gate[Unit], - stats: StatsReceiver - ): Type = { - val outcomeScope = stats.scope("outcome") - val unavailable = outcomeScope.counter("unavailable") - val suppress = outcomeScope.counter("suppress") - val allow = outcomeScope.counter("allow") - val failClosed = outcomeScope.counter("fail_closed") - val communityFailClosed = outcomeScope.counter("community_fail_closed") - val failOpen = outcomeScope.counter("fail_open") - - ValueHydrator[Option[FilteredState.Suppress], Ctx] { (curr, ctx) => - val request = TweetVisibilityRepository.Request( - tweet = ctx.tweet, - viewerId = ctx.opts.forUserId, - safetyLevel = ctx.opts.safetyLevel, - isInnerQuotedTweet = ctx.opts.isInnerQuotedTweet, - isRetweet = ctx.isRetweet, - hydrateConversationControl = ctx.tweetFieldRequested(Tweet.ConversationControlField), - isSourceTweet = ctx.opts.isSourceTweet - ) - - repo(request).liftToTry.flatMap { - // If FilteredState.Unavailable is returned from repo then throw it - case Return(Some(fs: FilteredState.Unavailable)) => - unavailable.incr() - Stitch.exception(fs) - // If FilteredState.Suppress is returned from repo then return it - case Return(Some(fs: FilteredState.Suppress)) => - suppress.incr() - Stitch.value(ValueState.modified(Some(fs))) - // If None is returned from repo then return unmodified - case Return(None) => - allow.incr() - ValueState.StitchUnmodifiedNone - // Propagate thrown exceptions if fail closed - case Throw(e) if failClosedInVF() => - failClosed.incr() - Stitch.exception(e) - // Community tweets are special cased to fail closed to avoid - // leaking tweets expected to be private to a community. - case Throw(e) if CommunityUtil.hasCommunity(request.tweet.communities) => - communityFailClosed.incr() - Stitch.exception(e) - case Throw(_) => - failOpen.incr() - Stitch.value(ValueState.unmodified(curr)) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnmentionDataHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnmentionDataHydrator.docx new file mode 100644 index 000000000..abb54bcbf Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnmentionDataHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnmentionDataHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnmentionDataHydrator.scala deleted file mode 100644 index dd6b1ee91..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnmentionDataHydrator.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.core.ValueState -import com.twitter.tweetypie.thriftscala.MentionEntity -import com.twitter.tweetypie.unmentions.thriftscala.UnmentionData - -object UnmentionDataHydrator { - type Type = ValueHydrator[Option[UnmentionData], Ctx] - - case class Ctx( - conversationId: Option[TweetId], - mentions: Seq[MentionEntity], - underlyingTweetCtx: TweetCtx) - extends TweetCtx.Proxy - - def apply(): Type = { - ValueHydrator.map[Option[UnmentionData], Ctx] { (_, ctx) => - val mentionedUserIds: Seq[UserId] = ctx.mentions.flatMap(_.userId) - - ValueState.modified( - Some(UnmentionData(ctx.conversationId, Option(mentionedUserIds).filter(_.nonEmpty))) - ) - } - }.onlyIf { (_, ctx) => - ctx.tweetFieldRequested(Tweet.UnmentionDataField) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnrequestedFieldScrubber.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnrequestedFieldScrubber.docx new file mode 100644 index 000000000..e20214ca5 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnrequestedFieldScrubber.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnrequestedFieldScrubber.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnrequestedFieldScrubber.scala deleted file mode 100644 index 1f69b7ecd..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UnrequestedFieldScrubber.scala +++ /dev/null @@ -1,211 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.tweetypie.additionalfields.AdditionalFields -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.thriftscala._ - -/** - * A hydrator that scrubs tweet fields that weren't requested. Those fields might be - * present because they were previously requested and were cached with the tweet. - */ -trait UnrequestedFieldScrubber { - def scrub(tweetResult: TweetResult): TweetResult - def scrub(tweetData: TweetData): TweetData - def scrub(tweet: Tweet): Tweet -} - -object UnrequestedFieldScrubber { - def apply(options: TweetQuery.Options): UnrequestedFieldScrubber = - if (!options.scrubUnrequestedFields) NullScrubber - else new ScrubberImpl(options.include) - - private object NullScrubber extends UnrequestedFieldScrubber { - def scrub(tweetResult: TweetResult): TweetResult = tweetResult - def scrub(tweetData: TweetData): TweetData = tweetData - def scrub(tweet: Tweet): Tweet = tweet - } - - class ScrubberImpl(i: TweetQuery.Include) extends UnrequestedFieldScrubber { - def scrub(tweetResult: TweetResult): TweetResult = - tweetResult.map(scrub(_)) - - def scrub(tweetData: TweetData): TweetData = - tweetData.copy( - tweet = scrub(tweetData.tweet), - sourceTweetResult = tweetData.sourceTweetResult.map(scrub(_)), - quotedTweetResult = - if (!i.quotedTweet) None - else tweetData.quotedTweetResult.map(qtr => qtr.map(scrub)) - ) - - def scrub(tweet: Tweet): Tweet = { - val tweet2 = scrubKnownFields(tweet) - - val unhandledFields = i.tweetFields -- AdditionalFields.CompiledFieldIds - - if (unhandledFields.isEmpty) { - tweet2 - } else { - tweet2.unsetFields(unhandledFields) - } - } - - def scrubKnownFields(tweet: Tweet): Tweet = { - @inline - def filter[A](fieldId: FieldId, value: Option[A]): Option[A] = - if (i.tweetFields.contains(fieldId)) value else None - - tweet.copy( - coreData = filter(Tweet.CoreDataField.id, tweet.coreData), - urls = filter(Tweet.UrlsField.id, tweet.urls), - mentions = filter(Tweet.MentionsField.id, tweet.mentions), - hashtags = filter(Tweet.HashtagsField.id, tweet.hashtags), - cashtags = filter(Tweet.CashtagsField.id, tweet.cashtags), - media = filter(Tweet.MediaField.id, tweet.media), - place = filter(Tweet.PlaceField.id, tweet.place), - quotedTweet = filter(Tweet.QuotedTweetField.id, tweet.quotedTweet), - takedownCountryCodes = - filter(Tweet.TakedownCountryCodesField.id, tweet.takedownCountryCodes), - counts = filter(Tweet.CountsField.id, tweet.counts.map(scrub)), - deviceSource = filter(Tweet.DeviceSourceField.id, tweet.deviceSource), - perspective = filter(Tweet.PerspectiveField.id, tweet.perspective), - cards = filter(Tweet.CardsField.id, tweet.cards), - card2 = filter(Tweet.Card2Field.id, tweet.card2), - language = filter(Tweet.LanguageField.id, tweet.language), - spamLabels = None, // unused - contributor = filter(Tweet.ContributorField.id, tweet.contributor), - profileGeoEnrichment = - filter(Tweet.ProfileGeoEnrichmentField.id, tweet.profileGeoEnrichment), - conversationMuted = filter(Tweet.ConversationMutedField.id, tweet.conversationMuted), - takedownReasons = filter(Tweet.TakedownReasonsField.id, tweet.takedownReasons), - selfThreadInfo = filter(Tweet.SelfThreadInfoField.id, tweet.selfThreadInfo), - // additional fields - mediaTags = filter(Tweet.MediaTagsField.id, tweet.mediaTags), - schedulingInfo = filter(Tweet.SchedulingInfoField.id, tweet.schedulingInfo), - bindingValues = filter(Tweet.BindingValuesField.id, tweet.bindingValues), - replyAddresses = None, // unused - obsoleteTwitterSuggestInfo = None, // unused - escherbirdEntityAnnotations = - filter(Tweet.EscherbirdEntityAnnotationsField.id, tweet.escherbirdEntityAnnotations), - spamLabel = filter(Tweet.SpamLabelField.id, tweet.spamLabel), - abusiveLabel = filter(Tweet.AbusiveLabelField.id, tweet.abusiveLabel), - lowQualityLabel = filter(Tweet.LowQualityLabelField.id, tweet.lowQualityLabel), - nsfwHighPrecisionLabel = - filter(Tweet.NsfwHighPrecisionLabelField.id, tweet.nsfwHighPrecisionLabel), - nsfwHighRecallLabel = filter(Tweet.NsfwHighRecallLabelField.id, tweet.nsfwHighRecallLabel), - abusiveHighRecallLabel = - filter(Tweet.AbusiveHighRecallLabelField.id, tweet.abusiveHighRecallLabel), - lowQualityHighRecallLabel = - filter(Tweet.LowQualityHighRecallLabelField.id, tweet.lowQualityHighRecallLabel), - personaNonGrataLabel = - filter(Tweet.PersonaNonGrataLabelField.id, tweet.personaNonGrataLabel), - recommendationsLowQualityLabel = filter( - Tweet.RecommendationsLowQualityLabelField.id, - tweet.recommendationsLowQualityLabel - ), - experimentationLabel = - filter(Tweet.ExperimentationLabelField.id, tweet.experimentationLabel), - tweetLocationInfo = filter(Tweet.TweetLocationInfoField.id, tweet.tweetLocationInfo), - cardReference = filter(Tweet.CardReferenceField.id, tweet.cardReference), - supplementalLanguage = - filter(Tweet.SupplementalLanguageField.id, tweet.supplementalLanguage), - selfPermalink = filter(Tweet.SelfPermalinkField.id, tweet.selfPermalink), - extendedTweetMetadata = - filter(Tweet.ExtendedTweetMetadataField.id, tweet.extendedTweetMetadata), - communities = filter(Tweet.CommunitiesField.id, tweet.communities), - visibleTextRange = filter(Tweet.VisibleTextRangeField.id, tweet.visibleTextRange), - spamHighRecallLabel = filter(Tweet.SpamHighRecallLabelField.id, tweet.spamHighRecallLabel), - duplicateContentLabel = - filter(Tweet.DuplicateContentLabelField.id, tweet.duplicateContentLabel), - liveLowQualityLabel = filter(Tweet.LiveLowQualityLabelField.id, tweet.liveLowQualityLabel), - nsfaHighRecallLabel = filter(Tweet.NsfaHighRecallLabelField.id, tweet.nsfaHighRecallLabel), - pdnaLabel = filter(Tweet.PdnaLabelField.id, tweet.pdnaLabel), - searchBlacklistLabel = - filter(Tweet.SearchBlacklistLabelField.id, tweet.searchBlacklistLabel), - lowQualityMentionLabel = - filter(Tweet.LowQualityMentionLabelField.id, tweet.lowQualityMentionLabel), - bystanderAbusiveLabel = - filter(Tweet.BystanderAbusiveLabelField.id, tweet.bystanderAbusiveLabel), - automationHighRecallLabel = - filter(Tweet.AutomationHighRecallLabelField.id, tweet.automationHighRecallLabel), - goreAndViolenceLabel = - filter(Tweet.GoreAndViolenceLabelField.id, tweet.goreAndViolenceLabel), - untrustedUrlLabel = filter(Tweet.UntrustedUrlLabelField.id, tweet.untrustedUrlLabel), - goreAndViolenceHighRecallLabel = filter( - Tweet.GoreAndViolenceHighRecallLabelField.id, - tweet.goreAndViolenceHighRecallLabel - ), - nsfwVideoLabel = filter(Tweet.NsfwVideoLabelField.id, tweet.nsfwVideoLabel), - nsfwNearPerfectLabel = - filter(Tweet.NsfwNearPerfectLabelField.id, tweet.nsfwNearPerfectLabel), - automationLabel = filter(Tweet.AutomationLabelField.id, tweet.automationLabel), - nsfwCardImageLabel = filter(Tweet.NsfwCardImageLabelField.id, tweet.nsfwCardImageLabel), - duplicateMentionLabel = - filter(Tweet.DuplicateMentionLabelField.id, tweet.duplicateMentionLabel), - bounceLabel = filter(Tweet.BounceLabelField.id, tweet.bounceLabel), - selfThreadMetadata = filter(Tweet.SelfThreadMetadataField.id, tweet.selfThreadMetadata), - composerSource = filter(Tweet.ComposerSourceField.id, tweet.composerSource), - editControl = filter(Tweet.EditControlField.id, tweet.editControl), - developerBuiltCardId = filter( - Tweet.DeveloperBuiltCardIdField.id, - tweet.developerBuiltCardId - ), - creativeEntityEnrichmentsForTweet = filter( - Tweet.CreativeEntityEnrichmentsForTweetField.id, - tweet.creativeEntityEnrichmentsForTweet - ), - previousCounts = filter(Tweet.PreviousCountsField.id, tweet.previousCounts), - mediaRefs = filter(Tweet.MediaRefsField.id, tweet.mediaRefs), - isCreativesContainerBackendTweet = filter( - Tweet.IsCreativesContainerBackendTweetField.id, - tweet.isCreativesContainerBackendTweet), - editPerspective = filter(Tweet.EditPerspectiveField.id, tweet.editPerspective), - noteTweet = filter(Tweet.NoteTweetField.id, tweet.noteTweet), - - // tweetypie-internal metadata - directedAtUserMetadata = - filter(Tweet.DirectedAtUserMetadataField.id, tweet.directedAtUserMetadata), - tweetypieOnlyTakedownReasons = - filter(Tweet.TweetypieOnlyTakedownReasonsField.id, tweet.tweetypieOnlyTakedownReasons), - mediaKeys = filter(Tweet.MediaKeysField.id, tweet.mediaKeys), - tweetypieOnlyTakedownCountryCodes = filter( - Tweet.TweetypieOnlyTakedownCountryCodesField.id, - tweet.tweetypieOnlyTakedownCountryCodes - ), - underlyingCreativesContainerId = filter( - Tweet.UnderlyingCreativesContainerIdField.id, - tweet.underlyingCreativesContainerId), - unmentionData = filter(Tweet.UnmentionDataField.id, tweet.unmentionData), - blockingUnmentions = filter(Tweet.BlockingUnmentionsField.id, tweet.blockingUnmentions), - settingsUnmentions = filter(Tweet.SettingsUnmentionsField.id, tweet.settingsUnmentions) - ) - } - - def scrub(counts: StatusCounts): StatusCounts = { - @inline - def filter[A](fieldId: FieldId, value: Option[A]): Option[A] = - if (i.countsFields.contains(fieldId)) value else None - - StatusCounts( - replyCount = filter(StatusCounts.ReplyCountField.id, counts.replyCount), - favoriteCount = filter(StatusCounts.FavoriteCountField.id, counts.favoriteCount), - retweetCount = filter(StatusCounts.RetweetCountField.id, counts.retweetCount), - quoteCount = filter(StatusCounts.QuoteCountField.id, counts.quoteCount), - bookmarkCount = filter(StatusCounts.BookmarkCountField.id, counts.bookmarkCount) - ) - } - - def scrub(media: MediaEntity): MediaEntity = { - @inline - def filter[A](fieldId: FieldId, value: Option[A]): Option[A] = - if (i.mediaFields.contains(fieldId)) value else None - - media.copy( - additionalMetadata = - filter(MediaEntity.AdditionalMetadataField.id, media.additionalMetadata) - ) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UrlEntityHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UrlEntityHydrator.docx new file mode 100644 index 000000000..59f4b0a1b Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UrlEntityHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UrlEntityHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UrlEntityHydrator.scala deleted file mode 100644 index 9ffdf0139..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/UrlEntityHydrator.scala +++ /dev/null @@ -1,122 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tco_util.DisplayUrl -import com.twitter.tco_util.InvalidUrlException -import com.twitter.tco_util.TcoSlug -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository._ -import com.twitter.tweetypie.thriftscala._ -import scala.util.control.NonFatal - -object UrlEntitiesHydrator { - type Type = ValueHydrator[Seq[UrlEntity], TweetCtx] - - def once(h: ValueHydrator[UrlEntity, TweetCtx]): Type = - TweetHydration.completeOnlyOnce( - queryFilter = queryFilter, - hydrationType = HydrationType.Urls, - hydrator = h.liftSeq - ) - - def queryFilter(opts: TweetQuery.Options): Boolean = - opts.include.tweetFields.contains(Tweet.UrlsField.id) -} - -/** - * Hydrates UrlEntities. If there is a failure to hydrate an entity, the entity is left - * unhydrated, so that we can try again later. The PartialEntityCleaner will remove - * the partial entity before returning to clients. - */ -object UrlEntityHydrator { - - /** - * a function type that takes a shorten-url and an expanded-url, and generates a - * "display url" (which isn't really a url). this may fail if the expanded-url - * can't be parsed as a valid url, in which case None is returned. - */ - type Truncator = (String, String) => Option[String] - - val hydratedField: FieldByPath = fieldByPath(Tweet.UrlsField) - val log: Logger = Logger(getClass) - - def apply(repo: UrlRepository.Type, stats: StatsReceiver): ValueHydrator[UrlEntity, TweetCtx] = { - val toDisplayUrl = truncator(stats) - - ValueHydrator[UrlEntity, TweetCtx] { (curr, _) => - val slug = getTcoSlug(curr) - - val result: Stitch[Option[Try[ExpandedUrl]]] = Stitch.collect(slug.map(repo(_).liftToTry)) - - result.map { - case Some(Return(expandedUrl)) => - ValueState.modified(update(curr, expandedUrl, toDisplayUrl)) - - case None => - ValueState.unmodified(curr) - - case Some(Throw(NotFound)) => - // If the UrlEntity contains an invalid t.co slug that can't be resolved, - // leave the entity unhydrated, to be removed later by the PartialEntityCleaner. - // We don't consider this a partial because the input is invalid and is not - // expected to succeed. - ValueState.unmodified(curr) - - case Some(Throw(_)) => - // On failure, use the t.co link as the expanded url so that it is still clickable, - // but also still flag the failure - ValueState.partial( - update(curr, ExpandedUrl(curr.url), toDisplayUrl), - hydratedField - ) - } - }.onlyIf((curr, ctx) => !ctx.isRetweet && isUnhydrated(curr)) - } - - /** - * a UrlEntity needs hydration if the expanded url is either unset or set to the - * shortened url . - */ - def isUnhydrated(entity: UrlEntity): Boolean = - entity.expanded.isEmpty || hydrationFailed(entity) - - /** - * Did the hydration of this URL entity fail? - */ - def hydrationFailed(entity: UrlEntity): Boolean = - entity.expanded.contains(entity.url) - - def update(entity: UrlEntity, expandedUrl: ExpandedUrl, toDisplayUrl: Truncator): UrlEntity = - entity.copy( - expanded = Some(expandedUrl.text), - display = toDisplayUrl(entity.url, expandedUrl.text) - ) - - def getTcoSlug(entity: UrlEntity): Option[UrlSlug] = - TcoSlug.unapply(entity.url).map(UrlSlug(_)) - - def truncator(stats: StatsReceiver): Truncator = { - val truncationStats = stats.scope("truncations") - val truncationsCounter = truncationStats.counter("count") - val truncationExceptionsCounter = truncationStats.counter("exceptions") - - (shortUrl, expandedUrl) => - try { - truncationsCounter.incr() - Some(DisplayUrl(shortUrl, Some(expandedUrl), true)) - } catch { - case NonFatal(ex) => - truncationExceptionsCounter.incr() - truncationStats.counter(ex.getClass.getName).incr() - ex match { - case InvalidUrlException(_) => - log.warn(s"failed to truncate: `$shortUrl` / `$expandedUrl`") - case _ => - log.warn(s"failed to truncate: `$shortUrl` / `$expandedUrl`", ex) - } - None - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ValueHydrator.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ValueHydrator.docx new file mode 100644 index 000000000..f8ed0d064 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ValueHydrator.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ValueHydrator.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ValueHydrator.scala deleted file mode 100644 index 0504d7429..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/ValueHydrator.scala +++ /dev/null @@ -1,200 +0,0 @@ -package com.twitter.tweetypie -package hydrator - -import com.twitter.servo.util.ExceptionCounter -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.EditState -import com.twitter.tweetypie.core.ValueState -import com.twitter.util.Try - -/** - * A ValueHydrator hydrates a value of type `A`, with a hydration context of type `C`, - * and produces a value of type ValueState[A] (ValueState encapsulates the value and - * its associated HydrationState). - * - * Because ValueHydrators take a value and produce a new value, they can easily be run - * in sequence, but not in parallel. To run hydrators in parallel, see [[EditHydrator]]. - * - * A series of ValueHydrators of the same type may be run in sequence via - * `ValueHydrator.inSequence`. - * - */ -class ValueHydrator[A, C] private (val run: (A, C) => Stitch[ValueState[A]]) { - - /** - * Apply this hydrator to a value, producing a ValueState. - */ - def apply(a: A, ctx: C): Stitch[ValueState[A]] = run(a, ctx) - - /** - * Apply with an empty context: only used in tests. - */ - def apply(a: A)(implicit ev: Unit <:< C): Stitch[ValueState[A]] = - apply(a, ev(())) - - /** - * Convert this ValueHydrator to the equivalent EditHydrator. - */ - def toEditHydrator: EditHydrator[A, C] = - EditHydrator[A, C] { (a, ctx) => this.run(a, ctx).map(value => EditState(_ => value)) } - - /** - * Chains two ValueHydrators in sequence. - */ - def andThen(next: ValueHydrator[A, C]): ValueHydrator[A, C] = - ValueHydrator[A, C] { (x0, ctx) => - for { - r1 <- run(x0, ctx) - r2 <- next.run(r1.value, ctx) - } yield { - ValueState(r2.value, r1.state ++ r2.state) - } - } - - /** - * Executes this ValueHydrator conditionally based on a Gate. - */ - def ifEnabled(gate: Gate[Unit]): ValueHydrator[A, C] = - onlyIf((_, _) => gate()) - - /** - * Executes this ValueHydrator conditionally based on a boolean function. - */ - def onlyIf(cond: (A, C) => Boolean): ValueHydrator[A, C] = - ValueHydrator { (a, c) => - if (cond(a, c)) { - run(a, c) - } else { - Stitch.value(ValueState.unit(a)) - } - } - - /** - * Converts a ValueHydrator of input type `A` to input type `Option[A]`. - */ - def liftOption: ValueHydrator[Option[A], C] = - liftOption(None) - - /** - * Converts a ValueHydrator of input type `A` to input type `Option[A]` with a - * default input value. - */ - def liftOption(default: A): ValueHydrator[Option[A], C] = - liftOption(Some(default)) - - private def liftOption(default: Option[A]): ValueHydrator[Option[A], C] = { - val none = Stitch.value(ValueState.unit(None)) - - ValueHydrator[Option[A], C] { (a, ctx) => - a.orElse(default) match { - case Some(a) => this.run(a, ctx).map(s => s.map(Some.apply)) - case None => none - } - } - } - - /** - * Converts a ValueHydrator of input type `A` to input type `Seq[A]`. - */ - def liftSeq: ValueHydrator[Seq[A], C] = - ValueHydrator[Seq[A], C] { (as, ctx) => - Stitch.traverse(as)(a => run(a, ctx)).map(rs => ValueState.sequence[A](rs)) - } - - /** - * Produces a new ValueHydrator that collects stats on the hydration. - */ - def observe( - stats: StatsReceiver, - mkExceptionCounter: (StatsReceiver, String) => ExceptionCounter = (stats, scope) => - new ExceptionCounter(stats, scope) - ): ValueHydrator[A, C] = { - val callCounter = stats.counter("calls") - val noopCounter = stats.counter("noop") - val modifiedCounter = stats.counter("modified") - val partialCounter = stats.counter("partial") - val completedCounter = stats.counter("completed") - - val exceptionCounter = mkExceptionCounter(stats, "failures") - - ValueHydrator[A, C] { (a, ctx) => - this.run(a, ctx).respond { - case Return(ValueState(_, state)) => - callCounter.incr() - - if (state.isEmpty) { - noopCounter.incr() - } else { - if (state.modified) modifiedCounter.incr() - if (state.failedFields.nonEmpty) partialCounter.incr() - if (state.completedHydrations.nonEmpty) completedCounter.incr() - } - case Throw(ex) => - callCounter.incr() - exceptionCounter(ex) - } - } - } - - /** - * Produces a new ValueHydrator that uses a lens to extract the value to hydrate, - * using this hydrator, and then to put the updated value back in the enclosing struct. - */ - def lensed[B](lens: Lens[B, A]): ValueHydrator[B, C] = - ValueHydrator[B, C] { (b, ctx) => - this.run(lens.get(b), ctx).map { - case ValueState(value, state) => - ValueState(lens.set(b, value), state) - } - } -} - -object ValueHydrator { - - /** - * Create a ValueHydrator from a function that returns Stitch[ValueState[A]] - */ - def apply[A, C](f: (A, C) => Stitch[ValueState[A]]): ValueHydrator[A, C] = - new ValueHydrator[A, C](f) - - /** - * Produces a ValueState instance with the given value and an empty HydrationState - */ - def unit[A, C]: ValueHydrator[A, C] = - ValueHydrator { (a, _) => Stitch.value(ValueState.unit(a)) } - - /** - * Runs several ValueHydrators in sequence. - */ - def inSequence[A, C](bs: ValueHydrator[A, C]*): ValueHydrator[A, C] = - bs match { - case Seq(b) => b - case Seq(b1, b2) => b1.andThen(b2) - case _ => bs.reduceLeft(_.andThen(_)) - } - - /** - * Creates a `ValueHydrator` from a Mutation. If the mutation returns None (indicating - * no change) the hydrator will return an ValueState.unmodified with the input value; - * otherwise, it will return an ValueState.modified with the mutated value. - * If the mutation throws an exception, it will be caught and lifted to Stitch.exception. - */ - def fromMutation[A, C](mutation: Mutation[A]): ValueHydrator[A, C] = - ValueHydrator[A, C] { (input, _) => - Stitch.const( - Try { - mutation(input) match { - case None => ValueState.unmodified(input) - case Some(output) => ValueState.modified(output) - } - } - ) - } - - /** - * Creates a Hydrator from a non-`Stitch` producing function. If the function throws - * an error it will be caught and converted to a Throw. - */ - def map[A, C](f: (A, C) => ValueState[A]): ValueHydrator[A, C] = - ValueHydrator[A, C] { (a, ctx) => Stitch.const(Try(f(a, ctx))) } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/package.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/package.docx new file mode 100644 index 000000000..e1048779b Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/package.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/package.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/package.scala deleted file mode 100644 index 0542cf4f5..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/hydrator/package.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.tweetypie - -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository.TweetQuery -import com.twitter.tweetypie.thriftscala.FieldByPath -import org.apache.thrift.protocol.TField -import com.twitter.context.TwitterContext - -package object hydrator { - type TweetDataValueHydrator = ValueHydrator[TweetData, TweetQuery.Options] - type TweetDataEditHydrator = EditHydrator[TweetData, TweetQuery.Options] - - def fieldByPath(fields: TField*): FieldByPath = FieldByPath(fields.map(_.id)) - - val TwitterContext: TwitterContext = - com.twitter.context.TwitterContext(com.twitter.tweetypie.TwitterContextPermit) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/BUILD b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/BUILD deleted file mode 100644 index dc5edd30e..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "image-fetcher-service/thrift/src/main/thrift:thrift-scala", - "mediaservices/commons/src/main/thrift:thrift-scala", - "mediaservices/mediainfo-server/thrift/src/main/thrift:thrift-scala", - "tweetypie/servo/util", - "tweetypie/common/src/thrift/com/twitter/tweetypie:media-entity-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:tweet-scala", - "tweetypie/server/src/main/scala/com/twitter/tweetypie", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/backends", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/core", - "tweetypie/common/src/scala/com/twitter/tweetypie/media", - "user-image-service/thrift/src/main/thrift:thrift-scala", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - ], -) diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/BUILD.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/BUILD.docx new file mode 100644 index 000000000..f5b187900 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/BUILD.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaClient.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaClient.docx new file mode 100644 index 000000000..9eb71a29a Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaClient.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaClient.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaClient.scala deleted file mode 100644 index c33ed5e66..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaClient.scala +++ /dev/null @@ -1,288 +0,0 @@ -package com.twitter.tweetypie -package media - -import com.twitter.mediainfo.server.{thriftscala => mis} -import com.twitter.mediaservices.commons.mediainformation.thriftscala.UserDefinedProductMetadata -import com.twitter.mediaservices.commons.photurkey.thriftscala.PrivacyType -import com.twitter.mediaservices.commons.servercommon.thriftscala.{ServerError => CommonServerError} -import com.twitter.mediaservices.commons.thriftscala.ProductKey -import com.twitter.mediaservices.commons.thriftscala.MediaKey -import com.twitter.servo.util.FutureArrow -import com.twitter.thumbingbird.{thriftscala => ifs} -import com.twitter.tweetypie.backends.MediaInfoService -import com.twitter.tweetypie.backends.UserImageService -import com.twitter.tweetypie.core.UpstreamFailure -import com.twitter.user_image_service.{thriftscala => uis} -import com.twitter.user_image_service.thriftscala.MediaUpdateAction -import com.twitter.user_image_service.thriftscala.MediaUpdateAction.Delete -import com.twitter.user_image_service.thriftscala.MediaUpdateAction.Undelete -import java.nio.ByteBuffer -import scala.util.control.NoStackTrace - -/** - * The MediaClient trait encapsulates the various operations we make to the different media services - * backends. - */ -trait MediaClient { - import MediaClient._ - - /** - * On tweet creation, if the tweet contains media upload ids, we call this operation to process - * that media and get back metadata about the media. - */ - def processMedia: ProcessMedia - - /** - * On the read path, when hydrating a MediaEntity, we call this operation to get metadata - * about existing media. - */ - def getMediaMetadata: GetMediaMetadata - - def deleteMedia: DeleteMedia - - def undeleteMedia: UndeleteMedia -} - -/** - * Request type for the MediaClient.updateMedia operation. - */ -private case class UpdateMediaRequest( - mediaKey: MediaKey, - action: MediaUpdateAction, - tweetId: TweetId) - -case class DeleteMediaRequest(mediaKey: MediaKey, tweetId: TweetId) { - private[media] def toUpdateMediaRequest = UpdateMediaRequest(mediaKey, Delete, tweetId) -} - -case class UndeleteMediaRequest(mediaKey: MediaKey, tweetId: TweetId) { - private[media] def toUpdateMediaRequest = UpdateMediaRequest(mediaKey, Undelete, tweetId) -} - -/** - * Request type for the MediaClient.processMedia operation. - */ -case class ProcessMediaRequest( - mediaIds: Seq[MediaId], - userId: UserId, - tweetId: TweetId, - isProtected: Boolean, - productMetadata: Option[Map[MediaId, UserDefinedProductMetadata]]) { - private[media] def toProcessTweetMediaRequest = - uis.ProcessTweetMediaRequest(mediaIds, userId, tweetId) - - private[media] def toUpdateProductMetadataRequests(mediaKeys: Seq[MediaKey]) = - productMetadata match { - case None => Seq() - case Some(map) => - mediaKeys.flatMap { mediaKey => - map.get(mediaKey.mediaId).map { metadata => - uis.UpdateProductMetadataRequest(ProductKey(tweetId.toString, mediaKey), metadata) - } - } - } -} - -/** - * Request type for the MediaClient.getMediaMetdata operation. - */ -case class MediaMetadataRequest( - mediaKey: MediaKey, - tweetId: TweetId, - isProtected: Boolean, - extensionsArgs: Option[ByteBuffer]) { - private[media] def privacyType = MediaClient.toPrivacyType(isProtected) - - /** - * For debugging purposes, make a copy of the byte buffer at object - * creation time, so that we can inspect the original buffer if there - * is an error. - * - * Once we have found the problem, this method should be removed. - */ - val savedExtensionArgs: Option[ByteBuffer] = - extensionsArgs.map { buf => - val b = buf.asReadOnlyBuffer() - val ary = new Array[Byte](b.remaining) - b.get(ary) - ByteBuffer.wrap(ary) - } - - private[media] def toGetTweetMediaInfoRequest = - mis.GetTweetMediaInfoRequest( - mediaKey = mediaKey, - tweetId = Some(tweetId), - privacyType = privacyType, - stratoExtensionsArgs = extensionsArgs - ) -} - -object MediaClient { - import MediaExceptions._ - - /** - * Operation type for processing uploaded media during tweet creation. - */ - type ProcessMedia = FutureArrow[ProcessMediaRequest, Seq[MediaKey]] - - /** - * Operation type for deleting and undeleting tweets. - */ - private[media] type UpdateMedia = FutureArrow[UpdateMediaRequest, Unit] - - type UndeleteMedia = FutureArrow[UndeleteMediaRequest, Unit] - - type DeleteMedia = FutureArrow[DeleteMediaRequest, Unit] - - /** - * Operation type for getting media metadata for existing media during tweet reads. - */ - type GetMediaMetadata = FutureArrow[MediaMetadataRequest, MediaMetadata] - - /** - * Builds a UpdateMedia FutureArrow using UserImageService endpoints. - */ - private[media] object UpdateMedia { - def apply(updateTweetMedia: UserImageService.UpdateTweetMedia): UpdateMedia = - FutureArrow[UpdateMediaRequest, Unit] { r => - updateTweetMedia(uis.UpdateTweetMediaRequest(r.mediaKey, r.action, Some(r.tweetId))).unit - }.translateExceptions(handleMediaExceptions) - } - - /** - * Builds a ProcessMedia FutureArrow using UserImageService endpoints. - */ - object ProcessMedia { - - def apply( - updateProductMetadata: UserImageService.UpdateProductMetadata, - processTweetMedia: UserImageService.ProcessTweetMedia - ): ProcessMedia = { - - val updateProductMetadataSeq = updateProductMetadata.liftSeq - - FutureArrow[ProcessMediaRequest, Seq[MediaKey]] { req => - for { - mediaKeys <- processTweetMedia(req.toProcessTweetMediaRequest).map(_.mediaKeys) - _ <- updateProductMetadataSeq(req.toUpdateProductMetadataRequests(mediaKeys)) - } yield { - sortKeysByIds(req.mediaIds, mediaKeys) - } - }.translateExceptions(handleMediaExceptions) - } - - /** - * Sort the mediaKeys Seq based on the media id ordering specified by the - * caller's request mediaIds Seq. - */ - private def sortKeysByIds(mediaIds: Seq[MediaId], mediaKeys: Seq[MediaKey]): Seq[MediaKey] = { - val idToKeyMap = mediaKeys.map(key => (key.mediaId, key)).toMap - mediaIds.flatMap(idToKeyMap.get) - } - } - - /** - * Builds a GetMediaMetadata FutureArrow using MediaInfoService endpoints. - */ - object GetMediaMetadata { - - private[this] val log = Logger(getClass) - - def apply(getTweetMediaInfo: MediaInfoService.GetTweetMediaInfo): GetMediaMetadata = - FutureArrow[MediaMetadataRequest, MediaMetadata] { req => - getTweetMediaInfo(req.toGetTweetMediaInfoRequest).map { res => - MediaMetadata( - res.mediaKey, - res.assetUrlHttps, - res.sizes.toSet, - res.mediaInfo, - res.additionalMetadata.flatMap(_.productMetadata), - res.stratoExtensionsReply, - res.additionalMetadata - ) - } - }.translateExceptions(handleMediaExceptions) - } - - private[media] def toPrivacyType(isProtected: Boolean): PrivacyType = - if (isProtected) PrivacyType.Protected else PrivacyType.Public - - /** - * Constructs an implementation of the MediaClient interface using backend instances. - */ - def fromBackends( - userImageService: UserImageService, - mediaInfoService: MediaInfoService - ): MediaClient = - new MediaClient { - - val getMediaMetadata = - GetMediaMetadata( - getTweetMediaInfo = mediaInfoService.getTweetMediaInfo - ) - - val processMedia = - ProcessMedia( - userImageService.updateProductMetadata, - userImageService.processTweetMedia - ) - - private val updateMedia = - UpdateMedia( - userImageService.updateTweetMedia - ) - - val deleteMedia: FutureArrow[DeleteMediaRequest, Unit] = - FutureArrow[DeleteMediaRequest, Unit](r => updateMedia(r.toUpdateMediaRequest)) - - val undeleteMedia: FutureArrow[UndeleteMediaRequest, Unit] = - FutureArrow[UndeleteMediaRequest, Unit](r => updateMedia(r.toUpdateMediaRequest)) - } -} - -/** - * Exceptions from the various media services backends that indicate bad requests (validation - * failures) are converted to a MediaClientException. Exceptions that indicate a server - * error are converted to a UpstreamFailure.MediaServiceServerError. - * - * MediaNotFound: Given media id does not exist. It could have been expired - * BadMedia: Given media is corrupted and can not be processed. - * InvalidMedia: Given media has failed to pass one or more validations (size, dimensions, type etc.) - * BadRequest Request is bad, but reason not available - */ -object MediaExceptions { - import UpstreamFailure.MediaServiceServerError - - // Extends NoStackTrace because the circumstances in which the - // exceptions are generated don't yield useful stack traces - // (e.g. you can't tell from the stack trace anything about what - // backend call was being made.) - abstract class MediaClientException(message: String) extends Exception(message) with NoStackTrace - - class MediaNotFound(message: String) extends MediaClientException(message) - class BadMedia(message: String) extends MediaClientException(message) - class InvalidMedia(message: String) extends MediaClientException(message) - class BadRequest(message: String) extends MediaClientException(message) - - // translations from various media service errors into MediaExceptions - val handleMediaExceptions: PartialFunction[Any, Exception] = { - case uis.BadRequest(msg, reason) => - reason match { - case Some(uis.BadRequestReason.MediaNotFound) => new MediaNotFound(msg) - case Some(uis.BadRequestReason.BadMedia) => new BadMedia(msg) - case Some(uis.BadRequestReason.InvalidMedia) => new InvalidMedia(msg) - case _ => new BadRequest(msg) - } - case ifs.BadRequest(msg, reason) => - reason match { - case Some(ifs.BadRequestReason.NotFound) => new MediaNotFound(msg) - case _ => new BadRequest(msg) - } - case mis.BadRequest(msg, reason) => - reason match { - case Some(mis.BadRequestReason.MediaNotFound) => new MediaNotFound(msg) - case _ => new BadRequest(msg) - } - case ex: CommonServerError => MediaServiceServerError(ex) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyClassifier.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyClassifier.docx new file mode 100644 index 000000000..540152b05 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyClassifier.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyClassifier.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyClassifier.scala deleted file mode 100644 index 013bd0dea..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyClassifier.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.tweetypie.media - -import com.twitter.mediaservices.commons.thriftscala.MediaKey -import com.twitter.mediaservices.commons.thriftscala.MediaCategory - -object MediaKeyClassifier { - - class Classifier(categories: Set[MediaCategory]) { - - def apply(mediaKey: MediaKey): Boolean = - categories.contains(mediaKey.mediaCategory) - - def unapply(mediaKey: MediaKey): Option[MediaKey] = - apply(mediaKey) match { - case false => None - case true => Some(mediaKey) - } - } - - val isImage: Classifier = new Classifier(Set(MediaCategory.TweetImage)) - val isGif: Classifier = new Classifier(Set(MediaCategory.TweetGif)) - val isVideo: Classifier = new Classifier( - Set(MediaCategory.TweetVideo, MediaCategory.AmplifyVideo) - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyUtil.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyUtil.docx new file mode 100644 index 000000000..df71a6ba0 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyUtil.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyUtil.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyUtil.scala deleted file mode 100644 index 6a62e1d3d..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaKeyUtil.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.tweetypie.media - -import com.twitter.mediaservices.commons.thriftscala._ -import com.twitter.mediaservices.commons.tweetmedia.thriftscala._ -import com.twitter.tweetypie.thriftscala.MediaEntity - -object MediaKeyUtil { - - def get(mediaEntity: MediaEntity): MediaKey = - mediaEntity.mediaKey.getOrElse { - throw new IllegalStateException("""Media key undefined. This state is unexpected, the media - |key should be set by the tweet creation for new tweets - |and by `MediaKeyHydrator` for legacy tweets.""".stripMargin) - } - - def contentType(mediaKey: MediaKey): MediaContentType = - mediaKey.mediaCategory match { - case MediaCategory.TweetImage => MediaContentType.ImageJpeg - case MediaCategory.TweetGif => MediaContentType.VideoMp4 - case MediaCategory.TweetVideo => MediaContentType.VideoGeneric - case MediaCategory.AmplifyVideo => MediaContentType.VideoGeneric - case mediaCats => throw new NotImplementedError(mediaCats.toString) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaMetadata.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaMetadata.docx new file mode 100644 index 000000000..8c5cb90b4 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaMetadata.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaMetadata.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaMetadata.scala deleted file mode 100644 index 135ec014d..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/media/MediaMetadata.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.tweetypie -package media - -import com.twitter.mediaservices.commons.mediainformation.{thriftscala => mic} -import com.twitter.mediaservices.commons.thriftscala.MediaKey -import com.twitter.mediaservices.commons.tweetmedia.thriftscala._ -import com.twitter.tweetypie.thriftscala._ -import java.nio.ByteBuffer - -/** - * MediaMetadata encapsulates the metadata about tweet media that we receive from - * the various media services backends on tweet create or on tweet read. This data, - * combined with data stored on the tweet, is sufficient to hydrate tweet media entities. - */ -case class MediaMetadata( - mediaKey: MediaKey, - assetUrlHttps: String, - sizes: Set[MediaSize], - mediaInfo: MediaInfo, - productMetadata: Option[mic.UserDefinedProductMetadata] = None, - extensionsReply: Option[ByteBuffer] = None, - additionalMetadata: Option[mic.AdditionalMetadata] = None) { - def assetUrlHttp: String = MediaUrl.httpsToHttp(assetUrlHttps) - - def attributableUserId: Option[UserId] = - additionalMetadata.flatMap(_.ownershipInfo).flatMap(_.attributableUserId) - - def updateEntity( - mediaEntity: MediaEntity, - tweetUserId: UserId, - includeAdditionalMetadata: Boolean - ): MediaEntity = { - // Abort if we accidentally try to replace the media. This - // indicates a logic error that caused mismatched media info. - // This could be internal or external to TweetyPie. - require( - mediaEntity.mediaId == mediaKey.mediaId, - "Tried to update media with mediaId=%s with mediaInfo.mediaId=%s" - .format(mediaEntity.mediaId, mediaKey.mediaId) - ) - - mediaEntity.copy( - mediaUrl = assetUrlHttp, - mediaUrlHttps = assetUrlHttps, - sizes = sizes, - mediaInfo = Some(mediaInfo), - extensionsReply = extensionsReply, - // the following two fields are deprecated and will be removed soon - nsfw = false, - mediaPath = MediaUrl.mediaPathFromUrl(assetUrlHttps), - metadata = productMetadata, - additionalMetadata = additionalMetadata.filter(_ => includeAdditionalMetadata), - // MIS allows media to be shared among authorized users so add in sourceUserId if it doesn't - // match the current tweet's userId. - sourceUserId = attributableUserId.filter(_ != tweetUserId) - ) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/package.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/package.docx new file mode 100644 index 000000000..f07aad7c5 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/package.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/package.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/package.scala deleted file mode 100644 index c2f836e97..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/package.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter - -import com.twitter.mediaservices.commons.thriftscala.MediaKey -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.tweetypie.thriftscala._ -import com.twitter.gizmoduck.thriftscala.QueryFields - -package object tweetypie { - // common imports that many classes need, will probably expand this list in the future. - type Logger = com.twitter.util.logging.Logger - val Logger: com.twitter.util.logging.Logger.type = com.twitter.util.logging.Logger - type StatsReceiver = com.twitter.finagle.stats.StatsReceiver - val TweetLenses: com.twitter.tweetypie.util.TweetLenses.type = - com.twitter.tweetypie.util.TweetLenses - - type Future[A] = com.twitter.util.Future[A] - val Future: com.twitter.util.Future.type = com.twitter.util.Future - - type Duration = com.twitter.util.Duration - val Duration: com.twitter.util.Duration.type = com.twitter.util.Duration - - type Time = com.twitter.util.Time - val Time: com.twitter.util.Time.type = com.twitter.util.Time - - type Try[A] = com.twitter.util.Try[A] - val Try: com.twitter.util.Try.type = com.twitter.util.Try - - type Throw[A] = com.twitter.util.Throw[A] - val Throw: com.twitter.util.Throw.type = com.twitter.util.Throw - - type Return[A] = com.twitter.util.Return[A] - val Return: com.twitter.util.Return.type = com.twitter.util.Return - - type Gate[T] = com.twitter.servo.util.Gate[T] - val Gate: com.twitter.servo.util.Gate.type = com.twitter.servo.util.Gate - - type Effect[A] = com.twitter.servo.util.Effect[A] - val Effect: com.twitter.servo.util.Effect.type = com.twitter.servo.util.Effect - - type FutureArrow[A, B] = com.twitter.servo.util.FutureArrow[A, B] - val FutureArrow: com.twitter.servo.util.FutureArrow.type = com.twitter.servo.util.FutureArrow - - type FutureEffect[A] = com.twitter.servo.util.FutureEffect[A] - val FutureEffect: com.twitter.servo.util.FutureEffect.type = com.twitter.servo.util.FutureEffect - - type Lens[A, B] = com.twitter.servo.data.Lens[A, B] - val Lens: com.twitter.servo.data.Lens.type = com.twitter.servo.data.Lens - - type Mutation[A] = com.twitter.servo.data.Mutation[A] - val Mutation: com.twitter.servo.data.Mutation.type = com.twitter.servo.data.Mutation - - type User = com.twitter.gizmoduck.thriftscala.User - val User: com.twitter.gizmoduck.thriftscala.User.type = com.twitter.gizmoduck.thriftscala.User - type Safety = com.twitter.gizmoduck.thriftscala.Safety - val Safety: com.twitter.gizmoduck.thriftscala.Safety.type = - com.twitter.gizmoduck.thriftscala.Safety - type UserField = com.twitter.gizmoduck.thriftscala.QueryFields - val UserField: QueryFields.type = com.twitter.gizmoduck.thriftscala.QueryFields - - type Tweet = thriftscala.Tweet - val Tweet: com.twitter.tweetypie.thriftscala.Tweet.type = thriftscala.Tweet - - type ThriftTweetService = TweetServiceInternal.MethodPerEndpoint - - type TweetId = Long - type UserId = Long - type MediaId = Long - type AppId = Long - type KnownDeviceToken = String - type ConversationId = Long - type CommunityId = Long - type PlaceId = String - type FieldId = Short - type Count = Long - type CountryCode = String // ISO 3166-1-alpha-2 - type CreativesContainerId = Long - - def hasGeo(tweet: Tweet): Boolean = - TweetLenses.placeId.get(tweet).nonEmpty || - TweetLenses.geoCoordinates.get(tweet).nonEmpty - - def getUserId(tweet: Tweet): UserId = TweetLenses.userId.get(tweet) - def getText(tweet: Tweet): String = TweetLenses.text.get(tweet) - def getCreatedAt(tweet: Tweet): Long = TweetLenses.createdAt.get(tweet) - def getCreatedVia(tweet: Tweet): String = TweetLenses.createdVia.get(tweet) - def getReply(tweet: Tweet): Option[Reply] = TweetLenses.reply.get(tweet) - def getDirectedAtUser(tweet: Tweet): Option[DirectedAtUser] = - TweetLenses.directedAtUser.get(tweet) - def getShare(tweet: Tweet): Option[Share] = TweetLenses.share.get(tweet) - def getQuotedTweet(tweet: Tweet): Option[QuotedTweet] = TweetLenses.quotedTweet.get(tweet) - def getUrls(tweet: Tweet): Seq[UrlEntity] = TweetLenses.urls.get(tweet) - def getMedia(tweet: Tweet): Seq[MediaEntity] = TweetLenses.media.get(tweet) - def getMediaKeys(tweet: Tweet): Seq[MediaKey] = TweetLenses.mediaKeys.get(tweet) - def getMentions(tweet: Tweet): Seq[MentionEntity] = TweetLenses.mentions.get(tweet) - def getCashtags(tweet: Tweet): Seq[CashtagEntity] = TweetLenses.cashtags.get(tweet) - def getHashtags(tweet: Tweet): Seq[HashtagEntity] = TweetLenses.hashtags.get(tweet) - def getMediaTagMap(tweet: Tweet): Map[MediaId, Seq[MediaTag]] = TweetLenses.mediaTagMap.get(tweet) - def isRetweet(tweet: Tweet): Boolean = tweet.coreData.flatMap(_.share).nonEmpty - def isSelfReply(authorUserId: UserId, r: Reply): Boolean = - r.inReplyToStatusId.isDefined && (r.inReplyToUserId == authorUserId) - def isSelfReply(tweet: Tweet): Boolean = { - getReply(tweet).exists { r => isSelfReply(getUserId(tweet), r) } - } - def getConversationId(tweet: Tweet): Option[TweetId] = TweetLenses.conversationId.get(tweet) - def getSelfThreadMetadata(tweet: Tweet): Option[SelfThreadMetadata] = - TweetLenses.selfThreadMetadata.get(tweet) - def getCardReference(tweet: Tweet): Option[CardReference] = TweetLenses.cardReference.get(tweet) - def getEscherbirdAnnotations(tweet: Tweet): Option[EscherbirdEntityAnnotations] = - TweetLenses.escherbirdEntityAnnotations.get(tweet) - def getCommunities(tweet: Tweet): Option[Communities] = TweetLenses.communities.get(tweet) - def getTimestamp(tweet: Tweet): Time = - if (SnowflakeId.isSnowflakeId(tweet.id)) SnowflakeId(tweet.id).time - else Time.fromSeconds(getCreatedAt(tweet).toInt) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/BUILD b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/BUILD deleted file mode 100644 index a57db5f55..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/BUILD +++ /dev/null @@ -1,82 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", - "3rdparty/jvm/com/ibm/icu:icu4j", - "3rdparty/jvm/com/twitter/bijection:core", - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/bijection:thrift", - "3rdparty/jvm/org/apache/thrift:libthrift", - "audience-rewards/thrift/src/main/thrift:thrift-scala", - "creatives-container/thrift/src/main/thrift:creatives-container-service-scala", - "featureswitches/featureswitches-core/src/main/scala:recipient", - "featureswitches/featureswitches-core/src/main/scala:useragent", - "finagle/finagle-core/src/main", - "flock-client/src/main/scala", - "flock-client/src/main/thrift:thrift-scala", - "geoduck/util/src/main/scala/com/twitter/geoduck/util/primitives", - "geoduck/util/src/main/scala/com/twitter/geoduck/util/service", - "passbird/thrift-only/src/main/thrift:thrift-scala", - "scrooge/scrooge-core", - "tweetypie/servo/json", - "tweetypie/servo/repo", - "tweetypie/servo/repo/src/main/thrift:thrift-scala", - "tweetypie/servo/util", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/common/text/language:language-identifier", - "src/java/com/twitter/common/text/pipeline", - "src/scala/com/twitter/search/blender/services/strato", - "src/scala/com/twitter/takedown/util", - "src/thrift/com/twitter/consumer_privacy/mention_controls:thrift-scala", - "src/thrift/com/twitter/context:twitter-context-scala", - "src/thrift/com/twitter/dataproducts:enrichments_profilegeo-scala", - "src/thrift/com/twitter/dataproducts:service-scala", - "src/thrift/com/twitter/escherbird:media-annotation-structs-scala", - "src/thrift/com/twitter/escherbird/common:common-scala", - "src/thrift/com/twitter/escherbird/metadata:metadata-service-scala", - "src/thrift/com/twitter/expandodo:cards-scala", - "src/thrift/com/twitter/expandodo:only-scala", - "src/thrift/com/twitter/geoduck:geoduck-scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/service/scarecrow/gen:scarecrow-scala", - "src/thrift/com/twitter/service/scarecrow/gen:tiered-actions-scala", - "src/thrift/com/twitter/service/talon/gen:thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-label-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/spam/rtf:tweet-rtf-event-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:media-entity-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:service-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-compat/src/main/scala/com/twitter/stitch/compat", - "stitch/stitch-core", - "stitch/stitch-timelineservice", - "strato/src/main/scala/com/twitter/strato/catalog", - "strato/src/main/scala/com/twitter/strato/client", - "strato/src/main/scala/com/twitter/strato/data", - "strato/src/main/scala/com/twitter/strato/thrift", - "tweetypie/server/src/main/scala/com/twitter/tweetypie", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/backends", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/core", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/media", - "tweetypie/server/src/main/thrift:compiled-scala", - "tweetypie/common/src/scala/com/twitter/tweetypie/client_id", - "tweetypie/common/src/scala/com/twitter/tweetypie/media", - "tweetypie/common/src/scala/com/twitter/tweetypie/storage", - "tweetypie/common/src/scala/com/twitter/tweetypie/util", - "twitter-context/src/main/scala", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - "util/util-stats/src/main/scala", - "vibes/src/main/thrift/com/twitter/vibes:vibes-scala", - "visibility/common/src/main/scala/com/twitter/visibility/common/tflock", - "visibility/common/src/main/scala/com/twitter/visibility/common/user_result", - "visibility/common/src/main/thrift/com/twitter/visibility:action-scala", - "visibility/lib:tweets", - ], -) diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/BUILD.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/BUILD.docx new file mode 100644 index 000000000..73f9cdf0f Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/BUILD.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CacheStitch.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CacheStitch.docx new file mode 100644 index 000000000..617d60d4c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CacheStitch.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CacheStitch.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CacheStitch.scala deleted file mode 100644 index fd1ad5fd3..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CacheStitch.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.servo.repository._ -import com.twitter.stitch.Stitch -import com.twitter.util.Try - -object CacheStitch { - - /** - * Cacheable defines a function that takes a cache query and a Try value, - * and returns what should be written to cache, as a Option[StitchLockingCache.Val]. - * - * None signifies that this value should not be written to cache. - * - * Val can be one of Found[V], NotFound, and Deleted. The function will determine what kinds - * of values and exceptions (captured in the Try) correspond to which kind of cached values. - */ - type Cacheable[Q, V] = (Q, Try[V]) => Option[StitchLockingCache.Val[V]] - - // Cache successful values as Found, stitch.NotFound as NotFound, and don't cache other exceptions - def cacheFoundAndNotFound[K, V]: CacheStitch.Cacheable[K, V] = - (_, t: Try[V]) => - t match { - // Write successful values as Found - case Return(v) => Some(StitchLockingCache.Val.Found[V](v)) - - // Write stitch.NotFound as NotFound - case Throw(com.twitter.stitch.NotFound) => Some(StitchLockingCache.Val.NotFound) - - // Don't write other exceptions back to cache - case _ => None - } -} - -case class CacheStitch[Q, K, V]( - repo: Q => Stitch[V], - cache: StitchLockingCache[K, V], - queryToKey: Q => K, - handler: CachedResult.Handler[K, V], - cacheable: CacheStitch.Cacheable[Q, V]) - extends (Q => Stitch[V]) { - import com.twitter.servo.repository.CachedResultAction._ - - private[this] def getFromCache(key: K): Stitch[CachedResult[K, V]] = { - cache - .get(key) - .handle { - case t => CachedResult.Failed(key, t) - } - } - - // Exposed for testing - private[repository] def readThrough(query: Q): Stitch[V] = - repo(query).liftToTry.applyEffect { value: Try[V] => - cacheable(query, value) match { - case Some(v) => - // cacheable returned Some of a StitchLockingCache.Val to cache - // - // This is async to ensure that we don't wait for the cache - // update to complete before returning. This also ignores - // any exceptions from setting the value. - Stitch.async(cache.lockAndSet(queryToKey(query), v)) - case None => - // cacheable returned None so don't cache - Stitch.Unit - } - }.lowerFromTry - - private[this] def handle(query: Q, action: CachedResultAction[V]): Stitch[V] = - action match { - case HandleAsFound(value) => Stitch(value) - case HandleAsMiss => readThrough(query) - case HandleAsDoNotCache => repo(query) - case HandleAsFailed(t) => Stitch.exception(t) - case HandleAsNotFound => Stitch.NotFound - case t: TransformSubAction[V] => handle(query, t.action).map(t.f) - case SoftExpiration(subAction) => - Stitch - .async(readThrough(query)) - .flatMap { _ => handle(query, subAction) } - } - - override def apply(query: Q): Stitch[V] = - getFromCache(queryToKey(query)) - .flatMap { result: CachedResult[K, V] => handle(query, handler(result)) } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CachingTweetRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CachingTweetRepository.docx new file mode 100644 index 000000000..dbcf15458 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CachingTweetRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CachingTweetRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CachingTweetRepository.scala deleted file mode 100644 index 0ebf12998..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CachingTweetRepository.scala +++ /dev/null @@ -1,329 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.twitter.finagle.tracing.Trace -import com.twitter.servo.cache._ -import com.twitter.servo.repository._ -import com.twitter.servo.util.Transformer -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.client_id.ClientIdHelper -import com.twitter.tweetypie.core.FilteredState.Unavailable.BounceDeleted -import com.twitter.tweetypie.core.FilteredState.Unavailable.TweetDeleted -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository.CachedBounceDeleted.isBounceDeleted -import com.twitter.tweetypie.repository.CachedBounceDeleted.toBounceDeletedTweetResult -import com.twitter.tweetypie.thriftscala.CachedTweet -import com.twitter.util.Base64Long - -case class TweetKey(cacheVersion: Int, id: TweetId) - extends ScopedCacheKey("t", "t", cacheVersion, Base64Long.toBase64(id)) - -case class TweetKeyFactory(cacheVersion: Int) { - val fromId: TweetId => TweetKey = (id: TweetId) => TweetKey(cacheVersion, id) - val fromTweet: Tweet => TweetKey = (tweet: Tweet) => fromId(tweet.id) - val fromCachedTweet: CachedTweet => TweetKey = (ms: CachedTweet) => fromTweet(ms.tweet) -} - -// Helper methods for working with cached bounce-deleted tweets, -// grouped together here to keep the definitions of "bounce -// deleted" in one place. -object CachedBounceDeleted { - // CachedTweet for use in CachingTweetStore - def toBounceDeletedCachedTweet(tweetId: TweetId): CachedTweet = - CachedTweet( - tweet = Tweet(id = tweetId), - isBounceDeleted = Some(true) - ) - - def isBounceDeleted(cached: Cached[CachedTweet]): Boolean = - cached.status == CachedValueStatus.Found && - cached.value.flatMap(_.isBounceDeleted).contains(true) - - // TweetResult for use in CachingTweetRepository - def toBounceDeletedTweetResult(tweetId: TweetId): TweetResult = - TweetResult( - TweetData( - tweet = Tweet(id = tweetId), - isBounceDeleted = true - ) - ) - - def isBounceDeleted(tweetResult: TweetResult): Boolean = - tweetResult.value.isBounceDeleted -} - -object TweetResultCache { - def apply( - tweetDataCache: Cache[TweetId, Cached[TweetData]] - ): Cache[TweetId, Cached[TweetResult]] = { - val transformer: Transformer[Cached[TweetResult], Cached[TweetData]] = - new Transformer[Cached[TweetResult], Cached[TweetData]] { - def to(cached: Cached[TweetResult]) = - Return(cached.map(_.value)) - - def from(cached: Cached[TweetData]) = - Return(cached.map(TweetResult(_))) - } - - new KeyValueTransformingCache( - tweetDataCache, - transformer, - identity - ) - } -} - -object TweetDataCache { - def apply( - cachedTweetCache: Cache[TweetKey, Cached[CachedTweet]], - tweetKeyFactory: TweetId => TweetKey - ): Cache[TweetId, Cached[TweetData]] = { - val transformer: Transformer[Cached[TweetData], Cached[CachedTweet]] = - new Transformer[Cached[TweetData], Cached[CachedTweet]] { - def to(cached: Cached[TweetData]) = - Return(cached.map(_.toCachedTweet)) - - def from(cached: Cached[CachedTweet]) = - Return(cached.map(c => TweetData.fromCachedTweet(c, cached.cachedAt))) - } - - new KeyValueTransformingCache( - cachedTweetCache, - transformer, - tweetKeyFactory - ) - } -} - -object TombstoneTtl { - import CachedResult._ - - def fixed(ttl: Duration): CachedNotFound[TweetId] => Duration = - _ => ttl - - /** - * A simple ttl calculator that is set to `min` if the age is less than `from`, - * then linearly interpolated between `min` and `max` when the age is between `from` and `to`, - * and then equal to `max` if the age is greater than `to`. - */ - def linear( - min: Duration, - max: Duration, - from: Duration, - to: Duration - ): CachedNotFound[TweetId] => Duration = { - val rate = (max - min).inMilliseconds / (to - from).inMilliseconds.toDouble - cached => { - if (SnowflakeId.isSnowflakeId(cached.key)) { - val age = cached.cachedAt - SnowflakeId(cached.key).time - if (age <= from) min - else if (age >= to) max - else min + (age - from) * rate - } else { - // When it's not a snowflake id, cache it for the maximum time. - max - } - } - } - - /** - * Checks if the given `cached` value is an expired tombstone - */ - def isExpired( - tombstoneTtl: CachedNotFound[TweetId] => Duration, - cached: CachedNotFound[TweetId] - ): Boolean = - Time.now - cached.cachedAt > tombstoneTtl(cached) -} - -object CachingTweetRepository { - import CachedResult._ - import CachedResultAction._ - - val failuresLog: Logger = Logger("com.twitter.tweetypie.repository.CachingTweetRepoFailures") - - def apply( - cache: LockingCache[TweetId, Cached[TweetResult]], - tombstoneTtl: CachedNotFound[TweetId] => Duration, - stats: StatsReceiver, - clientIdHelper: ClientIdHelper, - logCacheExceptions: Gate[Unit] = Gate.False, - )( - underlying: TweetResultRepository.Type - ): TweetResultRepository.Type = { - val cachingRepo: ((TweetId, TweetQuery.Options)) => Stitch[TweetResult] = - CacheStitch[(TweetId, TweetQuery.Options), TweetId, TweetResult]( - repo = underlying.tupled, - cache = StitchLockingCache( - underlying = cache, - picker = new TweetRepoCachePicker[TweetResult](_.value.cachedAt) - ), - queryToKey = _._1, // extract tweet id from (TweetId, TweetQuery.Options) - handler = mkHandler(tombstoneTtl, stats, logCacheExceptions, clientIdHelper), - cacheable = cacheable - ) - - (tweetId, options) => - if (options.cacheControl.readFromCache) { - cachingRepo((tweetId, options)) - } else { - underlying(tweetId, options) - } - } - - val cacheable: CacheStitch.Cacheable[(TweetId, TweetQuery.Options), TweetResult] = { - case ((tweetId, options), tweetResult) => - if (!options.cacheControl.writeToCache) { - None - } else { - tweetResult match { - // Write stitch.NotFound as a NotFound cache entry - case Throw(com.twitter.stitch.NotFound) => - Some(StitchLockingCache.Val.NotFound) - - // Write FilteredState.TweetDeleted as a Deleted cache entry - case Throw(TweetDeleted) => - Some(StitchLockingCache.Val.Deleted) - - // Write BounceDeleted as a Found cache entry, with the CachedTweet.isBounceDeleted flag. - // servo.cache.thriftscala.CachedValueStatus.Deleted tombstones do not allow for storing - // app-defined metadata. - case Throw(BounceDeleted) => - Some(StitchLockingCache.Val.Found(toBounceDeletedTweetResult(tweetId))) - - // Regular found tweets are not written to cache here - instead the cacheable result is - // written to cache via TweetHydration.cacheChanges - case Return(_: TweetResult) => None - - // Don't write other exceptions back to cache - case _ => None - } - } - } - - object LogLens { - private[this] val mapper = new ObjectMapper().registerModule(DefaultScalaModule) - - def logMessage(logger: Logger, clientIdHelper: ClientIdHelper, data: (String, Any)*): Unit = { - val allData = data ++ defaultData(clientIdHelper) - val msg = mapper.writeValueAsString(Map(allData: _*)) - logger.info(msg) - } - - private def defaultData(clientIdHelper: ClientIdHelper): Seq[(String, Any)] = { - val viewer = TwitterContext() - Seq( - "client_id" -> clientIdHelper.effectiveClientId, - "trace_id" -> Trace.id.traceId.toString, - "audit_ip" -> viewer.flatMap(_.auditIp), - "application_id" -> viewer.flatMap(_.clientApplicationId), - "user_agent" -> viewer.flatMap(_.userAgent), - "authenticated_user_id" -> viewer.flatMap(_.authenticatedUserId) - ) - } - } - - def mkHandler( - tombstoneTtl: CachedNotFound[TweetId] => Duration, - stats: StatsReceiver, - logCacheExceptions: Gate[Unit], - clientIdHelper: ClientIdHelper, - ): Handler[TweetId, TweetResult] = { - val baseHandler = defaultHandler[TweetId, TweetResult] - val cacheErrorState = HydrationState(modified = false, cacheErrorEncountered = true) - val cachedFoundCounter = stats.counter("cached_found") - val notFoundCounter = stats.counter("not_found") - val cachedNotFoundAsNotFoundCounter = stats.counter("cached_not_found_as_not_found") - val cachedNotFoundAsMissCounter = stats.counter("cached_not_found_as_miss") - val cachedDeletedCounter = stats.counter("cached_deleted") - val cachedBounceDeletedCounter = stats.counter("cached_bounce_deleted") - val failedCounter = stats.counter("failed") - val otherCounter = stats.counter("other") - - { - case res @ CachedFound(_, tweetResult, _, _) => - if (isBounceDeleted(tweetResult)) { - cachedBounceDeletedCounter.incr() - HandleAsFailed(FilteredState.Unavailable.BounceDeleted) - } else { - cachedFoundCounter.incr() - baseHandler(res) - } - - case res @ NotFound(_) => - notFoundCounter.incr() - baseHandler(res) - - // expires NotFound tombstones if old enough - case cached @ CachedNotFound(_, _, _) => - if (TombstoneTtl.isExpired(tombstoneTtl, cached)) { - cachedNotFoundAsMissCounter.incr() - HandleAsMiss - } else { - cachedNotFoundAsNotFoundCounter.incr() - HandleAsNotFound - } - - case CachedDeleted(_, _, _) => - cachedDeletedCounter.incr() - HandleAsFailed(FilteredState.Unavailable.TweetDeleted) - - // don't attempt to write back to cache on a cache read failure - case Failed(k, t) => - // After result is found, mark it with cacheErrorEncountered - failedCounter.incr() - - if (logCacheExceptions()) { - LogLens.logMessage( - failuresLog, - clientIdHelper, - "type" -> "cache_failed", - "tweet_id" -> k, - "throwable" -> t.getClass.getName - ) - } - - TransformSubAction[TweetResult](HandleAsDoNotCache, _.mapState(_ ++ cacheErrorState)) - - case res => - otherCounter.incr() - baseHandler(res) - } - - } -} - -/** - * A LockingCache.Picker for use with CachingTweetRepository which prevents overwriting values in - * cache that are newer than the value previously read from cache. - */ -class TweetRepoCachePicker[T](cachedAt: T => Option[Time]) extends LockingCache.Picker[Cached[T]] { - private val newestPicker = new PreferNewestCached[T] - - override def apply(newValue: Cached[T], oldValue: Cached[T]): Option[Cached[T]] = { - oldValue.status match { - // never overwrite a `Deleted` tombstone via read-through. - case CachedValueStatus.Deleted => None - - // only overwrite a `Found` value with an update based off of that same cache entry. - case CachedValueStatus.Found => - newValue.value.flatMap(cachedAt) match { - // if prevCacheAt is the same as oldValue.cachedAt, then the value in cache hasn't changed - case Some(prevCachedAt) if prevCachedAt == oldValue.cachedAt => Some(newValue) - // otherwise, the value in cache has changed since we read it, so don't overwrite - case _ => None - } - - // we may hit an expired/older tombstone, which should be safe to overwrite with a fresh - // tombstone of a new value returned from Manhattan. - case CachedValueStatus.NotFound => newestPicker(newValue, oldValue) - - // we shouldn't see any other CachedValueStatus, but if we do, play it safe and don't - // overwrite (it will be as if the read that triggered this never happened) - case _ => None - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/Card2Repository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/Card2Repository.docx new file mode 100644 index 000000000..564ae2154 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/Card2Repository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/Card2Repository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/Card2Repository.scala deleted file mode 100644 index 9b6f4b154..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/Card2Repository.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.expandodo.thriftscala._ -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup -import com.twitter.tweetypie.backends.Expandodo - -sealed trait Card2Key { - def toCard2Request: Card2Request -} - -final case class UrlCard2Key(url: String) extends Card2Key { - override def toCard2Request: Card2Request = - Card2Request(`type` = Card2RequestType.ByUrl, url = Some(url)) -} - -final case class ImmediateValuesCard2Key(values: Seq[Card2ImmediateValue], tweetId: TweetId) - extends Card2Key { - override def toCard2Request: Card2Request = - Card2Request( - `type` = Card2RequestType.ByImmediateValues, - immediateValues = Some(values), - statusId = Some(tweetId) - ) -} - -object Card2Repository { - type Type = (Card2Key, Card2RequestOptions) => Stitch[Card2] - - def apply(getCards2: Expandodo.GetCards2, maxRequestSize: Int): Type = { - case class RequestGroup(opts: Card2RequestOptions) extends SeqGroup[Card2Key, Option[Card2]] { - override def run(keys: Seq[Card2Key]): Future[Seq[Try[Option[Card2]]]] = - LegacySeqGroup.liftToSeqTry( - getCards2((keys.map(_.toCard2Request), opts)).map { res => - res.responsesCode match { - case Card2ResponsesCode.Ok => - res.responses.map(_.card) - - case _ => - // treat all other failure cases as card-not-found - Seq.fill(keys.size)(None) - } - } - ) - - override def maxSize: Int = maxRequestSize - } - - (card2Key, opts) => - Stitch - .call(card2Key, RequestGroup(opts)) - .lowerFromOption() - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardRepository.docx new file mode 100644 index 000000000..7f44baa93 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardRepository.scala deleted file mode 100644 index b420b5814..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardRepository.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.expandodo.thriftscala._ -import com.twitter.stitch.MapGroup -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.backends.Expandodo - -object CardRepository { - type Type = String => Stitch[Seq[Card]] - - def apply(getCards: Expandodo.GetCards, maxRequestSize: Int): Type = { - object RequestGroup extends MapGroup[String, Seq[Card]] { - override def run(urls: Seq[String]): Future[String => Try[Seq[Card]]] = - getCards(urls.toSet).map { responseMap => url => - responseMap.get(url) match { - case None => Throw(NotFound) - case Some(r) => Return(r.cards.getOrElse(Nil)) - } - } - - override def maxSize: Int = maxRequestSize - } - - url => Stitch.call(url, RequestGroup) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardUsersRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardUsersRepository.docx new file mode 100644 index 000000000..ef57f6fbc Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardUsersRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardUsersRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardUsersRepository.scala deleted file mode 100644 index 3cf546bb7..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CardUsersRepository.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.expandodo.thriftscala._ -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup -import com.twitter.tweetypie.backends.Expandodo - -object CardUsersRepository { - type CardUri = String - type Type = (CardUri, Context) => Stitch[Option[Set[UserId]]] - - case class Context(perspectiveUserId: UserId) extends AnyVal - - case class GetUsersGroup(perspectiveId: UserId, getCardUsers: Expandodo.GetCardUsers) - extends SeqGroup[CardUri, GetCardUsersResponse] { - protected override def run(keys: Seq[CardUri]): Future[Seq[Try[GetCardUsersResponse]]] = - LegacySeqGroup.liftToSeqTry( - getCardUsers( - GetCardUsersRequests( - requests = keys.map(k => GetCardUsersRequest(k)), - perspectiveUserId = Some(perspectiveId) - ) - ).map(_.responses) - ) - } - - def apply(getCardUsers: Expandodo.GetCardUsers): Type = - (cardUri, ctx) => - Stitch.call(cardUri, GetUsersGroup(ctx.perspectiveUserId, getCardUsers)).map { resp => - val authorUserIds = resp.authorUserIds.map(_.toSet) - val siteUserIds = resp.siteUserIds.map(_.toSet) - - if (authorUserIds.isEmpty) { - siteUserIds - } else if (siteUserIds.isEmpty) { - authorUserIds - } else { - Some(authorUserIds.get ++ siteUserIds.get) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationControlRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationControlRepository.docx new file mode 100644 index 000000000..4befed07e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationControlRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationControlRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationControlRepository.scala deleted file mode 100644 index 64052b116..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationControlRepository.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.FilteredState.Unavailable.TweetDeleted -import com.twitter.tweetypie.thriftscala.ConversationControl - -/** - * This repository loads up the conversation control values for a tweet which controls who can reply - * to a tweet. Because the conversation control values are stored on the root tweet of a conversation, - * we need to make sure that the code is able to load the data from the root tweet. To ensure this, - * no visibility filtering options are set on the query to load the root tweet fields. - * - * If visibility filtering was enabled, and the root tweet was filtered for the requesting user, - * then the conversation control data would not be returned and enforcement would effectively be - * side-stepped. - */ -object ConversationControlRepository { - private[this] val log = Logger(getClass) - type Type = (TweetId, CacheControl) => Stitch[Option[ConversationControl]] - - def apply(repo: TweetRepository.Type, stats: StatsReceiver): Type = - (conversationId: TweetId, cacheControl: CacheControl) => { - val options = TweetQuery.Options( - include = TweetQuery.Include(Set(Tweet.ConversationControlField.id)), - // We want the root tweet of a conversation that we're looking up to be - // cached with the same policy as the tweet we're looking up. - cacheControl = cacheControl, - enforceVisibilityFiltering = false, - safetyLevel = SafetyLevel.FilterNone - ) - - repo(conversationId, options) - .map(rootTweet => rootTweet.conversationControl) - .handle { - // We don't know of any case where tweets would return NotFound, but for - // for pragmatic reasons, we're opening the conversation for replies - // in case a bug causing tweets to be NotFound exists. - case NotFound => - stats.counter("tweet_not_found") - None - // If no root tweet is found, the reply has no conversation controls - // this is by design, deleting the root tweet "opens" the conversation - case TweetDeleted => - stats.counter("tweet_deleted") - None - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationIdRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationIdRepository.docx new file mode 100644 index 000000000..aa51e4363 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationIdRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationIdRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationIdRepository.scala deleted file mode 100644 index b9a9b26ad..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationIdRepository.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.flockdb.client._ -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup - -case class ConversationIdKey(tweetId: TweetId, parentId: TweetId) - -object ConversationIdRepository { - type Type = ConversationIdKey => Stitch[TweetId] - - def apply(multiSelectOne: Iterable[Select[StatusGraph]] => Future[Seq[Option[Long]]]): Type = - key => Stitch.call(key, Group(multiSelectOne)) - - private case class Group( - multiSelectOne: Iterable[Select[StatusGraph]] => Future[Seq[Option[Long]]]) - extends SeqGroup[ConversationIdKey, TweetId] { - - private[this] def getConversationIds( - keys: Seq[ConversationIdKey], - getLookupId: ConversationIdKey => TweetId - ): Future[Map[ConversationIdKey, TweetId]] = { - val distinctIds = keys.map(getLookupId).distinct - val tflockQueries = distinctIds.map(ConversationGraph.to) - if (tflockQueries.isEmpty) { - Future.value(Map[ConversationIdKey, TweetId]()) - } else { - multiSelectOne(tflockQueries).map { results => - // first, we need to match up the distinct ids requested with the corresponding result - val resultMap = - distinctIds - .zip(results) - .collect { - case (id, Some(conversationId)) => id -> conversationId - } - .toMap - - // then we need to map keys into the above map - keys.flatMap { key => resultMap.get(getLookupId(key)).map(key -> _) }.toMap - } - } - } - - /** - * Returns a key-value result that maps keys to the tweet's conversation IDs. - * - * Example: - * Tweet B is a reply to tweet A with conversation ID c. - * We want to get B's conversation ID. Then, for the request - * - * ConversationIdRequest(B.id, A.id) - * - * our key-value result's "found" map will contain a pair (B.id -> c). - */ - protected override def run(keys: Seq[ConversationIdKey]): Future[Seq[Try[TweetId]]] = - LegacySeqGroup.liftToSeqTry( - for { - // Try to get the conversation IDs for the parent tweets - convIdsFromParent <- getConversationIds(keys, _.parentId) - - // Collect the tweet IDs whose parents' conversation IDs couldn't be found. - // We assume that happened in one of two cases: - // * for a tweet whose parent has been deleted - // * for a tweet whose parent is the root of a conversation - // Note: In either case, we will try to look up the conversation ID of the tweet whose parent - // couldn't be found. If that can't be found either, we will eventually return the parent ID. - tweetsWhoseParentsDontHaveConvoIds = keys.toSet -- convIdsFromParent.keys - - // Collect the conversation IDs for the tweets whose parents have not been found, now using the - // tweets' own IDs. - convIdsFromTweet <- - getConversationIds(tweetsWhoseParentsDontHaveConvoIds.toSeq, _.tweetId) - - // Combine the by-parent-ID and by-tweet-ID results. - convIdMap = convIdsFromParent ++ convIdsFromTweet - - // Assign conversation IDs to all not-found tweet IDs. - // A tweet might not have received a conversation ID if - // * the parent of the tweet is the root of the conversation, and we are in the write path - // for creating the tweet. In that case, the conversation ID should be the tweet's parent - // ID. - // * it had been created before TFlock started handling conversation IDs. In that case, the - // conversation ID will just point to the parent tweet so that we can have a conversation of - // at least two tweets. - // So in both cases, we want to return the tweet's parent ID. - } yield { - keys.map { - case k @ ConversationIdKey(t, p) => convIdMap.getOrElse(k, p) - } - } - ) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationMutedRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationMutedRepository.docx new file mode 100644 index 000000000..b551a5b12 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationMutedRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationMutedRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationMutedRepository.scala deleted file mode 100644 index 16e08f46c..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ConversationMutedRepository.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.stitch.Stitch - -object ConversationMutedRepository { - - /** - * Same type as com.twitter.stitch.timelineservice.TimelineService.GetConversationMuted but - * without using Arrow. - */ - type Type = (UserId, TweetId) => Stitch[Boolean] -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CreativesContainerMaterializationRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CreativesContainerMaterializationRepository.docx new file mode 100644 index 000000000..5a738e23b Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CreativesContainerMaterializationRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CreativesContainerMaterializationRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CreativesContainerMaterializationRepository.scala deleted file mode 100644 index d74c1c185..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/CreativesContainerMaterializationRepository.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.container.thriftscala.MaterializeAsTweetFieldsRequest -import com.twitter.container.thriftscala.MaterializeAsTweetRequest -import com.twitter.container.{thriftscala => ccs} -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.Return -import com.twitter.tweetypie.{thriftscala => tp} -import com.twitter.tweetypie.backends -import com.twitter.tweetypie.thriftscala.GetTweetFieldsResult -import com.twitter.tweetypie.thriftscala.GetTweetResult -import com.twitter.util.Future -import com.twitter.util.Try - -/** - * A special kind of tweet is that, when [[tp.Tweet.underlyingCreativesContainerId]] is presented. - * tweetypie will delegate hydration of this tweet to creatives container service. - */ -object CreativesContainerMaterializationRepository { - - type GetTweetType = - (ccs.MaterializeAsTweetRequest, Option[tp.GetTweetOptions]) => Stitch[tp.GetTweetResult] - - type GetTweetFieldsType = - ( - ccs.MaterializeAsTweetFieldsRequest, - tp.GetTweetFieldsOptions - ) => Stitch[tp.GetTweetFieldsResult] - - def apply( - materializeAsTweet: backends.CreativesContainerService.MaterializeAsTweet - ): GetTweetType = { - case class RequestGroup(opts: Option[tp.GetTweetOptions]) - extends SeqGroup[ccs.MaterializeAsTweetRequest, tp.GetTweetResult] { - override protected def run( - keys: Seq[MaterializeAsTweetRequest] - ): Future[Seq[Try[GetTweetResult]]] = - materializeAsTweet(ccs.MaterializeAsTweetRequests(keys, opts)).map { - res: Seq[GetTweetResult] => res.map(Return(_)) - } - } - - (request, options) => Stitch.call(request, RequestGroup(options)) - } - - def materializeAsTweetFields( - materializeAsTweetFields: backends.CreativesContainerService.MaterializeAsTweetFields - ): GetTweetFieldsType = { - case class RequestGroup(opts: tp.GetTweetFieldsOptions) - extends SeqGroup[ccs.MaterializeAsTweetFieldsRequest, tp.GetTweetFieldsResult] { - override protected def run( - keys: Seq[MaterializeAsTweetFieldsRequest] - ): Future[Seq[Try[GetTweetFieldsResult]]] = - materializeAsTweetFields(ccs.MaterializeAsTweetFieldsRequests(keys, opts)).map { - res: Seq[GetTweetFieldsResult] => res.map(Return(_)) - } - } - - (request, options) => Stitch.call(request, RequestGroup(options)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeletedTweetVisibilityRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeletedTweetVisibilityRepository.docx new file mode 100644 index 000000000..2a219ed91 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeletedTweetVisibilityRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeletedTweetVisibilityRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeletedTweetVisibilityRepository.scala deleted file mode 100644 index 711e603c1..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeletedTweetVisibilityRepository.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.spam.rtf.thriftscala.FilteredReason -import com.twitter.spam.rtf.thriftscala.{SafetyLevel => ThriftSafetyLevel} -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.TweetId -import com.twitter.tweetypie.core.FilteredState.HasFilteredReason -import com.twitter.tweetypie.core.FilteredState.Unavailable.BounceDeleted -import com.twitter.tweetypie.core.FilteredState.Unavailable.SourceTweetNotFound -import com.twitter.tweetypie.core.FilteredState.Unavailable.TweetDeleted -import com.twitter.tweetypie.repository.VisibilityResultToFilteredState.toFilteredStateUnavailable -import com.twitter.visibility.interfaces.tweets.DeletedTweetVisibilityLibrary -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.TweetDeleteReason -import com.twitter.visibility.models.TweetDeleteReason.TweetDeleteReason -import com.twitter.visibility.models.ViewerContext - -/** - * Generate FilteredReason for tweet entities in following delete states: - * com.twitter.tweetypie.core.FilteredState.Unavailable - * - SourceTweetNotFound(true) - * - TweetDeleted - * - BounceDeleted - * - * Callers of this repository should be ready to handle empty response (Stitch.None) - * from the underlying VF library when: - * 1.the tweet should not NOT be filtered for the given safety level - * 2.the tweet is not a relevant content to be handled by the library - */ -object DeletedTweetVisibilityRepository { - type Type = VisibilityRequest => Stitch[Option[FilteredReason]] - - case class VisibilityRequest( - filteredState: Throwable, - tweetId: TweetId, - safetyLevel: Option[ThriftSafetyLevel], - viewerId: Option[Long], - isInnerQuotedTweet: Boolean) - - def apply( - visibilityLibrary: DeletedTweetVisibilityLibrary.Type - ): Type = { request => - toVisibilityTweetDeleteState(request.filteredState, request.isInnerQuotedTweet) - .map { deleteReason => - val safetyLevel = SafetyLevel.fromThrift( - request.safetyLevel.getOrElse(ThriftSafetyLevel.FilterDefault) - ) - val isRetweet = request.filteredState == SourceTweetNotFound(true) - visibilityLibrary( - DeletedTweetVisibilityLibrary.Request( - request.tweetId, - safetyLevel, - ViewerContext.fromContextWithViewerIdFallback(request.viewerId), - deleteReason, - isRetweet, - request.isInnerQuotedTweet - ) - ).map(toFilteredStateUnavailable) - .map { - //Accept FilteredReason - case Some(fs) if fs.isInstanceOf[HasFilteredReason] => - Some(fs.asInstanceOf[HasFilteredReason].filteredReason) - case _ => None - } - } - .getOrElse(Stitch.None) - } - - /** - * @return map an error from tweet hydration to a VF model TweetDeleteReason, - * None when the error is not related to delete state tweets. - */ - private def toVisibilityTweetDeleteState( - tweetDeleteState: Throwable, - isInnerQuotedTweet: Boolean - ): Option[TweetDeleteReason] = { - tweetDeleteState match { - case TweetDeleted => Some(TweetDeleteReason.Deleted) - case BounceDeleted => Some(TweetDeleteReason.BounceDeleted) - case SourceTweetNotFound(true) if !isInnerQuotedTweet => Some(TweetDeleteReason.Deleted) - case _ => None - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeviceSourceRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeviceSourceRepository.docx new file mode 100644 index 000000000..a86b7d6a4 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeviceSourceRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeviceSourceRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeviceSourceRepository.scala deleted file mode 100644 index f88513458..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/DeviceSourceRepository.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.passbird.clientapplication.thriftscala.ClientApplication -import com.twitter.passbird.clientapplication.thriftscala.GetClientApplicationsResponse -import com.twitter.servo.cache.ScopedCacheKey -import com.twitter.stitch.MapGroup -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.DeviceSource - -// converts the device source parameter value to lower-case, to make the cached -// key case-insensitive -case class DeviceSourceKey(param: String) extends ScopedCacheKey("t", "ds", 1, param.toLowerCase) - -object DeviceSourceRepository { - type Type = String => Stitch[DeviceSource] - - type GetClientApplications = FutureArrow[Seq[Long], GetClientApplicationsResponse] - - val DefaultUrl = "https://help.twitter.com/en/using-twitter/how-to-tweet#source-labels" - - def formatUrl(name: String, url: String): String = s"""$name""" - - /** - * Construct an html a tag from the client application - * name and url for the display field because some - * clients depend on this. - */ - def deviceSourceDisplay( - name: String, - urlOpt: Option[String] - ): String = - urlOpt match { - case Some(url) => formatUrl(name = name, url = url) // data sanitized by passbird - case None => - formatUrl(name = name, url = DefaultUrl) // data sanitized by passbird - } - - def toDeviceSource(app: ClientApplication): DeviceSource = - DeviceSource( - // The id field used to represent the id of a row - // in the now deprecated device_sources mysql table. - id = 0L, - parameter = "oauth:" + app.id, - internalName = "oauth:" + app.id, - name = app.name, - url = app.url.getOrElse(""), - display = deviceSourceDisplay(app.name, app.url), - clientAppId = Some(app.id) - ) - - def apply( - parseAppId: String => Option[Long], - getClientApplications: GetClientApplications - ): DeviceSourceRepository.Type = { - val getClientApplicationsGroup = new MapGroup[Long, DeviceSource] { - def run(ids: Seq[Long]): Future[Long => Try[DeviceSource]] = - getClientApplications(ids).map { response => id => - response.found.get(id) match { - case Some(app) => Return(toDeviceSource(app)) - case None => Throw(NotFound) - } - } - } - - appIdStr => - parseAppId(appIdStr) match { - case Some(appId) => - Stitch.call(appId, getClientApplicationsGroup) - case None => - Stitch.exception(NotFound) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/EscherbirdAnnotationRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/EscherbirdAnnotationRepository.docx new file mode 100644 index 000000000..9ea5cf0a7 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/EscherbirdAnnotationRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/EscherbirdAnnotationRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/EscherbirdAnnotationRepository.scala deleted file mode 100644 index 57857c386..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/EscherbirdAnnotationRepository.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup -import com.twitter.tweetypie.backends.Escherbird -import com.twitter.tweetypie.thriftscala.EscherbirdEntityAnnotations - -object EscherbirdAnnotationRepository { - type Type = Tweet => Stitch[Option[EscherbirdEntityAnnotations]] - - def apply(annotate: Escherbird.Annotate): Type = - // use a `SeqGroup` to group the future-calls together, even though they can be - // executed independently, in order to help keep hydration between different tweets - // in sync, to improve batching in hydrators which occur later in the pipeline. - tweet => - Stitch - .call(tweet, LegacySeqGroup(annotate.liftSeq)) - .map { annotations => - if (annotations.isEmpty) None - else Some(EscherbirdEntityAnnotations(annotations)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoScrubTimestampRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoScrubTimestampRepository.docx new file mode 100644 index 000000000..b094b965d Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoScrubTimestampRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoScrubTimestampRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoScrubTimestampRepository.scala deleted file mode 100644 index 476790b60..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoScrubTimestampRepository.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.servo.cache.ScopedCacheKey -import com.twitter.stitch.Stitch -import com.twitter.util.Base64Long - -case class GeoScrubTimestampKey(userId: UserId) - extends ScopedCacheKey("t", "gs", 1, Base64Long.toBase64(userId)) - -object GeoScrubTimestampRepository { - type Type = UserId => Stitch[Time] - - def apply(getLastGeoScrubTime: UserId => Stitch[Option[Time]]): Type = - userId => getLastGeoScrubTime(userId).lowerFromOption() -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoduckPlaceRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoduckPlaceRepository.docx new file mode 100644 index 000000000..d77247569 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoduckPlaceRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoduckPlaceRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoduckPlaceRepository.scala deleted file mode 100644 index 483f3f73f..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/GeoduckPlaceRepository.scala +++ /dev/null @@ -1,132 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.geoduck.common.{thriftscala => Geoduck} -import com.twitter.geoduck.service.thriftscala.GeoContext -import com.twitter.geoduck.service.thriftscala.Key -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.geoduck.util.service.GeoduckLocate -import com.twitter.geoduck.util.service.LocationResponseExtractors -import com.twitter.geoduck.util.{primitives => GDPrimitive} -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup -import com.twitter.tweetypie.{thriftscala => TP} - -object GeoduckPlaceConverter { - - def LocationResponseToTPPlace(lang: String, lr: LocationResponse): Option[TP.Place] = - GDPrimitive.Place - .fromLocationResponse(lr) - .headOption - .map(apply(lang, _)) - - def convertPlaceType(pt: Geoduck.PlaceType): TP.PlaceType = pt match { - case Geoduck.PlaceType.Unknown => TP.PlaceType.Unknown - case Geoduck.PlaceType.Country => TP.PlaceType.Country - case Geoduck.PlaceType.Admin => TP.PlaceType.Admin - case Geoduck.PlaceType.City => TP.PlaceType.City - case Geoduck.PlaceType.Neighborhood => TP.PlaceType.Neighborhood - case Geoduck.PlaceType.Poi => TP.PlaceType.Poi - case Geoduck.PlaceType.ZipCode => TP.PlaceType.Admin - case Geoduck.PlaceType.Metro => TP.PlaceType.Admin - case Geoduck.PlaceType.Admin0 => TP.PlaceType.Admin - case Geoduck.PlaceType.Admin1 => TP.PlaceType.Admin - case _ => - throw new IllegalStateException(s"Invalid place type: $pt") - } - - def convertPlaceName(gd: Geoduck.PlaceName): TP.PlaceName = - TP.PlaceName( - name = gd.name, - language = gd.language.getOrElse("en"), - `type` = convertPlaceNameType(gd.nameType), - preferred = gd.preferred - ) - - def convertPlaceNameType(pt: Geoduck.PlaceNameType): TP.PlaceNameType = pt match { - case Geoduck.PlaceNameType.Normal => TP.PlaceNameType.Normal - case Geoduck.PlaceNameType.Abbreviation => TP.PlaceNameType.Abbreviation - case Geoduck.PlaceNameType.Synonym => TP.PlaceNameType.Synonym - case _ => - throw new IllegalStateException(s"Invalid place name type: $pt") - } - - def convertAttributes(attrs: collection.Set[Geoduck.PlaceAttribute]): Map[String, String] = - attrs.map(attr => attr.key -> attr.value.getOrElse("")).toMap - - def convertBoundingBox(geom: GDPrimitive.Geometry): Seq[TP.GeoCoordinates] = - geom.coordinates.map { coord => - TP.GeoCoordinates( - latitude = coord.lat, - longitude = coord.lon - ) - } - - def apply(queryLang: String, geoplace: GDPrimitive.Place): TP.Place = { - val bestname = geoplace.bestName(queryLang).getOrElse(geoplace.hexId) - TP.Place( - id = geoplace.hexId, - `type` = convertPlaceType(geoplace.placeType), - name = bestname, - fullName = geoplace.fullName(queryLang).getOrElse(bestname), - attributes = convertAttributes(geoplace.attributes), - boundingBox = geoplace.boundingBox.map(convertBoundingBox), - countryCode = geoplace.countryCode, - containers = Some(geoplace.cone.map(_.hexId).toSet + geoplace.hexId), - countryName = geoplace.countryName(queryLang) - ) - } - - def convertGDKey(key: Key, lang: String): PlaceKey = { - val Key.PlaceId(pid) = key - PlaceKey("%016x".format(pid), lang) - } -} - -object GeoduckPlaceRepository { - val context: GeoContext = - GeoContext( - placeFields = Set( - Geoduck.PlaceQueryFields.Attributes, - Geoduck.PlaceQueryFields.BoundingBox, - Geoduck.PlaceQueryFields.PlaceNames, - Geoduck.PlaceQueryFields.Cone - ), - placeTypes = Set( - Geoduck.PlaceType.Country, - Geoduck.PlaceType.Admin0, - Geoduck.PlaceType.Admin1, - Geoduck.PlaceType.City, - Geoduck.PlaceType.Neighborhood - ), - includeCountryCode = true, - hydrateCone = true - ) - - def apply(geoduck: GeoduckLocate): PlaceRepository.Type = { - val geoduckGroup = LegacySeqGroup((ids: Seq[Key.PlaceId]) => geoduck(context, ids)) - - placeKey => - val placeId = - try { - Stitch.value( - Key.PlaceId(java.lang.Long.parseUnsignedLong(placeKey.placeId, 16)) - ) - } catch { - case _: NumberFormatException => Stitch.exception(NotFound) - } - - placeId - .flatMap(id => Stitch.call(id, geoduckGroup)) - .rescue { case LocationResponseExtractors.Failure(ex) => Stitch.exception(ex) } - .map { resp => - GDPrimitive.Place - .fromLocationResponse(resp) - .headOption - .map(GeoduckPlaceConverter(placeKey.language, _)) - } - .lowerFromOption() - } - -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/LastQuoteOfQuoterRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/LastQuoteOfQuoterRepository.docx new file mode 100644 index 000000000..6124beba0 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/LastQuoteOfQuoterRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/LastQuoteOfQuoterRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/LastQuoteOfQuoterRepository.scala deleted file mode 100644 index 9c853b85c..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/LastQuoteOfQuoterRepository.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.flockdb.client.QuoteTweetsIndexGraph -import com.twitter.flockdb.client.TFlockClient -import com.twitter.flockdb.client.UserTimelineGraph -import com.twitter.stitch.Stitch - -object LastQuoteOfQuoterRepository { - type Type = (TweetId, UserId) => Stitch[Boolean] - - def apply( - tflockReadClient: TFlockClient - ): Type = - (tweetId, userId) => { - // Select the tweets authored by userId quoting tweetId. - // By intersecting the tweet quotes with this user's tweets. - val quotesFromQuotingUser = QuoteTweetsIndexGraph - .from(tweetId) - .intersect(UserTimelineGraph.from(userId)) - - Stitch.callFuture(tflockReadClient.selectAll(quotesFromQuotingUser).map(_.size <= 1)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ManhattanTweetRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ManhattanTweetRepository.docx new file mode 100644 index 000000000..9daa76129 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ManhattanTweetRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ManhattanTweetRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ManhattanTweetRepository.scala deleted file mode 100644 index dd87a2e99..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ManhattanTweetRepository.scala +++ /dev/null @@ -1,147 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie -import com.twitter.tweetypie.client_id.ClientIdHelper -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.storage.TweetStorageClient.GetStoredTweet -import com.twitter.tweetypie.storage.TweetStorageClient.GetTweet -import com.twitter.tweetypie.storage._ -import scala.util.control.NoStackTrace - -case class StorageGetTweetFailure(tweetId: TweetId, underlying: Throwable) - extends Exception(s"tweetId=$tweetId", underlying) - with NoStackTrace - -object ManhattanTweetRepository { - private[this] val logger = Logger(getClass) - - def apply( - getTweet: TweetStorageClient.GetTweet, - getStoredTweet: TweetStorageClient.GetStoredTweet, - shortCircuitLikelyPartialTweetReads: Gate[Duration], - statsReceiver: StatsReceiver, - clientIdHelper: ClientIdHelper, - ): TweetResultRepository.Type = { - def likelyAvailable(tweetId: TweetId): Boolean = - if (SnowflakeId.isSnowflakeId(tweetId)) { - val tweetAge: Duration = Time.now.since(SnowflakeId(tweetId).time) - !shortCircuitLikelyPartialTweetReads(tweetAge) - } else { - true // Not a snowflake id, so should definitely be available - } - - val likelyPartialTweetReadsCounter = statsReceiver.counter("likely_partial_tweet_reads") - - (tweetId, options) => - if (!likelyAvailable(tweetId)) { - likelyPartialTweetReadsCounter.incr() - val currentClient = - clientIdHelper.effectiveClientId.getOrElse(ClientIdHelper.UnknownClientId) - logger.debug(s"likely_partial_tweet_read $tweetId $currentClient") - Stitch.exception(NotFound) - } else if (options.fetchStoredTweets) { - getStoredTweet(tweetId).liftToTry.flatMap(handleGetStoredTweetResponse(tweetId, _)) - } else { - getTweet(tweetId).liftToTry.flatMap(handleGetTweetResponse(tweetId, _)) - } - } - - private def handleGetTweetResponse( - tweetId: tweetypie.TweetId, - response: Try[GetTweet.Response] - ): Stitch[TweetResult] = { - response match { - case Return(GetTweet.Response.Found(tweet)) => - Stitch.value(TweetResult(TweetData(tweet = tweet), HydrationState.modified)) - case Return(GetTweet.Response.NotFound) => - Stitch.exception(NotFound) - case Return(GetTweet.Response.Deleted) => - Stitch.exception(FilteredState.Unavailable.TweetDeleted) - case Return(_: GetTweet.Response.BounceDeleted) => - Stitch.exception(FilteredState.Unavailable.BounceDeleted) - case Throw(_: storage.RateLimited) => - Stitch.exception(OverCapacity(s"Storage overcapacity, tweetId=$tweetId")) - case Throw(e) => - Stitch.exception(StorageGetTweetFailure(tweetId, e)) - } - } - - private def handleGetStoredTweetResponse( - tweetId: tweetypie.TweetId, - response: Try[GetStoredTweet.Response] - ): Stitch[TweetResult] = { - def translateErrors( - getStoredTweetErrs: Seq[GetStoredTweet.Error] - ): Seq[StoredTweetResult.Error] = { - getStoredTweetErrs.map { - case GetStoredTweet.Error.TweetIsCorrupt => StoredTweetResult.Error.Corrupt - case GetStoredTweet.Error.ScrubbedFieldsPresent => - StoredTweetResult.Error.ScrubbedFieldsPresent - case GetStoredTweet.Error.TweetFieldsMissingOrInvalid => - StoredTweetResult.Error.FieldsMissingOrInvalid - case GetStoredTweet.Error.TweetShouldBeHardDeleted => - StoredTweetResult.Error.ShouldBeHardDeleted - } - } - - def toTweetResult( - tweet: Tweet, - state: Option[TweetStateRecord], - errors: Seq[GetStoredTweet.Error] - ): TweetResult = { - val translatedErrors = translateErrors(errors) - val canHydrate: Boolean = - !translatedErrors.contains(StoredTweetResult.Error.Corrupt) && - !translatedErrors.contains(StoredTweetResult.Error.FieldsMissingOrInvalid) - - val storedTweetResult = state match { - case None => StoredTweetResult.Present(translatedErrors, canHydrate) - case Some(TweetStateRecord.HardDeleted(_, softDeletedAtMsec, hardDeletedAtMsec)) => - StoredTweetResult.HardDeleted(softDeletedAtMsec, hardDeletedAtMsec) - case Some(TweetStateRecord.SoftDeleted(_, softDeletedAtMsec)) => - StoredTweetResult.SoftDeleted(softDeletedAtMsec, translatedErrors, canHydrate) - case Some(TweetStateRecord.BounceDeleted(_, deletedAtMsec)) => - StoredTweetResult.BounceDeleted(deletedAtMsec, translatedErrors, canHydrate) - case Some(TweetStateRecord.Undeleted(_, undeletedAtMsec)) => - StoredTweetResult.Undeleted(undeletedAtMsec, translatedErrors, canHydrate) - case Some(TweetStateRecord.ForceAdded(_, addedAtMsec)) => - StoredTweetResult.ForceAdded(addedAtMsec, translatedErrors, canHydrate) - } - - TweetResult( - TweetData(tweet = tweet, storedTweetResult = Some(storedTweetResult)), - HydrationState.modified) - } - - val tweetResult = response match { - case Return(GetStoredTweet.Response.FoundAny(tweet, state, _, _, errors)) => - toTweetResult(tweet, state, errors) - case Return(GetStoredTweet.Response.Failed(tweetId, _, _, _, errors)) => - val tweetData = TweetData( - tweet = Tweet(tweetId), - storedTweetResult = Some(StoredTweetResult.Failed(translateErrors(errors)))) - TweetResult(tweetData, HydrationState.modified) - case Return(GetStoredTweet.Response.HardDeleted(tweetId, state, _, _)) => - toTweetResult(Tweet(tweetId), state, Seq()) - case Return(GetStoredTweet.Response.NotFound(tweetId)) => { - val tweetData = TweetData( - tweet = Tweet(tweetId), - storedTweetResult = Some(StoredTweetResult.NotFound) - ) - TweetResult(tweetData, HydrationState.modified) - } - case _ => { - val tweetData = TweetData( - tweet = Tweet(tweetId), - storedTweetResult = Some(StoredTweetResult.Failed(Seq(StoredTweetResult.Error.Corrupt)))) - TweetResult(tweetData, HydrationState.modified) - } - } - - Stitch.value(tweetResult) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/MediaMetadataRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/MediaMetadataRepository.docx new file mode 100644 index 000000000..e3774a1ea Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/MediaMetadataRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/MediaMetadataRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/MediaMetadataRepository.scala deleted file mode 100644 index f9aa5a832..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/MediaMetadataRepository.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.media.MediaMetadata -import com.twitter.tweetypie.media.MediaMetadataRequest - -object MediaMetadataRepository { - type Type = MediaMetadataRequest => Stitch[MediaMetadata] - - def apply(getMediaMetadata: FutureArrow[MediaMetadataRequest, MediaMetadata]): Type = { - // use an `SeqGroup` to group the future-calls together, even though they can be - // executed independently, in order to help keep hydration between different tweets - // in sync, to improve batching in hydrators which occur later in the pipeline. - val requestGroup = SeqGroup[MediaMetadataRequest, MediaMetadata] { - requests: Seq[MediaMetadataRequest] => - Future.collect(requests.map(r => getMediaMetadata(r).liftToTry)) - } - mediaReq => Stitch.call(mediaReq, requestGroup) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ParentUserIdRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ParentUserIdRepository.docx new file mode 100644 index 000000000..b5613b0c4 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ParentUserIdRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ParentUserIdRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ParentUserIdRepository.scala deleted file mode 100644 index 8c7092a53..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ParentUserIdRepository.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.FilteredState.Unavailable.BounceDeleted -import com.twitter.tweetypie.core.FilteredState.Unavailable.SourceTweetNotFound -import com.twitter.tweetypie.core.FilteredState.Unavailable.TweetDeleted - -object ParentUserIdRepository { - type Type = Tweet => Stitch[Option[UserId]] - - case class ParentTweetNotFound(tweetId: TweetId) extends Exception - - def apply(tweetRepo: TweetRepository.Type): Type = { - val options = TweetQuery.Options(TweetQuery.Include(Set(Tweet.CoreDataField.id))) - - tweet => - getShare(tweet) match { - case Some(share) if share.sourceStatusId == share.parentStatusId => - Stitch.value(Some(share.sourceUserId)) - case Some(share) => - tweetRepo(share.parentStatusId, options) - .map(tweet => Some(getUserId(tweet))) - .rescue { - case NotFound | TweetDeleted | BounceDeleted | SourceTweetNotFound(_) => - Stitch.exception(ParentTweetNotFound(share.parentStatusId)) - } - case None => - Stitch.None - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PastedMediaRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PastedMediaRepository.docx new file mode 100644 index 000000000..0fdaa3d1a Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PastedMediaRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PastedMediaRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PastedMediaRepository.scala deleted file mode 100644 index dd21e4ec1..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PastedMediaRepository.scala +++ /dev/null @@ -1,129 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core.FilteredState -import com.twitter.tweetypie.media.Media -import com.twitter.tweetypie.media.MediaUrl -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.util.MediaId -import java.nio.ByteBuffer - -case class PastedMedia(mediaEntities: Seq[MediaEntity], mediaTags: Map[MediaId, Seq[MediaTag]]) { - - /** - * Updates the copied media entities to have the same indices as the given UrlEntity. - */ - def updateEntities(urlEntity: UrlEntity): PastedMedia = - if (mediaEntities.isEmpty) this - else copy(mediaEntities = mediaEntities.map(Media.copyFromUrlEntity(_, urlEntity))) - - def merge(that: PastedMedia): PastedMedia = - PastedMedia( - mediaEntities = this.mediaEntities ++ that.mediaEntities, - mediaTags = this.mediaTags ++ that.mediaTags - ) - - /** - * Return a new PastedMedia that contains only the first maxMediaEntities media entities - */ - def take(maxMediaEntities: Int): PastedMedia = { - val entities = this.mediaEntities.take(maxMediaEntities) - val mediaIds = entities.map(_.mediaId) - val pastedTags = mediaTags.filterKeys { id => mediaIds.contains(id) } - - PastedMedia( - mediaEntities = entities, - mediaTags = pastedTags - ) - } - - def mergeTweetMediaTags(ownedTags: Option[TweetMediaTags]): Option[TweetMediaTags] = { - val merged = ownedTags.map(_.tagMap).getOrElse(Map.empty) ++ mediaTags - if (merged.nonEmpty) { - Some(TweetMediaTags(merged)) - } else { - None - } - } -} - -object PastedMedia { - import MediaUrl.Permalink.hasTweetId - - val empty: PastedMedia = PastedMedia(Nil, Map.empty) - - /** - * @param tweet: the tweet whose media URL was pasted. - * - * @return the media that should be copied to a tweet that has a - * link to the media in this tweet, along with its protection - * status. The returned media entities will have sourceStatusId - * and sourceUserId set appropriately for inclusion in a different - * tweet. - */ - def getMediaEntities(tweet: Tweet): Seq[MediaEntity] = - getMedia(tweet).collect { - case mediaEntity if hasTweetId(mediaEntity, tweet.id) => - setSource(mediaEntity, tweet.id, getUserId(tweet)) - } - - def setSource(mediaEntity: MediaEntity, tweetId: TweetId, userId: TweetId): MediaEntity = - mediaEntity.copy( - sourceStatusId = Some(tweetId), - sourceUserId = Some(mediaEntity.sourceUserId.getOrElse(userId)) - ) -} - -object PastedMediaRepository { - type Type = (TweetId, Ctx) => Stitch[PastedMedia] - - case class Ctx( - includeMediaEntities: Boolean, - includeAdditionalMetadata: Boolean, - includeMediaTags: Boolean, - extensionsArgs: Option[ByteBuffer], - safetyLevel: SafetyLevel) { - def asTweetQueryOptions: TweetQuery.Options = - TweetQuery.Options( - enforceVisibilityFiltering = true, - extensionsArgs = extensionsArgs, - safetyLevel = safetyLevel, - include = TweetQuery.Include( - tweetFields = - Set(Tweet.CoreDataField.id) ++ - (if (includeMediaEntities) Set(Tweet.MediaField.id) else Set.empty) ++ - (if (includeMediaTags) Set(Tweet.MediaTagsField.id) else Set.empty), - mediaFields = if (includeMediaEntities && includeAdditionalMetadata) { - Set(MediaEntity.AdditionalMetadataField.id) - } else { - Set.empty - }, - // don't recursively load pasted media - pastedMedia = false - ) - ) - } - - /** - * A Repository of PastedMedia fetched from other tweets. We query the tweet with - * default global visibility filtering enabled, so we won't see entities for users that - * are protected, deactivated, suspended, etc. - */ - def apply(tweetRepo: TweetRepository.Type): Type = - (tweetId, ctx) => - tweetRepo(tweetId, ctx.asTweetQueryOptions) - .flatMap { t => - val entities = PastedMedia.getMediaEntities(t) - if (entities.nonEmpty) { - Stitch.value(PastedMedia(entities, getMediaTagMap(t))) - } else { - Stitch.NotFound - } - } - .rescue { - // drop filtered tweets - case _: FilteredState => Stitch.NotFound - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PenguinLanguageRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PenguinLanguageRepository.docx new file mode 100644 index 000000000..ae3850eea Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PenguinLanguageRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PenguinLanguageRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PenguinLanguageRepository.scala deleted file mode 100644 index 26525ab6c..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PenguinLanguageRepository.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.ibm.icu.util.ULocale -import com.twitter.common.text.pipeline.TwitterLanguageIdentifier -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup -import com.twitter.tweetypie.repository.LanguageRepository.Text -import com.twitter.tweetypie.thriftscala._ -import com.twitter.util.FuturePool -import com.twitter.util.logging.Logger - -object LanguageRepository { - type Type = Text => Stitch[Option[Language]] - type Text = String -} - -object PenguinLanguageRepository { - private val identifier = new TwitterLanguageIdentifier.Builder().buildForTweet() - private val log = Logger(getClass) - - def isRightToLeft(lang: String): Boolean = - new ULocale(lang).getCharacterOrientation == "right-to-left" - - def apply(futurePool: FuturePool): LanguageRepository.Type = { - val identifyOne = - FutureArrow[Text, Option[Language]] { text => - futurePool { - try { - Some(identifier.identify(text)) - } catch { - case e: IllegalArgumentException => - val userId = TwitterContext().map(_.userId) - val encodedText = com.twitter.util.Base64StringEncoder.encode(text.getBytes) - log.info(s"${e.getMessage} : USER ID - $userId : TEXT - $encodedText") - None - } - }.map { - case Some(langWithScore) => - val lang = langWithScore.getLocale.getLanguage - Some( - Language( - language = lang, - rightToLeft = isRightToLeft(lang), - confidence = langWithScore.getScore - )) - case None => None - } - } - - text => Stitch.call(text, LegacySeqGroup(identifyOne.liftSeq)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PerspectiveRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PerspectiveRepository.docx new file mode 100644 index 000000000..c587b7ca2 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PerspectiveRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PerspectiveRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PerspectiveRepository.scala deleted file mode 100644 index ac609097a..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PerspectiveRepository.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives -import com.twitter.timelineservice.thriftscala.TimelineEntryPerspective - -object PerspectiveRepository { - - /** - * Same type as com.twitter.stitch.timelineservice.TimelineService.GetPerspectives but without - * using Arrow. - */ - type Type = GetPerspectives.Query => Stitch[TimelineEntryPerspective] -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PlaceRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PlaceRepository.docx new file mode 100644 index 000000000..de9ec9025 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PlaceRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PlaceRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PlaceRepository.scala deleted file mode 100644 index 8219eb350..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/PlaceRepository.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.servo.cache.ScopedCacheKey -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.thriftscala.Place - -case class PlaceKey(placeId: PlaceId, language: String) - extends ScopedCacheKey("t", "geo", 1, placeId, language) - -object PlaceRepository { - type Type = PlaceKey => Stitch[Place] -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ProfileGeoRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ProfileGeoRepository.docx new file mode 100644 index 000000000..affa7beff Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ProfileGeoRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ProfileGeoRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ProfileGeoRepository.scala deleted file mode 100644 index 6968c71c1..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/ProfileGeoRepository.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.dataproducts.enrichments.thriftscala._ -import com.twitter.gizmoduck.thriftscala.UserResponseState._ -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup -import com.twitter.tweetypie.backends.GnipEnricherator -import com.twitter.tweetypie.thriftscala.GeoCoordinates - -case class ProfileGeoKey(tweetId: TweetId, userId: Option[UserId], coords: Option[GeoCoordinates]) { - def key: TweetData = - TweetData( - tweetId = tweetId, - userId = userId, - coordinates = coords.map(ProfileGeoRepository.convertGeo) - ) -} - -object ProfileGeoRepository { - type Type = ProfileGeoKey => Stitch[ProfileGeoEnrichment] - - case class UnexpectedState(state: EnrichmentHydrationState) extends Exception(state.name) - - def convertGeo(coords: GeoCoordinates): TweetyPieGeoCoordinates = - TweetyPieGeoCoordinates( - latitude = coords.latitude, - longitude = coords.longitude, - geoPrecision = coords.geoPrecision, - display = coords.display - ) - - def apply(hydrateProfileGeo: GnipEnricherator.HydrateProfileGeo): Type = { - import EnrichmentHydrationState._ - - val emptyEnrichmentStitch = Stitch.value(ProfileGeoEnrichment()) - - val profileGeoGroup = SeqGroup[TweetData, ProfileGeoResponse] { keys: Seq[TweetData] => - // Gnip ignores writePath and treats all requests as reads - LegacySeqGroup.liftToSeqTry( - hydrateProfileGeo(ProfileGeoRequest(requests = keys, writePath = false)) - ) - } - - (geoKey: ProfileGeoKey) => - Stitch - .call(geoKey.key, profileGeoGroup) - .flatMap { - case ProfileGeoResponse(_, Success, Some(enrichment), _) => - Stitch.value(enrichment) - case ProfileGeoResponse(_, Success, None, _) => - // when state is Success enrichment should always be Some, but default to be safe - emptyEnrichmentStitch - case ProfileGeoResponse( - _, - UserLookupError, - _, - Some(DeactivatedUser | SuspendedUser | NotFound) - ) => - emptyEnrichmentStitch - case r => - Stitch.exception(UnexpectedState(r.state)) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuotedTweetVisibilityRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuotedTweetVisibilityRepository.docx new file mode 100644 index 000000000..c5bd84407 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuotedTweetVisibilityRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuotedTweetVisibilityRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuotedTweetVisibilityRepository.scala deleted file mode 100644 index ed8116476..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuotedTweetVisibilityRepository.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.spam.rtf.thriftscala.{SafetyLevel => ThriftSafetyLevel} -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository.VisibilityResultToFilteredState.toFilteredState -import com.twitter.visibility.configapi.configs.VisibilityDeciderGates -import com.twitter.visibility.interfaces.tweets.QuotedTweetVisibilityLibrary -import com.twitter.visibility.interfaces.tweets.QuotedTweetVisibilityRequest -import com.twitter.visibility.interfaces.tweets.TweetAndAuthor -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext - -/** - * This repository handles visibility filtering of inner quoted tweets - * based on relationships between the inner and outer tweets. This is - * additive to independent visibility filtering of the inner tweet. - */ -object QuotedTweetVisibilityRepository { - type Type = Request => Stitch[Option[FilteredState]] - - case class Request( - outerTweetId: TweetId, - outerAuthorId: UserId, - innerTweetId: TweetId, - innerAuthorId: UserId, - viewerId: Option[UserId], - safetyLevel: ThriftSafetyLevel) - - def apply( - quotedTweetVisibilityLibrary: QuotedTweetVisibilityLibrary.Type, - visibilityDeciderGates: VisibilityDeciderGates, - ): QuotedTweetVisibilityRepository.Type = { request: Request => - quotedTweetVisibilityLibrary( - QuotedTweetVisibilityRequest( - quotedTweet = TweetAndAuthor(request.innerTweetId, request.innerAuthorId), - outerTweet = TweetAndAuthor(request.outerTweetId, request.outerAuthorId), - ViewerContext.fromContextWithViewerIdFallback(request.viewerId), - safetyLevel = SafetyLevel.fromThrift(request.safetyLevel) - ) - ).map(visibilityResult => - toFilteredState( - visibilityResult = visibilityResult, - disableLegacyInterstitialFilteredReason = - visibilityDeciderGates.disableLegacyInterstitialFilteredReason())) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuoterHasAlreadyQuotedRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuoterHasAlreadyQuotedRepository.docx new file mode 100644 index 000000000..cfb8dae07 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuoterHasAlreadyQuotedRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuoterHasAlreadyQuotedRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuoterHasAlreadyQuotedRepository.scala deleted file mode 100644 index 7de373848..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/QuoterHasAlreadyQuotedRepository.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.flockdb.client.QuotersGraph -import com.twitter.flockdb.client.TFlockClient -import com.twitter.stitch.Stitch - -object QuoterHasAlreadyQuotedRepository { - type Type = (TweetId, UserId) => Stitch[Boolean] - - def apply( - tflockReadClient: TFlockClient - ): Type = - (tweetId, userId) => Stitch.callFuture(tflockReadClient.contains(QuotersGraph, tweetId, userId)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RelationshipRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RelationshipRepository.docx new file mode 100644 index 000000000..7f3597b5a Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RelationshipRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RelationshipRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RelationshipRepository.scala deleted file mode 100644 index 9b6304b4a..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RelationshipRepository.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.servo.util.FutureArrow -import com.twitter.socialgraph.thriftscala._ -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup - -object RelationshipKey { - def blocks(sourceId: UserId, destinationId: UserId): RelationshipKey = - RelationshipKey(sourceId, destinationId, RelationshipType.Blocking) - - def follows(sourceId: UserId, destinationId: UserId): RelationshipKey = - RelationshipKey(sourceId, destinationId, RelationshipType.Following) - - def mutes(sourceId: UserId, destinationId: UserId): RelationshipKey = - RelationshipKey(sourceId, destinationId, RelationshipType.Muting) - - def reported(sourceId: UserId, destinationId: UserId): RelationshipKey = - RelationshipKey(sourceId, destinationId, RelationshipType.ReportedAsSpam) -} - -case class RelationshipKey( - sourceId: UserId, - destinationId: UserId, - relationship: RelationshipType) { - def asExistsRequest: ExistsRequest = - ExistsRequest( - source = sourceId, - target = destinationId, - relationships = Seq(Relationship(relationship)) - ) -} - -object RelationshipRepository { - type Type = RelationshipKey => Stitch[Boolean] - - def apply( - exists: FutureArrow[(Seq[ExistsRequest], Option[RequestContext]), Seq[ExistsResult]], - maxRequestSize: Int - ): Type = { - val relationshipGroup: SeqGroup[RelationshipKey, Boolean] = - new SeqGroup[RelationshipKey, Boolean] { - override def run(keys: Seq[RelationshipKey]): Future[Seq[Try[Boolean]]] = - LegacySeqGroup.liftToSeqTry( - exists((keys.map(_.asExistsRequest), None)).map(_.map(_.exists))) - override val maxSize: Int = maxRequestSize - } - - relationshipKey => Stitch.call(relationshipKey, relationshipGroup) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RetweetSpamCheckRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RetweetSpamCheckRepository.docx new file mode 100644 index 000000000..b37524261 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RetweetSpamCheckRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RetweetSpamCheckRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RetweetSpamCheckRepository.scala deleted file mode 100644 index 610f3f3c4..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/RetweetSpamCheckRepository.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.service.gen.scarecrow.{thriftscala => scarecrow} -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.backends.Scarecrow - -object RetweetSpamCheckRepository { - type Type = scarecrow.Retweet => Stitch[scarecrow.TieredAction] - - def apply(checkRetweet: Scarecrow.CheckRetweet): Type = - retweet => Stitch.callFuture(checkRetweet(retweet)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StitchLockingCache.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StitchLockingCache.docx new file mode 100644 index 000000000..9531d382c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StitchLockingCache.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StitchLockingCache.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StitchLockingCache.scala deleted file mode 100644 index 7808d465f..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StitchLockingCache.scala +++ /dev/null @@ -1,161 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.servo.cache.{CachedValueStatus => Status, LockingCache => KVLockingCache, _} -import com.twitter.servo.repository.{CachedResult => Result} -import com.twitter.stitch.MapGroup -import com.twitter.stitch.Group -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Time -import com.twitter.util.Try - -/** - * Adapts a key-value locking cache to Arrow and - * normalizes the results to `CachedResult`. - */ -trait StitchLockingCache[K, V] { - val get: K => Stitch[Result[K, V]] - val lockAndSet: (K, StitchLockingCache.Val[V]) => Stitch[Unit] - val delete: K => Stitch[Boolean] -} - -object StitchLockingCache { - - /** - * Value intended to be written back to cache using lockAndSet. - * - * Note that only a subset of CachedValueStatus values are eligible for writing: - * Found, NotFound, and Deleted - */ - sealed trait Val[+V] - object Val { - case class Found[V](value: V) extends Val[V] - case object NotFound extends Val[Nothing] - case object Deleted extends Val[Nothing] - } - - /** - * A Group for batching get requests to a [[KVLockingCache]]. - */ - private case class GetGroup[K, V](cache: KVLockingCache[K, Cached[V]], override val maxSize: Int) - extends MapGroup[K, Result[K, V]] { - - private[this] def cachedToResult(key: K, cached: Cached[V]): Try[Result[K, V]] = - cached.status match { - case Status.NotFound => Return(Result.CachedNotFound(key, cached.cachedAt)) - case Status.Deleted => Return(Result.CachedDeleted(key, cached.cachedAt)) - case Status.SerializationFailed => Return(Result.SerializationFailed(key)) - case Status.DeserializationFailed => Return(Result.DeserializationFailed(key)) - case Status.Evicted => Return(Result.NotFound(key)) - case Status.DoNotCache => Return(Result.DoNotCache(key, cached.doNotCacheUntil)) - case Status.Found => - cached.value match { - case None => Return(Result.NotFound(key)) - case Some(value) => Return(Result.CachedFound(key, value, cached.cachedAt)) - } - case _ => Throw(new UnsupportedOperationException) - } - - override protected def run(keys: Seq[K]): Future[K => Try[Result[K, V]]] = - cache.get(keys).map { (result: KeyValueResult[K, Cached[V]]) => key => - result.found.get(key) match { - case Some(cached) => cachedToResult(key, cached) - case None => - result.failed.get(key) match { - case Some(t) => Return(Result.Failed(key, t)) - case None => Return(Result.NotFound(key)) - } - } - } - } - - /** - * Used in the implementation of LockAndSetGroup. This is just a - * glorified tuple with special equality semantics where calls with - * the same key will compare equal. MapGroup will use this as a key - * in a Map, which will prevent duplicate lockAndSet calls with the - * same key. We don't care which one we use - */ - private class LockAndSetCall[K, V](val key: K, val value: V) { - override def equals(other: Any): Boolean = - other match { - case call: LockAndSetCall[_, _] => call.key == key - case _ => false - } - - override def hashCode(): Int = key.hashCode - } - - /** - * A Group for `lockAndSet` calls to a [[KVLockingCache]]. This is - * necessary to avoid writing back a key multiple times if it is - * appears more than once in a batch. LockAndSetCall considers two - * calls equal even if the values differ because multiple lockAndSet - * calls for the same key will eventually result in only one being - * chosen by the cache anyway, and this avoids conflicting - * lockAndSet calls. - * - * For example, consider a tweet that mentions @jack twice - * when @jack is not in cache. That will result in two queries to - * load @jack, which will be deduped by the Group when the repo is - * called. Despite the fact that it is loaded only once, each of the - * two loads is oblivious to the other, so each of them attempts to - * write the value back to cache, resulting in two `lockAndSet` - * calls for @jack, so we have to dedupe them again. - */ - private case class LockAndSetGroup[K, V]( - cache: KVLockingCache[K, V], - picker: KVLockingCache.Picker[V]) - extends MapGroup[LockAndSetCall[K, V], Option[V]] { - - override def run( - calls: Seq[LockAndSetCall[K, V]] - ): Future[LockAndSetCall[K, V] => Try[Option[V]]] = - Future - .collect { - calls.map { call => - // This is masked to prevent interrupts to the overall - // request from interrupting writes back to cache. - cache - .lockAndSet(call.key, KVLockingCache.PickingHandler(call.value, picker)) - .masked - .liftToTry - } - } - .map(responses => calls.zip(responses).toMap) - } - - def apply[K, V]( - underlying: KVLockingCache[K, Cached[V]], - picker: KVLockingCache.Picker[Cached[V]], - maxRequestSize: Int = Int.MaxValue - ): StitchLockingCache[K, V] = - new StitchLockingCache[K, V] { - override val get: K => Stitch[Result[K, V]] = { - val group: Group[K, Result[K, V]] = GetGroup(underlying, maxRequestSize) - - (key: K) => Stitch.call(key, group) - } - - override val lockAndSet: (K, Val[V]) => Stitch[Unit] = { - val group = LockAndSetGroup(underlying, picker) - - (key: K, value: Val[V]) => { - val now = Time.now - val cached: Cached[V] = - value match { - case Val.Found(v) => Cached[V](Some(v), Status.Found, now, Some(now)) - case Val.NotFound => Cached[V](None, Status.NotFound, now, Some(now)) - case Val.Deleted => Cached[V](None, Status.Deleted, now, Some(now)) - } - - Stitch.call(new LockAndSetCall(key, cached), group).unit - } - } - - override val delete: K => Stitch[Boolean] = - (key: K) => Stitch.callFuture(underlying.delete(key)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityAccessRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityAccessRepository.docx new file mode 100644 index 000000000..17d6e0882 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityAccessRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityAccessRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityAccessRepository.scala deleted file mode 100644 index 2658446a3..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityAccessRepository.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.CommunityId -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} - -object StratoCommunityAccessRepository { - type Type = CommunityId => Stitch[Option[CommunityAccess]] - - sealed trait CommunityAccess - object CommunityAccess { - case object Public extends CommunityAccess - case object Closed extends CommunityAccess - case object Private extends CommunityAccess - } - - val column = "communities/access.Community" - - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[CommunityId, Unit, CommunityAccess] = - client.fetcher[CommunityId, CommunityAccess](column) - - communityId => fetcher.fetch(communityId).map(_.v) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityMembershipRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityMembershipRepository.docx new file mode 100644 index 000000000..3c8a66fc8 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityMembershipRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityMembershipRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityMembershipRepository.scala deleted file mode 100644 index cfeb070c1..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoCommunityMembershipRepository.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.CommunityId -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} - -object StratoCommunityMembershipRepository { - type Type = CommunityId => Stitch[Boolean] - - val column = "communities/isMember.Community" - - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[CommunityId, Unit, Boolean] = - client.fetcher[CommunityId, Boolean](column) - - communityId => fetcher.fetch(communityId).map(_.v.getOrElse(false)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoPromotedTweetRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoPromotedTweetRepository.docx new file mode 100644 index 000000000..25a606fd5 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoPromotedTweetRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoPromotedTweetRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoPromotedTweetRepository.scala deleted file mode 100644 index 8c510e533..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoPromotedTweetRepository.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.tweetypie.TweetId -import com.twitter.strato.client.{Client => StratoClient} - -object StratoPromotedTweetRepository { - type Type = TweetId => Stitch[Boolean] - - val column = "tweetypie/isPromoted.Tweet" - - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[TweetId, Unit, Boolean] = - client.fetcher[TweetId, Boolean](column) - - tweetId => fetcher.fetch(tweetId).map(f => f.v.getOrElse(false)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSafetyLabelsRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSafetyLabelsRepository.docx new file mode 100644 index 000000000..2c57ac5f3 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSafetyLabelsRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSafetyLabelsRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSafetyLabelsRepository.scala deleted file mode 100644 index 68f537fce..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSafetyLabelsRepository.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.search.blender.services.strato.UserSearchSafetySettings -import com.twitter.spam.rtf.thriftscala.SafetyLabel -import com.twitter.spam.rtf.thriftscala.SafetyLabelMap -import com.twitter.spam.rtf.thriftscala.SafetyLabelType -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.strato.thrift.ScroogeConvImplicits._ -import com.twitter.visibility.common.UserSearchSafetySource - -object StratoSafetyLabelsRepository { - type Type = (TweetId, SafetyLabelType) => Stitch[Option[SafetyLabel]] - - def apply(client: StratoClient): Type = { - val safetyLabelMapRepo = StratoSafetyLabelMapRepository(client) - - (tweetId, safetyLabelType) => - safetyLabelMapRepo(tweetId).map( - _.flatMap(_.labels).flatMap(_.get(safetyLabelType)) - ) - } -} - -object StratoSafetyLabelMapRepository { - type Type = TweetId => Stitch[Option[SafetyLabelMap]] - - val column = "visibility/baseTweetSafetyLabelMap" - - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[TweetId, Unit, SafetyLabelMap] = - client.fetcher[TweetId, SafetyLabelMap](column) - - tweetId => fetcher.fetch(tweetId).map(_.v) - } -} - -object StratoUserSearchSafetySourceRepository { - type Type = UserId => Stitch[UserSearchSafetySettings] - - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[UserId, Unit, UserSearchSafetySettings] = - client.fetcher[UserId, UserSearchSafetySettings](UserSearchSafetySource.Column) - - userId => fetcher.fetch(userId).map(_.v.getOrElse(UserSearchSafetySource.DefaultSetting)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSubscriptionVerificationRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSubscriptionVerificationRepository.docx new file mode 100644 index 000000000..5b5b960b1 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSubscriptionVerificationRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSubscriptionVerificationRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSubscriptionVerificationRepository.scala deleted file mode 100644 index 1fb825c6b..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSubscriptionVerificationRepository.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.tweetypie.UserId -import com.twitter.strato.client.{Client => StratoClient} - -object StratoSubscriptionVerificationRepository { - type Type = (UserId, String) => Stitch[Boolean] - - val column = "subscription-services/subscription-verification/cacheProtectedHasAccess.User" - - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[UserId, Seq[String], Seq[String]] = - client.fetcher[UserId, Seq[String], Seq[String]](column) - - (userId, resource) => fetcher.fetch(userId, Seq(resource)).map(f => f.v.nonEmpty) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowEligibleRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowEligibleRepository.docx new file mode 100644 index 000000000..dea31171e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowEligibleRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowEligibleRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowEligibleRepository.scala deleted file mode 100644 index e86352c37..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowEligibleRepository.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.tweetypie.UserId - -object StratoSuperFollowEligibleRepository { - type Type = UserId => Stitch[Boolean] - - val column = "audiencerewards/audienceRewardsService/getSuperFollowEligibility.User" - - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[UserId, Unit, Boolean] = - client.fetcher[UserId, Boolean](column) - - userId => fetcher.fetch(userId).map(_.v.getOrElse(false)) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowRelationsRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowRelationsRepository.docx new file mode 100644 index 000000000..b441f9127 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowRelationsRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowRelationsRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowRelationsRepository.scala deleted file mode 100644 index e6fa65268..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/StratoSuperFollowRelationsRepository.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.tweetypie.Future -import com.twitter.tweetypie.UserId -import com.twitter.tweetypie.core.TweetCreateFailure -import com.twitter.tweetypie.thriftscala.ExclusiveTweetControl -import com.twitter.tweetypie.thriftscala.TweetCreateState - -object StratoSuperFollowRelationsRepository { - type Type = (UserId, UserId) => Stitch[Boolean] - - def apply(client: StratoClient): Type = { - - val column = "audiencerewards/superFollows/hasSuperFollowingRelationshipV2" - - val fetcher: Fetcher[HasSuperFollowingRelationshipRequest, Unit, Boolean] = - client.fetcher[HasSuperFollowingRelationshipRequest, Boolean](column) - - (authorId, viewerId) => { - // Owner of an exclusive tweet chain can respond to their own - // tweets / replies, despite not super following themselves - if (authorId == viewerId) { - Stitch.True - } else { - val key = HasSuperFollowingRelationshipRequest(authorId, viewerId) - // The default relation for this column is "missing", aka None. - // This needs to be mapped to false since Super Follows are a sparse relation. - fetcher.fetch(key).map(_.v.getOrElse(false)) - } - } - } - - object Validate { - def apply( - exclusiveTweetControl: Option[ExclusiveTweetControl], - userId: UserId, - superFollowRelationsRepo: StratoSuperFollowRelationsRepository.Type - ): Future[Unit] = { - Stitch - .run { - exclusiveTweetControl.map(_.conversationAuthorId) match { - // Don't do exclusive tweet validation on non exclusive tweets. - case None => - Stitch.value(true) - - case Some(convoAuthorId) => - superFollowRelationsRepo(userId, convoAuthorId) - } - }.map { - case true => Future.Unit - case false => - Future.exception(TweetCreateFailure.State(TweetCreateState.SourceTweetNotFound)) - }.flatten - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetCountsRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetCountsRepository.docx new file mode 100644 index 000000000..7d5511536 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetCountsRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetCountsRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetCountsRepository.scala deleted file mode 100644 index 82bbd2930..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetCountsRepository.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.flockdb.client._ -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup - -sealed trait TweetCountKey { - // The flockdb Select used to calculate the count from TFlock - def toSelect: Select[StatusGraph] - - // The Tweet id for this count - def tweetId: TweetId - - // com.twitter.servo.cache.MemcacheCache calls toString to turn this key into a cache key - def toString: String -} - -case class RetweetsKey(tweetId: TweetId) extends TweetCountKey { - lazy val toSelect: Select[StatusGraph] = RetweetsGraph.from(tweetId) - override lazy val toString: String = "cnts:rt:" + tweetId -} - -case class RepliesKey(tweetId: TweetId) extends TweetCountKey { - lazy val toSelect: Select[StatusGraph] = RepliesToTweetsGraph.from(tweetId) - override lazy val toString: String = "cnts:re:" + tweetId -} - -case class FavsKey(tweetId: TweetId) extends TweetCountKey { - lazy val toSelect: Select[StatusGraph] = FavoritesGraph.to(tweetId) - override lazy val toString: String = "cnts:fv:" + tweetId -} - -case class QuotesKey(tweetId: TweetId) extends TweetCountKey { - lazy val toSelect: Select[StatusGraph] = QuotersGraph.from(tweetId) - override lazy val toString: String = "cnts:qt:" + tweetId -} - -case class BookmarksKey(tweetId: TweetId) extends TweetCountKey { - lazy val toSelect: Select[StatusGraph] = BookmarksGraph.to(tweetId) - override lazy val toString: String = "cnts:bm:" + tweetId -} - -object TweetCountsRepository { - type Type = TweetCountKey => Stitch[Count] - - def apply(tflock: TFlockClient, maxRequestSize: Int): Type = { - object RequestGroup extends SeqGroup[TweetCountKey, Count] { - override def run(keys: Seq[TweetCountKey]): Future[Seq[Try[MediaId]]] = { - val selects = MultiSelect[StatusGraph]() ++= keys.map(_.toSelect) - LegacySeqGroup.liftToSeqTry(tflock.multiCount(selects).map(counts => counts.map(_.toLong))) - } - override val maxSize: Int = maxRequestSize - } - - key => Stitch.call(key, RequestGroup) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetQuery.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetQuery.docx new file mode 100644 index 000000000..53d91d311 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetQuery.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetQuery.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetQuery.scala deleted file mode 100644 index efbd5b61f..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetQuery.scala +++ /dev/null @@ -1,147 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import java.nio.ByteBuffer - -object TweetQuery { - - /** - * Parent trait that indicates what triggered the tweet query. - */ - sealed trait Cause { - import Cause._ - - /** - * Is the tweet query hydrating the specified tweet for the purposes of a write? - */ - def writing(tweetId: TweetId): Boolean = - this match { - case w: Write if w.tweetId == tweetId => true - case _ => false - } - - /** - * Is the tweet query performing a regular read for any tweet? If the cause is - * a write on a different tweet, then any other tweet that is read in support of the write - * is considered a normal read, and is subject to read-path hydration. - */ - def reading(tweetId: TweetId): Boolean = - !writing(tweetId) - - /** - * Are we performing an insert after create on the specified tweet? An undelete operation - * performs an insert, but is not considered an initial insert. - */ - def initialInsert(tweetId: TweetId): Boolean = - this match { - case Insert(`tweetId`) => true - case _ => false - } - } - - object Cause { - case object Read extends Cause - trait Write extends Cause { - val tweetId: TweetId - } - case class Insert(tweetId: TweetId) extends Write - case class Undelete(tweetId: TweetId) extends Write - } - - /** - * Options for TweetQuery. - * - * @param include indicates which optionally hydrated fields on each tweet should be - * hydrated and included. - * @param enforceVisibilityFiltering whether Tweetypie visibility hydrators should be run to - * filter protected tweets, blocked quote tweets, contributor data, etc. This does not affect - * Visibility Library (http://go/vf) based filtering. - * @param cause indicates what triggered the read: a normal read, or a write operation. - * @param forExternalConsumption when true, the tweet is being read for rendering to an external - * client such as the iPhone Twitter app and is subject to being Dropped to prevent serving - * "bad" text to clients that might crash their OS. When false, the tweet is being read for internal - * non-client purposes and should never be Dropped. - * @param isInnerQuotedTweet Set by [[com.twitter.tweetypie.hydrator.QuotedTweetHydrator]], - * to be used by [[com.twitter.visibility.interfaces.tweets.TweetVisibilityLibrary]] - * so VisibilityFiltering library can execute Interstitial logic on inner quoted tweets. - * @param fetchStoredTweets Set by GetStoredTweetsHandler. If set to true, the Manhattan storage - * layer will fetch and construct Tweets regardless of what state they're in. - */ - case class Options( - include: TweetQuery.Include, - cacheControl: CacheControl = CacheControl.ReadWriteCache, - cardsPlatformKey: Option[String] = None, - excludeReported: Boolean = false, - enforceVisibilityFiltering: Boolean = false, - safetyLevel: SafetyLevel = SafetyLevel.FilterNone, - forUserId: Option[UserId] = None, - languageTag: String = "en", - extensionsArgs: Option[ByteBuffer] = None, - cause: Cause = Cause.Read, - scrubUnrequestedFields: Boolean = true, - requireSourceTweet: Boolean = true, - forExternalConsumption: Boolean = false, - simpleQuotedTweet: Boolean = false, - isInnerQuotedTweet: Boolean = false, - fetchStoredTweets: Boolean = false, - isSourceTweet: Boolean = false, - enableEditControlHydration: Boolean = true) - - case class Include( - tweetFields: Set[FieldId] = Set.empty, - countsFields: Set[FieldId] = Set.empty, - mediaFields: Set[FieldId] = Set.empty, - quotedTweet: Boolean = false, - pastedMedia: Boolean = false) { - - /** - * Accumulates additional (rather than replaces) field ids. - */ - def also( - tweetFields: Traversable[FieldId] = Nil, - countsFields: Traversable[FieldId] = Nil, - mediaFields: Traversable[FieldId] = Nil, - quotedTweet: Option[Boolean] = None, - pastedMedia: Option[Boolean] = None - ): Include = - copy( - tweetFields = this.tweetFields ++ tweetFields, - countsFields = this.countsFields ++ countsFields, - mediaFields = this.mediaFields ++ mediaFields, - quotedTweet = quotedTweet.getOrElse(this.quotedTweet), - pastedMedia = pastedMedia.getOrElse(this.pastedMedia) - ) - - /** - * Removes field ids. - */ - def exclude( - tweetFields: Traversable[FieldId] = Nil, - countsFields: Traversable[FieldId] = Nil, - mediaFields: Traversable[FieldId] = Nil - ): Include = - copy( - tweetFields = this.tweetFields -- tweetFields, - countsFields = this.countsFields -- countsFields, - mediaFields = this.mediaFields -- mediaFields - ) - - def ++(that: Include): Include = - copy( - tweetFields = this.tweetFields ++ that.tweetFields, - countsFields = this.countsFields ++ that.countsFields, - mediaFields = this.mediaFields ++ that.mediaFields, - quotedTweet = this.quotedTweet || that.quotedTweet, - pastedMedia = this.pastedMedia || that.pastedMedia - ) - } -} - -sealed case class CacheControl(writeToCache: Boolean, readFromCache: Boolean) - -object CacheControl { - val NoCache: CacheControl = CacheControl(false, false) - val ReadOnlyCache: CacheControl = CacheControl(false, true) - val ReadWriteCache: CacheControl = CacheControl(true, true) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetRepository.docx new file mode 100644 index 000000000..a8431ea11 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetRepository.scala deleted file mode 100644 index f0f24fafa..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetRepository.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ - -object TweetRepository { - type Type = (TweetId, TweetQuery.Options) => Stitch[Tweet] - type Optional = (TweetId, TweetQuery.Options) => Stitch[Option[Tweet]] - - def tweetGetter(repo: Optional, opts: TweetQuery.Options): FutureArrow[TweetId, Option[Tweet]] = - FutureArrow(tweetId => Stitch.run(repo(tweetId, opts))) - - def tweetGetter(repo: Optional): FutureArrow[(TweetId, TweetQuery.Options), Option[Tweet]] = - FutureArrow { case (tweetId, opts) => Stitch.run(repo(tweetId, opts)) } - - /** - * Converts a `TweetResultRepository.Type`-typed repo to an `TweetRepository.Type`-typed repo. - */ - def fromTweetResult(repo: TweetResultRepository.Type): Type = - (tweetId, options) => repo(tweetId, options).map(_.value.tweet) - - /** - * Converts a `Type`-typed repo to an `Optional`-typed - * repo, where NotFound or filtered tweets are returned as `None`. - */ - def optional(repo: Type): Optional = - (tweetId, options) => - repo(tweetId, options).liftToOption { case NotFound | (_: FilteredState) => true } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetResultRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetResultRepository.docx new file mode 100644 index 000000000..0129d67ea Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetResultRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetResultRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetResultRepository.scala deleted file mode 100644 index 2e8f50ffd..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetResultRepository.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.TweetId -import com.twitter.tweetypie.core.TweetResult - -object TweetResultRepository { - type Type = (TweetId, TweetQuery.Options) => Stitch[TweetResult] - - /** - * Short-circuits the request of invalid tweet ids (`<= 0`) by immediately throwing `NotFound`. - */ - def shortCircuitInvalidIds(repo: Type): Type = { - case (tweetId, _) if tweetId <= 0 => Stitch.NotFound - case (tweetId, options) => repo(tweetId, options) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetSpamCheckRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetSpamCheckRepository.docx new file mode 100644 index 000000000..e41b9a03b Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetSpamCheckRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetSpamCheckRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetSpamCheckRepository.scala deleted file mode 100644 index 98b3f5e47..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetSpamCheckRepository.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.service.gen.scarecrow.{thriftscala => scarecrow} -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.backends.Scarecrow - -object TweetSpamCheckRepository { - - type Type = (scarecrow.TweetNew, scarecrow.TweetContext) => Stitch[scarecrow.CheckTweetResponse] - - def apply(checkTweet: Scarecrow.CheckTweet2): Type = - (tweetNew, tweetContext) => Stitch.callFuture(checkTweet((tweetNew, tweetContext))) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetVisibilityRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetVisibilityRepository.docx new file mode 100644 index 000000000..708437ecf Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetVisibilityRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetVisibilityRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetVisibilityRepository.scala deleted file mode 100644 index f0017b2fd..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/TweetVisibilityRepository.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.logging.Logger -import com.twitter.spam.rtf.thriftscala.{SafetyLevel => ThriftSafetyLevel} -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.repository.VisibilityResultToFilteredState.toFilteredState -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.visibility.configapi.configs.VisibilityDeciderGates -import com.twitter.visibility.interfaces.tweets.TweetVisibilityLibrary -import com.twitter.visibility.interfaces.tweets.TweetVisibilityRequest -import com.twitter.visibility.models.SafetyLevel.DeprecatedSafetyLevel -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.ViewerContext - -/** - * This repository handles visibility filtering of tweets - * - * i.e. deciding whether to drop/suppress tweets based on viewer - * and safety level for instance. Rules in VF library can be thought as: - * - * (SafetyLevel)(Viewer, Content, Features) => Action - * - * SafetyLevel represents the product context in which the Viewer is - * requesting to view the Content. Example: TimelineHome, TweetDetail, - * Recommendations, Notifications - * - * Content here is mainly tweets (can be users, notifications, cards etc) - * - * Features might include safety labels and other metadata of a Tweet, - * flags set on a User (including the Viewer), relationships between Users - * (e.g. block, follow), relationships between Users and Content - * (e.g. reported for spam) - * - * We initialize VisibilityLibrary using UserSource and UserRelationshipSource: - * Stitch interfaces that provide methods to retrieve user and relationship - * information in Gizmoduck and SocialGraph repositories, respectively. - * This user and relationship info along with Tweet labels, provide necessary - * features to take a filtering decision. - * - * Actions supported in Tweetypie right now are Drop and Suppress. - * In the future, we might want to surface other granular actions such as - * Tombstone and Downrank which are supported in VF lib. - * - * The TweetVisibilityRepository has the following format: - * - * Request(Tweet, Option[SafetyLevel], Option[UserId]) => Stitch[Option[FilteredState]] - * - * SafetyLevel is plumbed from the tweet query options. - * - * In addition to the latency stats and rpc counts from VF library, we also capture - * unsupported and deprecated safety level stats here to inform the relevant clients. - * - * go/visibilityfiltering, go/visibilityfilteringdocs - * - */ -object TweetVisibilityRepository { - type Type = Request => Stitch[Option[FilteredState]] - - case class Request( - tweet: Tweet, - viewerId: Option[UserId], - safetyLevel: ThriftSafetyLevel, - isInnerQuotedTweet: Boolean, - isRetweet: Boolean, - hydrateConversationControl: Boolean, - isSourceTweet: Boolean) - - def apply( - visibilityLibrary: TweetVisibilityLibrary.Type, - visibilityDeciderGates: VisibilityDeciderGates, - log: Logger, - statsReceiver: StatsReceiver - ): TweetVisibilityRepository.Type = { - - val noTweetRulesCounter = statsReceiver.counter("no_tweet_rules_requests") - val deprecatedScope = statsReceiver.scope("deprecated_safety_level") - - request: Request => - SafetyLevel.fromThrift(request.safetyLevel) match { - case DeprecatedSafetyLevel => - deprecatedScope.counter(request.safetyLevel.name.toLowerCase()).incr() - log.warning("Deprecated SafetyLevel (%s) requested".format(request.safetyLevel.name)) - Stitch.None - case safetyLevel: SafetyLevel => - if (!TweetVisibilityLibrary.hasTweetRules(safetyLevel)) { - noTweetRulesCounter.incr() - Stitch.None - } else { - visibilityLibrary( - TweetVisibilityRequest( - tweet = request.tweet, - safetyLevel = safetyLevel, - viewerContext = ViewerContext.fromContextWithViewerIdFallback(request.viewerId), - isInnerQuotedTweet = request.isInnerQuotedTweet, - isRetweet = request.isRetweet, - hydrateConversationControl = request.hydrateConversationControl, - isSourceTweet = request.isSourceTweet - ) - ).map(visibilityResult => - toFilteredState( - visibilityResult = visibilityResult, - disableLegacyInterstitialFilteredReason = - visibilityDeciderGates.disableLegacyInterstitialFilteredReason())) - } - } - } - - /** - * We can skip visibility filtering when any of the following is true: - * - * - SafetyLevel is deprecated - * - SafetyLevel has no tweet rules - */ - def canSkipVisibilityFiltering(thriftSafetyLevel: ThriftSafetyLevel): Boolean = - SafetyLevel.fromThrift(thriftSafetyLevel) match { - case DeprecatedSafetyLevel => - true - case safetyLevel: SafetyLevel => - !TweetVisibilityLibrary.hasTweetRules(safetyLevel) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionInfoRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionInfoRepository.docx new file mode 100644 index 000000000..7c743d95f Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionInfoRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionInfoRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionInfoRepository.scala deleted file mode 100644 index c7165f95b..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionInfoRepository.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.consumer_privacy.mention_controls.thriftscala.UnmentionInfo -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.strato.thrift.ScroogeConvImplicits._ - -object UnmentionInfoRepository { - type Type = Tweet => Stitch[Option[UnmentionInfo]] - - val column = "consumer-privacy/mentions-management/unmentionInfoFromTweet" - case class UnmentionInfoView(asViewer: Option[Long]) - - /** - * Creates a function that extracts users fields from a tweet and checks - * if the extracted users have been unmentioned from the tweet's asssociated conversation. - * This function enables the prefetch caching of UnmentionInfo used by graphql during createTweet - * events and mirrors the logic found in the unmentionInfo Strato column found - * here: http://go/unmentionInfo.strato - * @param client Strato client - * @return - */ - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[Tweet, UnmentionInfoView, UnmentionInfo] = - client.fetcher[Tweet, UnmentionInfoView, UnmentionInfo](column) - - tweet => - tweet.coreData.flatMap(_.conversationId) match { - case Some(conversationId) => - val viewerUserId = TwitterContext().flatMap(_.userId) - fetcher - .fetch(tweet, UnmentionInfoView(viewerUserId)) - .map(_.v) - case _ => Stitch.None - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionedEntitiesRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionedEntitiesRepository.docx new file mode 100644 index 000000000..242374efc Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionedEntitiesRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionedEntitiesRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionedEntitiesRepository.scala deleted file mode 100644 index ea02cbdd3..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UnmentionedEntitiesRepository.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} - -/** - * Repository for fetching UserIds that have unmentioned themselves from a conversation. - */ -object UnmentionedEntitiesRepository { - type Type = (ConversationId, Seq[UserId]) => Stitch[Option[Seq[UserId]]] - - val column = "consumer-privacy/mentions-management/getUnmentionedUsersFromConversation" - case class GetUnmentionView(userIds: Option[Seq[Long]]) - - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[Long, GetUnmentionView, Seq[Long]] = - client.fetcher[Long, GetUnmentionView, Seq[Long]](column) - - (conversationId, userIds) => - if (userIds.nonEmpty) { - fetcher.fetch(conversationId, GetUnmentionView(Some(userIds))).map(_.v) - } else { - Stitch.None - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UrlRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UrlRepository.docx new file mode 100644 index 000000000..80c0527d1 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UrlRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UrlRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UrlRepository.scala deleted file mode 100644 index b2bf53bac..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UrlRepository.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.service.talon.thriftscala._ -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup -import com.twitter.tweetypie.backends.Talon -import com.twitter.tweetypie.client_id.ClientIdHelper -import com.twitter.tweetypie.core.OverCapacity - -case class UrlSlug(text: String) extends AnyVal -case class ExpandedUrl(text: String) extends AnyVal - -object UrlRepository { - type Type = UrlSlug => Stitch[ExpandedUrl] - - /** - * Builds a UrlRepository from a Talon.Expand arrow. - */ - def apply( - talonExpand: Talon.Expand, - tweetypieClientId: String, - statsReceiver: StatsReceiver, - clientIdHelper: ClientIdHelper, - ): Type = { - val observedTalonExpand: Talon.Expand = - talonExpand - .trackOutcome(statsReceiver, _ => clientIdHelper.effectiveClientId.getOrElse("unknown")) - - val expandGroup = SeqGroup[ExpandRequest, Try[ExpandResponse]] { requests => - LegacySeqGroup.liftToSeqTry( - Future.collect(requests.map(r => observedTalonExpand(r).liftToTry))) - } - - slug => - val request = toExpandRequest(slug, auditMessage(tweetypieClientId, clientIdHelper)) - - Stitch - .call(request, expandGroup) - .lowerFromTry - .flatMap(toExpandedUrl(slug, _)) - } - - def auditMessage(tweetypieClientId: String, clientIdHelper: ClientIdHelper): String = { - tweetypieClientId + clientIdHelper.effectiveClientId.mkString(":", "", "") - } - - def toExpandRequest(slug: UrlSlug, auditMessage: String): ExpandRequest = - ExpandRequest(userId = 0, shortUrl = slug.text, fromUser = false, auditMsg = Some(auditMessage)) - - def toExpandedUrl(slug: UrlSlug, res: ExpandResponse): Stitch[ExpandedUrl] = - res.responseCode match { - case ResponseCode.Ok => - // use Option(res.longUrl) because res.longUrl can be null - Option(res.longUrl) match { - case None => Stitch.NotFound - case Some(longUrl) => Stitch.value(ExpandedUrl(longUrl)) - } - - case ResponseCode.BadInput => - Stitch.NotFound - - // we shouldn't see other ResponseCodes, because Talon.Expand translates them to - // exceptions, but we have this catch-all just in case. - case _ => - Stitch.exception(OverCapacity("talon")) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserInfoRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserInfoRepository.docx new file mode 100644 index 000000000..55a60bdbb Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserInfoRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserInfoRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserInfoRepository.scala deleted file mode 100644 index 204b86cef..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserInfoRepository.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.gizmoduck.thriftscala.UserResponseState -import com.twitter.spam.rtf.thriftscala.{SafetyLevel => ThriftSafetyLevel} -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.tweetypie.core._ -import com.twitter.tweetypie.thriftscala.UserIdentity -import com.twitter.visibility.interfaces.tweets.UserUnavailableStateVisibilityLibrary -import com.twitter.visibility.interfaces.tweets.UserUnavailableStateVisibilityRequest -import com.twitter.visibility.models.SafetyLevel -import com.twitter.visibility.models.UserUnavailableStateEnum -import com.twitter.visibility.models.ViewerContext -import com.twitter.visibility.thriftscala.UserVisibilityResult - -/** - * Some types of user (e.g. frictionless users) may not - * have profiles, so a missing UserIdentity may mean that the user - * does not exist, or that the user does not have a profile. - */ -object UserIdentityRepository { - type Type = UserKey => Stitch[UserIdentity] - - def apply(repo: UserRepository.Type): Type = { key => - val opts = UserQueryOptions(Set(UserField.Profile), UserVisibility.Mentionable) - repo(key, opts) - .map { user => - user.profile.map { profile => - UserIdentity( - id = user.id, - screenName = profile.screenName, - realName = profile.name - ) - } - } - .lowerFromOption() - } -} - -object UserProtectionRepository { - type Type = UserKey => Stitch[Boolean] - - def apply(repo: UserRepository.Type): Type = { - val opts = UserQueryOptions(Set(UserField.Safety), UserVisibility.All) - - userKey => - repo(userKey, opts) - .map(user => user.safety.map(_.isProtected)) - .lowerFromOption() - } -} - -/** - * Query Gizmoduck to check if a user `forUserId` can see user `userKey`. - * If forUserId is Some(), this will also check protected relationship, - * if it's None, it will check others as per UserVisibility.Visible policy in - * UserRepository.scala. If forUserId is None, this doesn't verify any - * relationships, visibility is determined based solely on user's - * properties (eg. deactivated, suspended, etc) - */ -object UserVisibilityRepository { - type Type = Query => Stitch[Option[FilteredState.Unavailable]] - - case class Query( - userKey: UserKey, - forUserId: Option[UserId], - tweetId: TweetId, - isRetweet: Boolean, - isInnerQuotedTweet: Boolean, - safetyLevel: Option[ThriftSafetyLevel]) - - def apply( - repo: UserRepository.Type, - userUnavailableAuthorStateVisibilityLibrary: UserUnavailableStateVisibilityLibrary.Type - ): Type = - query => { - repo( - query.userKey, - UserQueryOptions( - Set(), - UserVisibility.Visible, - forUserId = query.forUserId, - filteredAsFailure = true, - safetyLevel = query.safetyLevel - ) - ) - // We don't actually care about the response here (User's data), only whether - // it was filtered or not - .map { case _ => None } - .rescue { - case fs: FilteredState.Unavailable => Stitch.value(Some(fs)) - case UserFilteredFailure(state, reason) => - userUnavailableAuthorStateVisibilityLibrary - .apply( - UserUnavailableStateVisibilityRequest( - query.safetyLevel - .map(SafetyLevel.fromThrift).getOrElse(SafetyLevel.FilterDefault), - query.tweetId, - ViewerContext.fromContextWithViewerIdFallback(query.forUserId), - toUserUnavailableState(state, reason), - query.isRetweet, - query.isInnerQuotedTweet - ) - ).map(VisibilityResultToFilteredState.toFilteredStateUnavailable) - case NotFound => Stitch.value(Some(FilteredState.Unavailable.Author.NotFound)) - } - } - - def toUserUnavailableState( - userResponseState: UserResponseState, - userVisibilityResult: Option[UserVisibilityResult] - ): UserUnavailableStateEnum = { - (userResponseState, userVisibilityResult) match { - case (UserResponseState.DeactivatedUser, _) => UserUnavailableStateEnum.Deactivated - case (UserResponseState.OffboardedUser, _) => UserUnavailableStateEnum.Offboarded - case (UserResponseState.ErasedUser, _) => UserUnavailableStateEnum.Erased - case (UserResponseState.SuspendedUser, _) => UserUnavailableStateEnum.Suspended - case (UserResponseState.ProtectedUser, _) => UserUnavailableStateEnum.Protected - case (_, Some(result)) => UserUnavailableStateEnum.Filtered(result) - case _ => UserUnavailableStateEnum.Unavailable - } - } -} - -object UserViewRepository { - type Type = Query => Stitch[User] - - case class Query( - userKey: UserKey, - forUserId: Option[UserId], - visibility: UserVisibility, - queryFields: Set[UserField] = Set(UserField.View)) - - def apply(repo: UserRepository.Type): UserViewRepository.Type = - query => - repo(query.userKey, UserQueryOptions(query.queryFields, query.visibility, query.forUserId)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserRepository.docx new file mode 100644 index 000000000..ba382384a Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserRepository.scala deleted file mode 100644 index ca80d9503..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserRepository.scala +++ /dev/null @@ -1,285 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.gizmoduck.thriftscala.LookupContext -import com.twitter.gizmoduck.thriftscala.UserResponseState -import com.twitter.gizmoduck.thriftscala.UserResult -import com.twitter.servo.cache.ScopedCacheKey -import com.twitter.servo.json.syntax._ -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.NotFound -import com.twitter.stitch.SeqGroup -import com.twitter.stitch.Stitch -import com.twitter.stitch.compat.LegacySeqGroup -import com.twitter.tweetypie.backends.Gizmoduck -import com.twitter.tweetypie.core._ -import com.twitter.util.Base64Long.toBase64 -import com.twitter.util.logging.Logger -import com.twitter.visibility.thriftscala.UserVisibilityResult -import scala.util.control.NoStackTrace - -sealed trait UserKey - -object UserKey { - def byId(userId: UserId): UserKey = UserIdKey(userId) - def byScreenName(screenName: String): UserKey = ScreenNameKey.toLowerCase(screenName) - def apply(userId: UserId): UserKey = UserIdKey(userId) - def apply(screenName: String): UserKey = ScreenNameKey.toLowerCase(screenName) -} - -case class UserIdKey(userId: UserId) - extends ScopedCacheKey("t", "usr", 1, "id", toBase64(userId)) - with UserKey - -object ScreenNameKey { - def toLowerCase(screenName: String): ScreenNameKey = ScreenNameKey(screenName.toLowerCase) -} - -/** - * Use UserKey.apply(String) instead of ScreenNameKey(String) to construct a key, - * as it will down-case the screen-name to better utilize the user cache. - */ -case class ScreenNameKey private (screenName: String) - extends ScopedCacheKey("t", "usr", 1, "sn", screenName) - with UserKey - -/** - * A set of flags, used in UserQuery, which control whether to include or filter out - * users in various non-standard states. - */ -case class UserVisibility( - filterProtected: Boolean, - filterSuspended: Boolean, - filterDeactivated: Boolean, - filterOffboardedAndErased: Boolean, - filterNoScreenName: Boolean, - filterPeriscope: Boolean, - filterSoft: Boolean) - -object UserVisibility { - - /** - * No filtering, can see every user that gizmoduck can return. - */ - val All: UserVisibility = UserVisibility( - filterProtected = false, - filterSuspended = false, - filterDeactivated = false, - filterOffboardedAndErased = false, - filterNoScreenName = false, - filterPeriscope = false, - filterSoft = false - ) - - /** - * Only includes users that would be visible to a non-logged in user, - * or a logged in user where the following graph is checked for - * protected users. - * - * no-screen-name, soft, and periscope users are visible, but not - * mentionable. - */ - val Visible: UserVisibility = UserVisibility( - filterProtected = true, - filterSuspended = true, - filterDeactivated = true, - filterOffboardedAndErased = true, - filterNoScreenName = false, - filterPeriscope = false, - filterSoft = false - ) - - val MediaTaggable: UserVisibility = UserVisibility( - filterProtected = false, - filterSuspended = true, - filterDeactivated = true, - filterOffboardedAndErased = true, - filterNoScreenName = true, - filterPeriscope = true, - filterSoft = true - ) - - /** - * Includes all mentionable users (filter deactivated/offboarded/erased/no-screen-name users) - */ - val Mentionable: UserVisibility = UserVisibility( - filterProtected = false, - filterSuspended = false, - filterDeactivated = false, - filterOffboardedAndErased = true, - filterNoScreenName = true, - filterPeriscope = true, - filterSoft = true - ) -} - -/** - * The `visibility` field includes a set of flags that indicate whether users in - * various non-standard states should be included in the `found` results, or filtered - * out. By default, "filtered out" means to treat them as `notFound`, but if `filteredAsFailure` - * is true, then the filtered users will be indicated in a [[UserFilteredFailure]] result. - */ -case class UserQueryOptions( - queryFields: Set[UserField] = Set.empty, - visibility: UserVisibility, - forUserId: Option[UserId] = None, - filteredAsFailure: Boolean = false, - safetyLevel: Option[SafetyLevel] = None) { - def toLookupContext: LookupContext = - LookupContext( - includeFailed = true, - forUserId = forUserId, - includeProtected = !visibility.filterProtected, - includeSuspended = !visibility.filterSuspended, - includeDeactivated = !visibility.filterDeactivated, - includeErased = !visibility.filterOffboardedAndErased, - includeNoScreenNameUsers = !visibility.filterNoScreenName, - includePeriscopeUsers = !visibility.filterPeriscope, - includeSoftUsers = !visibility.filterSoft, - includeOffboarded = !visibility.filterOffboardedAndErased, - safetyLevel = safetyLevel - ) -} - -case class UserLookupFailure(message: String, state: UserResponseState) extends RuntimeException { - override def getMessage(): String = - s"$message: responseState = $state" -} - -/** - * Indicates a failure due to the user being filtered. - * - * @see [[GizmoduckUserRepository.FilteredStates]] - */ -case class UserFilteredFailure(state: UserResponseState, reason: Option[UserVisibilityResult]) - extends Exception - with NoStackTrace - -object UserRepository { - type Type = (UserKey, UserQueryOptions) => Stitch[User] - type Optional = (UserKey, UserQueryOptions) => Stitch[Option[User]] - - def optional(repo: Type): Optional = - (userKey, queryOptions) => repo(userKey, queryOptions).liftNotFoundToOption - - def userGetter( - userRepo: UserRepository.Optional, - opts: UserQueryOptions - ): UserKey => Future[Option[User]] = - userKey => Stitch.run(userRepo(userKey, opts)) -} - -object GizmoduckUserRepository { - private[this] val log = Logger(getClass) - - def apply( - getById: Gizmoduck.GetById, - getByScreenName: Gizmoduck.GetByScreenName, - maxRequestSize: Int = Int.MaxValue - ): UserRepository.Type = { - case class GetBy[K]( - opts: UserQueryOptions, - get: ((LookupContext, Seq[K], Set[UserField])) => Future[Seq[UserResult]]) - extends SeqGroup[K, UserResult] { - override def run(keys: Seq[K]): Future[Seq[Try[UserResult]]] = - LegacySeqGroup.liftToSeqTry(get((opts.toLookupContext, keys, opts.queryFields))) - override def maxSize: Int = maxRequestSize - } - - (key, opts) => { - val result = - key match { - case UserIdKey(id) => Stitch.call(id, GetBy(opts, getById)) - case ScreenNameKey(sn) => Stitch.call(sn, GetBy(opts, getByScreenName)) - } - - result.flatMap(r => Stitch.const(toTryUser(r, opts.filteredAsFailure))) - } - } - - private def toTryUser( - userResult: UserResult, - filteredAsFailure: Boolean - ): Try[User] = - userResult.responseState match { - case s if s.forall(SuccessStates.contains(_)) => - userResult.user match { - case Some(u) => - Return(u) - - case None => - log.warn( - s"User expected to be present, but not found in:\n${userResult.prettyPrint}" - ) - // This should never happen, but if it does, treat it as the - // user being returned as NotFound. - Throw(NotFound) - } - - case Some(s) if NotFoundStates.contains(s) => - Throw(NotFound) - - case Some(s) if FilteredStates.contains(s) => - Throw(if (filteredAsFailure) UserFilteredFailure(s, userResult.unsafeReason) else NotFound) - - case Some(UserResponseState.Failed) => - def lookupFailure(msg: String) = - UserLookupFailure(msg, UserResponseState.Failed) - - Throw { - userResult.failureReason - .map { reason => - reason.internalServerError - .orElse { - reason.overCapacity.map { e => - // Convert Gizmoduck OverCapacity to Tweetypie - // OverCapacity exception, explaining that it was - // propagated from Gizmoduck. - OverCapacity(s"gizmoduck over capacity: ${e.message}") - } - } - .orElse(reason.unexpectedException.map(lookupFailure)) - .getOrElse(lookupFailure("failureReason empty")) - } - .getOrElse(lookupFailure("failureReason missing")) - } - - case Some(unexpected) => - Throw(UserLookupFailure("Unexpected response state", unexpected)) - } - - /** - * States that we expect to correspond to a user being returned. - */ - val SuccessStates: Set[UserResponseState] = - Set[UserResponseState]( - UserResponseState.Found, - UserResponseState.Partial - ) - - /** - * States that always correspond to a NotFound response. - */ - val NotFoundStates: Set[UserResponseState] = - Set[UserResponseState]( - UserResponseState.NotFound, - // These are really filtered out, but we treat them as not found - // since we don't have analogous filtering states for tweets. - UserResponseState.PeriscopeUser, - UserResponseState.SoftUser, - UserResponseState.NoScreenNameUser - ) - - /** - * Response states that correspond to a FilteredState - */ - val FilteredStates: Set[UserResponseState] = - Set( - UserResponseState.DeactivatedUser, - UserResponseState.OffboardedUser, - UserResponseState.ErasedUser, - UserResponseState.SuspendedUser, - UserResponseState.ProtectedUser, - UserResponseState.UnsafeUser - ) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserTakedownRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserTakedownRepository.docx new file mode 100644 index 000000000..b9a81f9a6 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserTakedownRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserTakedownRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserTakedownRepository.scala deleted file mode 100644 index 488c2cca6..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserTakedownRepository.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.stitch.Stitch -import com.twitter.takedown.util.TakedownReasons -import com.twitter.tseng.withholding.thriftscala.TakedownReason - -/** - * Query TakedownReason objects from gizmoduck - * - * No backfill job has been completed so there may exist users that have a takedown - * country_code without a corresponding UnspecifiedReason takedown_reason. Therefore, - * read from both fields and merge into a set of TakedownReason, translating raw takedown - * country_code into TakedownReason.UnspecifiedReason(country_code). - */ -object UserTakedownRepository { - type Type = UserId => Stitch[Set[TakedownReason]] - - val userQueryOptions: UserQueryOptions = - UserQueryOptions(Set(UserField.Takedowns), UserVisibility.All) - - def apply(userRepo: UserRepository.Type): UserTakedownRepository.Type = - userId => - userRepo(UserKey(userId = userId), userQueryOptions) - .map(_.takedowns.map(TakedownReasons.userTakedownsToReasons).getOrElse(Set.empty)) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserViewerRecipient.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserViewerRecipient.docx new file mode 100644 index 000000000..426ae3482 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserViewerRecipient.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserViewerRecipient.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserViewerRecipient.scala deleted file mode 100644 index 1dd2b0e92..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/UserViewerRecipient.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.tweetypie -package repository - -import com.twitter.context.thriftscala.Viewer -import com.twitter.featureswitches.Recipient -import com.twitter.featureswitches.TOOClient -import com.twitter.featureswitches.UserAgent -import com.twitter.tweetypie.StatsReceiver -import com.twitter.tweetypie.User -import com.twitter.tweetypie.UserId -import com.twitter.tweetypie.client_id.ClientIdHelper -import com.twitter.tweetypie.repository.UserViewerRecipient.UserIdMismatchException - -/** - * Provides a Recipient backed by a Gizmoduck User and TwitterContext Viewer for - * use in FeatureSwitch validation. - */ -object UserViewerRecipient { - object UserIdMismatchException extends Exception - - def apply(user: User, viewer: Viewer, stats: StatsReceiver): Option[Recipient] = { - // This is a workaround for thrift API clients that allow users to Tweet on behalf - // of other Twitter users. This is similar to go/contributors, however some platforms - // have enabled workflows that don't use the go/contributors auth platform, and - // therefore the TwitterContext Viewer isn't set up correctly for contributor requests. - if (viewer.userId.contains(user.id)) { - Some(new UserViewerRecipient(user, viewer)) - } else { - val mismatchScope = stats.scope(s"user_viewer_mismatch") - ClientIdHelper.default.effectiveClientIdRoot.foreach { clientId => - mismatchScope.scope("client").counter(clientId).incr() - } - mismatchScope.counter("total").incr() - None - } - } -} - -class UserViewerRecipient( - user: User, - viewer: Viewer) - extends Recipient { - - if (!viewer.userId.contains(user.id)) { - throw UserIdMismatchException - } - - override def userId: Option[UserId] = viewer.userId - - override def userRoles: Option[Set[String]] = user.roles.map(_.roles.toSet) - - override def deviceId: Option[String] = viewer.deviceId - - override def guestId: Option[Long] = viewer.guestId - - override def languageCode: Option[String] = viewer.requestLanguageCode - - override def signupCountryCode: Option[String] = user.safety.flatMap(_.signupCountryCode) - - override def countryCode: Option[String] = viewer.requestCountryCode - - override def userAgent: Option[UserAgent] = viewer.userAgent.flatMap(UserAgent(_)) - - override def isManifest: Boolean = false - - override def isVerified: Option[Boolean] = user.safety.map(_.verified) - - override def clientApplicationId: Option[Long] = viewer.clientApplicationId - - @Deprecated - override def isTwoffice: Option[Boolean] = None - - @Deprecated - override def tooClient: Option[TOOClient] = None - - @Deprecated - override def highWaterMark: Option[Long] = None -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VibeRepository.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VibeRepository.docx new file mode 100644 index 000000000..d538c5077 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VibeRepository.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VibeRepository.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VibeRepository.scala deleted file mode 100644 index 780773942..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VibeRepository.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.strato.thrift.ScroogeConvImplicits._ -import com.twitter.vibes.thriftscala.VibeV2 - -object VibeRepository { - type Type = Tweet => Stitch[Option[VibeV2]] - - val column = "vibes/vibe.Tweet" - case class VibeView(viewerId: Option[Long]) - - /** - * Creates a function that applies the vibes/vibe.Tweet strato column fetch on the given - * Tweet. Strato column source: go/vibe.strato - * @param client Strato client - * @return - */ - def apply(client: StratoClient): Type = { - val fetcher: Fetcher[Long, VibeView, VibeV2] = - client.fetcher[Long, VibeView, VibeV2](column) - tweet => - fetcher - .fetch(tweet.id, VibeView(None)) - .map(_.v) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VisibilityResultToFilteredState.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VisibilityResultToFilteredState.docx new file mode 100644 index 000000000..5dab01a7c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VisibilityResultToFilteredState.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VisibilityResultToFilteredState.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VisibilityResultToFilteredState.scala deleted file mode 100644 index 4eec0613f..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/VisibilityResultToFilteredState.scala +++ /dev/null @@ -1,209 +0,0 @@ -package com.twitter.tweetypie.repository - -import com.twitter.spam.rtf.thriftscala.FilteredReason -import com.twitter.spam.rtf.thriftscala.KeywordMatch -import com.twitter.spam.rtf.thriftscala.SafetyResult -import com.twitter.tweetypie.core.FilteredState -import com.twitter.tweetypie.core.FilteredState.Suppress -import com.twitter.tweetypie.core.FilteredState.Unavailable -import com.twitter.visibility.builder.VisibilityResult -import com.twitter.visibility.common.user_result.UserVisibilityResultHelper -import com.twitter.visibility.rules.Reason._ -import com.twitter.visibility.rules._ -import com.twitter.visibility.{thriftscala => vfthrift} - -object VisibilityResultToFilteredState { - def toFilteredStateUnavailable( - visibilityResult: VisibilityResult - ): Option[FilteredState.Unavailable] = { - val dropSafetyResult = Some( - Unavailable.Drop(FilteredReason.SafetyResult(visibilityResult.getSafetyResult)) - ) - - visibilityResult.verdict match { - case Drop(ExclusiveTweet, _) => - dropSafetyResult - - case Drop(NsfwViewerIsUnderage | NsfwViewerHasNoStatedAge | NsfwLoggedOut, _) => - dropSafetyResult - - case Drop(TrustedFriendsTweet, _) => - dropSafetyResult - - case _: LocalizedTombstone => dropSafetyResult - - case Drop(StaleTweet, _) => dropSafetyResult - - // legacy drop actions - case dropAction: Drop => unavailableFromDropAction(dropAction) - - // not an unavailable state that can be mapped - case _ => None - } - } - - def toFilteredState( - visibilityResult: VisibilityResult, - disableLegacyInterstitialFilteredReason: Boolean - ): Option[FilteredState] = { - val suppressSafetyResult = Some( - Suppress(FilteredReason.SafetyResult(visibilityResult.getSafetyResult)) - ) - val dropSafetyResult = Some( - Unavailable.Drop(FilteredReason.SafetyResult(visibilityResult.getSafetyResult)) - ) - - visibilityResult.verdict match { - case _: Appealable => suppressSafetyResult - - case _: Preview => suppressSafetyResult - - case _: InterstitialLimitedEngagements => suppressSafetyResult - - case _: EmergencyDynamicInterstitial => suppressSafetyResult - - case _: SoftIntervention => suppressSafetyResult - - case _: LimitedEngagements => suppressSafetyResult - - case _: TweetInterstitial => suppressSafetyResult - - case _: TweetVisibilityNudge => suppressSafetyResult - - case Interstitial( - ViewerBlocksAuthor | ViewerReportedAuthor | ViewerReportedTweet | ViewerMutesAuthor | - ViewerHardMutedAuthor | MutedKeyword | InterstitialDevelopmentOnly | HatefulConduct | - AbusiveBehavior, - _, - _) if disableLegacyInterstitialFilteredReason => - suppressSafetyResult - - case Interstitial( - ViewerBlocksAuthor | ViewerReportedAuthor | ViewerReportedTweet | - InterstitialDevelopmentOnly, - _, - _) => - suppressSafetyResult - - case _: ComplianceTweetNotice => suppressSafetyResult - - case Drop(ExclusiveTweet, _) => - dropSafetyResult - - case Drop(NsfwViewerIsUnderage | NsfwViewerHasNoStatedAge | NsfwLoggedOut, _) => - dropSafetyResult - - case Drop(TrustedFriendsTweet, _) => - dropSafetyResult - - case Drop(StaleTweet, _) => dropSafetyResult - - case _: LocalizedTombstone => dropSafetyResult - - case _: Avoid => suppressSafetyResult - - // legacy drop actions - case dropAction: Drop => unavailableFromDropAction(dropAction) - - // legacy suppress actions - case action => suppressFromVisibilityAction(action, !disableLegacyInterstitialFilteredReason) - } - } - - def toFilteredState( - userVisibilityResult: Option[vfthrift.UserVisibilityResult] - ): FilteredState.Unavailable = - userVisibilityResult - .collect { - case blockedUser if UserVisibilityResultHelper.isDropAuthorBlocksViewer(blockedUser) => - Unavailable.Drop(FilteredReason.AuthorBlockViewer(true)) - - /** - * Reuse states for author visibility issues from the [[UserRepository]] for consistency with - * other logic for handling the same types of author visibility filtering. - */ - case protectedUser if UserVisibilityResultHelper.isDropProtectedAuthor(protectedUser) => - Unavailable.Author.Protected - case suspendedUser if UserVisibilityResultHelper.isDropSuspendedAuthor(suspendedUser) => - Unavailable.Author.Suspended - case nsfwUser if UserVisibilityResultHelper.isDropNsfwAuthor(nsfwUser) => - Unavailable.Drop(FilteredReason.ContainNsfwMedia(true)) - case mutedByViewer if UserVisibilityResultHelper.isDropViewerMutesAuthor(mutedByViewer) => - Unavailable.Drop(FilteredReason.ViewerMutesAuthor(true)) - case blockedByViewer - if UserVisibilityResultHelper.isDropViewerBlocksAuthor(blockedByViewer) => - Unavailable.Drop( - FilteredReason.SafetyResult( - SafetyResult( - None, - vfthrift.Action.Drop( - vfthrift.Drop(Some(vfthrift.DropReason.ViewerBlocksAuthor(true))) - )))) - } - .getOrElse(FilteredState.Unavailable.Drop(FilteredReason.UnspecifiedReason(true))) - - private def unavailableFromDropAction(dropAction: Drop): Option[FilteredState.Unavailable] = - dropAction match { - case Drop(AuthorBlocksViewer, _) => - Some(Unavailable.Drop(FilteredReason.AuthorBlockViewer(true))) - case Drop(Unspecified, _) => - Some(Unavailable.Drop(FilteredReason.UnspecifiedReason(true))) - case Drop(MutedKeyword, _) => - Some(Unavailable.Drop(FilteredReason.TweetMatchesViewerMutedKeyword(KeywordMatch("")))) - case Drop(ViewerMutesAuthor, _) => - Some(Unavailable.Drop(FilteredReason.ViewerMutesAuthor(true))) - case Drop(Nsfw, _) => - Some(Unavailable.Drop(FilteredReason.ContainNsfwMedia(true))) - case Drop(NsfwMedia, _) => - Some(Unavailable.Drop(FilteredReason.ContainNsfwMedia(true))) - case Drop(PossiblyUndesirable, _) => - Some(Unavailable.Drop(FilteredReason.PossiblyUndesirable(true))) - case Drop(Bounce, _) => - Some(Unavailable.Drop(FilteredReason.TweetIsBounced(true))) - - /** - * Reuse states for author visibility issues from the [[UserRepository]] for consistency with - * other logic for handling the same types of author visibility filtering. - */ - case Drop(ProtectedAuthor, _) => - Some(Unavailable.Author.Protected) - case Drop(SuspendedAuthor, _) => - Some(Unavailable.Author.Suspended) - case Drop(OffboardedAuthor, _) => - Some(Unavailable.Author.Offboarded) - case Drop(DeactivatedAuthor, _) => - Some(Unavailable.Author.Deactivated) - case Drop(ErasedAuthor, _) => - Some(Unavailable.Author.Deactivated) - case _: Drop => - Some(Unavailable.Drop(FilteredReason.UnspecifiedReason(true))) - } - - private def suppressFromVisibilityAction( - action: Action, - enableLegacyFilteredReason: Boolean - ): Option[FilteredState.Suppress] = - action match { - case interstitial: Interstitial => - interstitial.reason match { - case MutedKeyword if enableLegacyFilteredReason => - Some(Suppress(FilteredReason.TweetMatchesViewerMutedKeyword(KeywordMatch("")))) - case ViewerMutesAuthor if enableLegacyFilteredReason => - Some(Suppress(FilteredReason.ViewerMutesAuthor(true))) - case ViewerHardMutedAuthor if enableLegacyFilteredReason => - Some(Suppress(FilteredReason.ViewerMutesAuthor(true))) - // Interstitial tweets are considered suppressed by Tweetypie. For - // legacy behavior reasons, these tweets should be dropped when - // appearing as a quoted tweet via a call to getTweets. - case Nsfw => - Some(Suppress(FilteredReason.ContainNsfwMedia(true))) - case NsfwMedia => - Some(Suppress(FilteredReason.ContainNsfwMedia(true))) - case PossiblyUndesirable => - Some(Suppress(FilteredReason.PossiblyUndesirable(true))) - case _ => - Some(Suppress(FilteredReason.PossiblyUndesirable(true))) - } - case _ => None - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/package.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/package.docx new file mode 100644 index 000000000..7cc9331b1 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/package.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/package.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/package.scala deleted file mode 100644 index 5aa38d1e2..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/repository/package.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.tweetypie - -import com.twitter.context.TwitterContext -package object repository { - // Bring Tweetypie permitted TwitterContext into scope - val TwitterContext: TwitterContext = - com.twitter.context.TwitterContext(com.twitter.tweetypie.TwitterContextPermit) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityService.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityService.docx new file mode 100644 index 000000000..fd2f56eb4 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityService.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityService.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityService.scala deleted file mode 100644 index c6480d546..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityService.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.tweetypie.serverutil - -import com.twitter.finagle.Service -import com.twitter.util.Activity -import com.twitter.util.Future - -/** - * Transforms an `Activity` that contains a `Service` into a `Service`. - * The implementation guarantees that the service is rebuilt only when the - * activity changes, not on every request. - */ -object ActivityService { - - def apply[Req, Rep](activity: Activity[Service[Req, Rep]]): Service[Req, Rep] = { - - val serviceEvent = - ActivityUtil.strict(activity).values.map(_.get) - - new Service[Req, Rep] { - - def apply(req: Req): Future[Rep] = - serviceEvent.toFuture.flatMap(_.apply(req)) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityUtil.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityUtil.docx new file mode 100644 index 000000000..1dcac7871 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityUtil.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityUtil.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityUtil.scala deleted file mode 100644 index 2ee6d9bd5..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ActivityUtil.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.tweetypie.serverutil - -import com.twitter.util.Activity -import com.twitter.util.Closable -import com.twitter.util.Var -import com.twitter.util.Witness - -object ActivityUtil { - - /** - * Makes the composition strict up to the point where it is called. - * Compositions based on the returned activity will have - * the default lazy behavior. - */ - def strict[T](activity: Activity[T]): Activity[T] = { - val state = Var(Activity.Pending: Activity.State[T]) - val event = activity.states - - Closable.closeOnCollect(event.register(Witness(state)), state) - - new Activity(state) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BUILD b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BUILD deleted file mode 100644 index c660ac645..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "finagle/finagle-core/src/main", - "finagle/finagle-memcached/src/main/scala", - "scrooge/scrooge-core", - "tweetypie/servo/util", - "tweetypie/common/src/thrift/com/twitter/tweetypie:tweet-scala", - "stitch/stitch-core/src/main/scala/com/twitter/stitch", - "tweetypie/server/src/main/scala/com/twitter/tweetypie", - "tweetypie/common/src/scala/com/twitter/tweetypie/client_id", - "tweetypie/common/src/scala/com/twitter/tweetypie/thriftscala/entities", - "tweetypie/common/src/scala/com/twitter/tweetypie/tweettext", - "twitter-config/yaml", - "util/util-hashing/src/main/scala", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - "util/util-stats/src/main/scala", - ], -) diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BUILD.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BUILD.docx new file mode 100644 index 000000000..38c414d3f Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BUILD.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BoringStackTrace.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BoringStackTrace.docx new file mode 100644 index 000000000..528436b03 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BoringStackTrace.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BoringStackTrace.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BoringStackTrace.scala deleted file mode 100644 index d9e57213a..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/BoringStackTrace.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.tweetypie.serverutil - -import com.twitter.finagle.ChannelException -import com.twitter.finagle.TimeoutException -import com.twitter.scrooge.ThriftException -import java.net.SocketException -import java.nio.channels.CancelledKeyException -import java.nio.channels.ClosedChannelException -import java.util.concurrent.CancellationException -import java.util.concurrent.{TimeoutException => JTimeoutException} -import org.apache.thrift.TApplicationException -import scala.util.control.NoStackTrace - -object BoringStackTrace { - - /** - * These exceptions are boring because they are expected to - * occasionally (or even regularly) happen during normal operation - * of the service. The intention is to make it easier to debug - * problems by making interesting exceptions easier to see. - * - * The best way to mark an exception as boring is to extend from - * NoStackTrace, since that is a good indication that we don't care - * about the details. - */ - def isBoring(t: Throwable): Boolean = - t match { - case _: NoStackTrace => true - case _: TimeoutException => true - case _: CancellationException => true - case _: JTimeoutException => true - case _: ChannelException => true - case _: SocketException => true - case _: ClosedChannelException => true - case _: CancelledKeyException => true - case _: ThriftException => true - // DeadlineExceededExceptions are propagated as: - // org.apache.thrift.TApplicationException: Internal error processing issue3: 'com.twitter.finagle.service.DeadlineFilter$DeadlineExceededException: exceeded request deadline of 100.milliseconds by 4.milliseconds. Deadline expired at 2020-08-27 17:07:46 +0000 and now it is 2020-08-27 17:07:46 +0000.' - case e: TApplicationException => - e.getMessage != null && e.getMessage.contains("DeadlineExceededException") - case _ => false - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/CaffeineMemcacheClient.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/CaffeineMemcacheClient.docx new file mode 100644 index 000000000..fec936a5f Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/CaffeineMemcacheClient.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/CaffeineMemcacheClient.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/CaffeineMemcacheClient.scala deleted file mode 100644 index f898c53fc..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/CaffeineMemcacheClient.scala +++ /dev/null @@ -1,174 +0,0 @@ -package com.twitter.tweetypie.serverutil - -import com.github.benmanes.caffeine.cache.stats.CacheStats -import com.github.benmanes.caffeine.cache.stats.StatsCounter -import com.github.benmanes.caffeine.cache.AsyncCacheLoader -import com.github.benmanes.caffeine.cache.AsyncLoadingCache -import com.github.benmanes.caffeine.cache.Caffeine -import com.twitter.finagle.memcached.protocol.Value -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.memcached.GetResult -import com.twitter.finagle.memcached.ProxyClient -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.{Promise => TwitterPromise} -import com.twitter.util.logging.Logger -import java.util.concurrent.TimeUnit.NANOSECONDS -import java.util.concurrent.CompletableFuture -import java.util.concurrent.Executor -import java.util.concurrent.TimeUnit -import java.util.function.BiConsumer -import java.util.function.Supplier -import java.lang -import java.util -import scala.collection.JavaConverters._ - -object CaffeineMemcacheClient { - val logger: Logger = Logger(getClass) - - /** - * Helper method to convert between Java 8's CompletableFuture and Twitter's Future. - */ - private def toTwitterFuture[T](cf: CompletableFuture[T]): Future[T] = { - if (cf.isDone && !cf.isCompletedExceptionally && !cf.isCancelled) { - Future.const(Return(cf.get())) - } else { - val p = new TwitterPromise[T] with TwitterPromise.InterruptHandler { - override protected def onInterrupt(t: Throwable): Unit = cf.cancel(true) - } - cf.whenComplete(new BiConsumer[T, Throwable] { - override def accept(result: T, exception: Throwable): Unit = { - if (exception != null) { - p.updateIfEmpty(Throw(exception)) - } else { - p.updateIfEmpty(Return(result)) - } - } - }) - p - } - } -} - -class CaffeineMemcacheClient( - override val proxyClient: Client, - val maximumSize: Int = 1000, - val ttl: Duration = Duration.fromSeconds(10), - stats: StatsReceiver = NullStatsReceiver) - extends ProxyClient { - import CaffeineMemcacheClient._ - - private[this] object Stats extends StatsCounter { - private val hits = stats.counter("hits") - private val miss = stats.counter("misses") - private val totalLoadTime = stats.stat("loads") - private val loadSuccess = stats.counter("loads-success") - private val loadFailure = stats.counter("loads-failure") - private val eviction = stats.counter("evictions") - private val evictionWeight = stats.counter("evictions-weight") - - override def recordHits(i: Int): Unit = hits.incr(i) - override def recordMisses(i: Int): Unit = miss.incr(i) - override def recordLoadSuccess(l: Long): Unit = { - loadSuccess.incr() - totalLoadTime.add(NANOSECONDS.toMillis(l)) - } - - override def recordLoadFailure(l: Long): Unit = { - loadFailure.incr() - totalLoadTime.add(NANOSECONDS.toMillis(l)) - } - - override def recordEviction(): Unit = recordEviction(1) - override def recordEviction(weight: Int): Unit = { - eviction.incr() - evictionWeight.incr(weight) - } - - /** - * We are currently not using this method. - */ - override def snapshot(): CacheStats = { - new CacheStats(0, 0, 0, 0, 0, 0, 0) - } - } - - private[this] object MemcachedAsyncCacheLoader extends AsyncCacheLoader[String, GetResult] { - private[this] val EmptyMisses: Set[String] = Set.empty - private[this] val EmptyFailures: Map[String, Throwable] = Map.empty - private[this] val EmptyHits: Map[String, Value] = Map.empty - - override def asyncLoad(key: String, executor: Executor): CompletableFuture[GetResult] = { - val f = new util.function.Function[util.Map[String, GetResult], GetResult] { - override def apply(r: util.Map[String, GetResult]): GetResult = r.get(key) - } - asyncLoadAll(Seq(key).asJava, executor).thenApply(f) - } - - /** - * Converts response from multi-key to single key. Memcache returns the result - * in one struct that contains all the hits, misses and exceptions. Caffeine - * requires a map from a key to the result, so we do that conversion here. - */ - override def asyncLoadAll( - keys: lang.Iterable[_ <: String], - executor: Executor - ): CompletableFuture[util.Map[String, GetResult]] = { - val result = new CompletableFuture[util.Map[String, GetResult]]() - proxyClient.getResult(keys.asScala).respond { - case Return(r) => - val map = new util.HashMap[String, GetResult]() - r.hits.foreach { - case (key, value) => - map.put( - key, - r.copy(hits = Map(key -> value), misses = EmptyMisses, failures = EmptyFailures) - ) - } - r.misses.foreach { key => - map.put(key, r.copy(hits = EmptyHits, misses = Set(key), failures = EmptyFailures)) - } - // We are passing through failures so that we maintain the contract expected by clients. - // Without passing through the failures, several metrics get lost. Some of these failures - // might get cached. The cache is short-lived, so we are not worried when it does - // get cached. - r.failures.foreach { - case (key, value) => - map.put( - key, - r.copy(hits = EmptyHits, misses = EmptyMisses, failures = Map(key -> value)) - ) - } - result.complete(map) - case Throw(ex) => - logger.warn("Error loading keys from memcached", ex) - result.completeExceptionally(ex) - } - result - } - } - - private[this] val cache: AsyncLoadingCache[String, GetResult] = - Caffeine - .newBuilder() - .maximumSize(maximumSize) - .refreshAfterWrite(ttl.inMilliseconds * 3 / 4, TimeUnit.MILLISECONDS) - .expireAfterWrite(ttl.inMilliseconds, TimeUnit.MILLISECONDS) - .recordStats(new Supplier[StatsCounter] { - override def get(): StatsCounter = Stats - }) - .buildAsync(MemcachedAsyncCacheLoader) - - override def getResult(keys: Iterable[String]): Future[GetResult] = { - val twitterFuture = toTwitterFuture(cache.getAll(keys.asJava)) - twitterFuture - .map { result => - val values = result.values().asScala - values.reduce(_ ++ _) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/DeviceSourceParser.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/DeviceSourceParser.docx new file mode 100644 index 000000000..f5b0d952b Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/DeviceSourceParser.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/DeviceSourceParser.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/DeviceSourceParser.scala deleted file mode 100644 index 1600269e3..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/DeviceSourceParser.scala +++ /dev/null @@ -1,100 +0,0 @@ -package com.twitter.tweetypie.serverutil - -/** - * Parse a device source into an OAuth app id. This mapping is - * neccesary when you need to request information about a client from - * a service that only knows about clients in terms of oauthIds. - * - * This happens either by parsing out an explicit "oauth:" app id or - * using a mapping from old non oauth clientIds like "web" and "sms" - * to oauthIds that have retroactively been assigned to those clients. - * If the legacy id cannot be found in the map and it's a non-numeric - * string, it's converted to the oauthId for twitter.com. - * - * Tweets with non oauth clientIds are still being created because - * thats how the monorail creates them. We also need to be able to - * process any app id string that is in old tweet data. - * - */ -object DeviceSourceParser { - - /** - * The oauth id for twitter.com. Also used as a default oauth id for - * other clients without their own - */ - val Web = 268278L - - /** - * The OAuth app ids for known legacy device sources. - */ - val legacyMapping: Map[String, Long] = Map[String, Long]( - "web" -> Web, - "tweetbutton" -> 6219130L, - "keitai_web" -> 38366L, - "sms" -> 241256L - ) - - /** - * Attempt to convert a client application id String into an OAuth - * id. - * - * The string must consist of the characters "oauth:" followed by a - * non-negative, decimal long. The text is case-insensitive, and - * whitespace at the beginning or end is ignored. - * - * We want to accept input as liberally as possible, because if we - * fail to do that here, it will get counted as a "legacy app id" - */ - val parseOAuthAppId: String => Option[Long] = { - // Case-insensitive, whitespace insensitive. The javaWhitespace - // character class is consistent with Character.isWhitespace, but is - // sadly different from \s. It will likely not matter in the long - // run, but this accepts more inputs and is easier to test (because - // we can use isWhitespace) - val OAuthAppIdRe = """(?i)\p{javaWhitespace}*oauth:(\d+)\p{javaWhitespace}*""".r - - _ match { - case OAuthAppIdRe(digits) => - // We should only get NumberFormatException when the number is - // larger than a Long, because the regex will rule out all of - // the other invalid cases. - try Some(digits.toLong) - catch { case _: NumberFormatException => None } - case _ => - None - } - } - - /** - * Attempt to convert a client application id String into an OAuth id or legacy identifier without - * any fallback behavior. - */ - val parseStrict: String => Option[Long] = - appIdStr => - parseOAuthAppId(appIdStr) - .orElse(legacyMapping.get(appIdStr)) - - /** - * Return true if a string can be used as a valid client application id or legacy identifier - */ - val isValid: String => Boolean = appIdStr => parseStrict(appIdStr).isDefined - - /** - * Build a parser that converts device sources to OAuth app ids, - * including performing the legacy mapping. - */ - val parseAppId: String => Option[Long] = { - val IsNumericRe = """-?[0-9]+""".r - - appIdStr => - parseStrict(appIdStr) - .orElse { - appIdStr match { - // We just fail the lookup if the app id looks like it's - // numeric. - case IsNumericRe() => None - case _ => Some(Web) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExceptionCounter.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExceptionCounter.docx new file mode 100644 index 000000000..b3b4485a1 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExceptionCounter.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExceptionCounter.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExceptionCounter.scala deleted file mode 100644 index 0a7c6e43b..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExceptionCounter.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.tweetypie.serverutil - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo -import com.twitter.servo.util.ExceptionCategorizer - -object ExceptionCounter { - // These throwables are alertable because they indicate conditions we never expect in production. - def isAlertable(throwable: Throwable): Boolean = - throwable match { - case e: RuntimeException => true - case e: Error => true - case _ => false - } - - // count how many exceptions are alertable and how many are boring - val tweetypieCategorizers: ExceptionCategorizer = - ExceptionCategorizer.const("alertableException").onlyIf(isAlertable) ++ - ExceptionCategorizer.const("boringException").onlyIf(BoringStackTrace.isBoring) - - val defaultCategorizer: ExceptionCategorizer = - ExceptionCategorizer.default() ++ tweetypieCategorizers - - def defaultCategorizer(name: String): ExceptionCategorizer = - ExceptionCategorizer.default(Seq(name)) ++ tweetypieCategorizers - - def apply(statsReceiver: StatsReceiver): servo.util.ExceptionCounter = - new servo.util.ExceptionCounter(statsReceiver, defaultCategorizer) - - def apply(statsReceiver: StatsReceiver, name: String): servo.util.ExceptionCounter = - new servo.util.ExceptionCounter(statsReceiver, defaultCategorizer(name)) - - def apply( - statsReceiver: StatsReceiver, - categorizer: ExceptionCategorizer - ): servo.util.ExceptionCounter = - new servo.util.ExceptionCounter(statsReceiver, categorizer) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExtendedTweetMetadataBuilder.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExtendedTweetMetadataBuilder.docx new file mode 100644 index 000000000..905318a6c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExtendedTweetMetadataBuilder.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExtendedTweetMetadataBuilder.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExtendedTweetMetadataBuilder.scala deleted file mode 100644 index 53a3bc18d..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/ExtendedTweetMetadataBuilder.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.tweetypie.serverutil - -import com.twitter.tweetypie.getCashtags -import com.twitter.tweetypie.getHashtags -import com.twitter.tweetypie.getMedia -import com.twitter.tweetypie.getMentions -import com.twitter.tweetypie.getText -import com.twitter.tweetypie.getUrls -import com.twitter.tweetypie.thriftscala.ExtendedTweetMetadata -import com.twitter.tweetypie.thriftscala.ShortenedUrl -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.tweetypie.tweettext.Offset -import com.twitter.tweetypie.tweettext.TextEntity -import com.twitter.tweetypie.tweettext.Truncator -import com.twitter.tweetypie.tweettext.TweetText -import com.twitter.tweetypie.thriftscala.entities.Implicits._ - -/** - * Computes the appropriate truncation index to support rendering on legacy clients. - */ -object ExtendedTweetMetadataBuilder { - import TweetText._ - - def apply(tweet: Tweet, selfPermalink: ShortenedUrl): ExtendedTweetMetadata = { - - def entityRanges[T: TextEntity](entities: Seq[T]): Seq[(Int, Int)] = - entities.map(e => (TextEntity.fromIndex(e).toInt, TextEntity.toIndex(e).toInt)) - - val allEntityRanges = - Offset.Ranges.fromCodePointPairs( - entityRanges(getUrls(tweet)) ++ - entityRanges(getMentions(tweet)) ++ - entityRanges(getMedia(tweet)) ++ - entityRanges(getHashtags(tweet)) ++ - entityRanges(getCashtags(tweet)) - ) - - val text = getText(tweet) - - val apiCompatibleTruncationIndex = - // need to leave enough space for ellipsis, space, and self-permalink - Truncator.truncationPoint( - text = text, - maxDisplayLength = OriginalMaxDisplayLength - selfPermalink.shortUrl.length - 2, - atomicUnits = allEntityRanges - ) - - ExtendedTweetMetadata( - apiCompatibleTruncationIndex = apiCompatibleTruncationIndex.codePointOffset.toInt - ) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/NullMemcacheClient.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/NullMemcacheClient.docx new file mode 100644 index 000000000..e02fb6be4 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/NullMemcacheClient.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/NullMemcacheClient.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/NullMemcacheClient.scala deleted file mode 100644 index 0cbecec88..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/NullMemcacheClient.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.tweetypie.serverutil - -import com.twitter.finagle.memcached -import com.twitter.finagle.memcached.CasResult -import com.twitter.io.Buf -import com.twitter.tweetypie.Future -import com.twitter.tweetypie.Time -import java.lang - -/** - * This will be used during CI test runs, in the no-cache scenarios for both DCs. - * We are treating this as cache of instantaneous expiry. MockClient uses an in-memory map as - * an underlying data-store, we extend it and prevent any writes to the map - thus making sure - * it's always empty. - */ -class NullMemcacheClient extends memcached.MockClient { - override def set(key: String, flags: Int, expiry: Time, value: Buf): Future[Unit] = Future.Done - - override def add(key: String, flags: Int, expiry: Time, value: Buf): Future[lang.Boolean] = - Future.value(true) - - override def append(key: String, flags: Int, expiry: Time, value: Buf): Future[lang.Boolean] = - Future.value(false) - - override def prepend(key: String, flags: Int, expiry: Time, value: Buf): Future[lang.Boolean] = - Future.value(false) - - override def replace(key: String, flags: Int, expiry: Time, value: Buf): Future[lang.Boolean] = - Future.value(false) - - override def checkAndSet( - key: String, - flags: Int, - expiry: Time, - value: Buf, - casUnique: Buf - ): Future[CasResult] = Future.value(CasResult.NotFound) - - override def delete(key: String): Future[lang.Boolean] = Future.value(false) - - override def incr(key: String, delta: Long): Future[Option[lang.Long]] = - Future.value(None) - - override def decr(key: String, delta: Long): Future[Option[lang.Long]] = - Future.value(None) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/PartnerMedia.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/PartnerMedia.docx new file mode 100644 index 000000000..1efbd38fd Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/PartnerMedia.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/PartnerMedia.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/PartnerMedia.scala deleted file mode 100644 index f2c32d7b4..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/PartnerMedia.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.tweetypie.serverutil - -import com.twitter.config.yaml.YamlMap -import scala.util.matching.Regex - -object PartnerMedia { - def load(yamlMap: YamlMap): Seq[Regex] = - (httpOrHttps(yamlMap) ++ httpOnly(yamlMap)).map(_.r) - - private def httpOrHttps(yamlMap: YamlMap): Seq[String] = - yamlMap.stringSeq("http_or_https").map("""^(?:https?\:\/\/)?""" + _) - - private def httpOnly(yamlMap: YamlMap): Seq[String] = - yamlMap.stringSeq("http_only").map("""^(?:http\:\/\/)?""" + _) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/StoredCard.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/StoredCard.docx new file mode 100644 index 000000000..c58985440 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/StoredCard.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/StoredCard.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/StoredCard.scala deleted file mode 100644 index 566d43c24..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/StoredCard.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.tweetypie.serverutil - -import com.twitter.tweetypie.thriftscala.CardReference -import com.twitter.util.Try -import java.net.URI -import scala.util.control.NonFatal - -/** - * Utility to extract the stored card id out of a CardReference - */ -object StoredCard { - - private val cardScheme = "card" - private val cardPrefix = s"$cardScheme://" - - /** - * Looks at the CardReference to determines if the cardUri points to a "stored" - * card id. Stored Card URIs are are expected to be in the format "card://" - * (case sensitive). In future these URIs can potentially be: - * "card://[/path[?queryString]]. Note that this utility cares just about the - * "Stored Card" types. So it just skips the other card types. - */ - def unapply(cr: CardReference): Option[Long] = { - try { - for { - uriStr <- Option(cr.cardUri) if uriStr.startsWith(cardPrefix) - uri <- Try(new URI(uriStr)).toOption - if uri.getScheme == cardScheme && uri.getHost != null - } yield uri.getHost.toLong // throws NumberFormatException non numeric host (cardIds) - } catch { - // The validations are done upstream by the TweetBuilder, so exceptions - // due to bad URIs will be swallowed. - case NonFatal(e) => None - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/BUILD b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/BUILD deleted file mode 100644 index 768daa991..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "tweetypie/servo/repo", - "tweetypie/servo/util", - "snowflake:id", - "tweetypie/server/src/main/scala/com/twitter/tweetypie", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/core", - "tweetypie/server/src/main/thrift:compiled-scala", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - ], -) diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/BUILD.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/BUILD.docx new file mode 100644 index 000000000..fc9cd5200 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/BUILD.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/TweetCacheWrite.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/TweetCacheWrite.docx new file mode 100644 index 000000000..c99cd1b18 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/TweetCacheWrite.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/TweetCacheWrite.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/TweetCacheWrite.scala deleted file mode 100644 index 6f1f49cd0..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/TweetCacheWrite.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.tweetypie.serverutil.logcachewrites - -import com.twitter.servo.cache.Cached -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.tweetypie.TweetId -import com.twitter.tweetypie.core.Serializer -import com.twitter.tweetypie.thriftscala.CachedTweet -import com.twitter.util.Time -import java.util.Base64 - -/** - * A record of a tweet cache write. This is used for debugging. These log - * messages are scribed to test_tweetypie_tweet_cache_writes. - */ -case class TweetCacheWrite( - tweetId: TweetId, - timestamp: Time, - action: String, - value: Option[Cached[CachedTweet]]) { - - /** - * Convert to a tab-separated string suitable for writing to a log message. - * - * Fields are: - * - Tweet id - * - Timestamp: - * If the tweet id is a snowflake id, this is an offset since tweet creation. - * If it is not a snowflake id, then this is a Unix epoch time in - * milliseconds. (The idea is that for most tweets, this encoding will make - * it easier to see the interval between events and whether it occured soon - * after tweet creation.) - * - Cache action ("set", "add", "replace", "cas", "delete") - * - Base64-encoded Cached[CachedTweet] struct - */ - def toLogMessage: String = { - val builder = new java.lang.StringBuilder - val timestampOffset = - if (SnowflakeId.isSnowflakeId(tweetId)) { - SnowflakeId(tweetId).unixTimeMillis.asLong - } else { - 0 - } - builder - .append(tweetId) - .append('\t') - .append(timestamp.inMilliseconds - timestampOffset) - .append('\t') - .append(action) - .append('\t') - value.foreach { ct => - // When logging, we end up serializing the value twice, once for the - // cache write and once for the logging. This is suboptimal, but the - // assumption is that we only do this for a small fraction of cache - // writes, so it should be ok. The reason that this is necessary is - // because we want to do the filtering on the deserialized value, so - // the serialized value is not available at the level that we are - // doing the filtering. - val thriftBytes = Serializer.CachedTweet.CachedCompact.to(ct).get - builder.append(Base64.getEncoder.encodeToString(thriftBytes)) - } - builder.toString - } -} - -object TweetCacheWrite { - case class ParseException(msg: String, cause: Exception) extends RuntimeException(cause) { - override def getMessage: String = s"Failed to parse as TweetCacheWrite: $msg" - } - - /** - * Parse a TweetCacheWrite object from the result of TweetCacheWrite.toLogMessage - */ - def fromLogMessage(msg: String): TweetCacheWrite = - try { - val (tweetIdStr, timestampStr, action, cachedTweetStr) = - msg.split('\t') match { - case Array(f1, f2, f3) => (f1, f2, f3, None) - case Array(f1, f2, f3, f4) => (f1, f2, f3, Some(f4)) - } - val tweetId = tweetIdStr.toLong - val timestamp = { - val offset = - if (SnowflakeId.isSnowflakeId(tweetId)) { - SnowflakeId(tweetId).unixTimeMillis.asLong - } else { - 0 - } - Time.fromMilliseconds(timestampStr.toLong + offset) - } - val value = cachedTweetStr.map { str => - val thriftBytes = Base64.getDecoder.decode(str) - Serializer.CachedTweet.CachedCompact.from(thriftBytes).get - } - - TweetCacheWrite(tweetIdStr.toLong, timestamp, action, value) - } catch { - case e: Exception => throw ParseException(msg, e) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/WriteLoggingCache.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/WriteLoggingCache.docx new file mode 100644 index 000000000..94b579fbb Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/WriteLoggingCache.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/WriteLoggingCache.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/WriteLoggingCache.scala deleted file mode 100644 index a332c8e59..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil/logcachewrites/WriteLoggingCache.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.tweetypie.serverutil.logcachewrites - -import com.twitter.servo.cache.Checksum -import com.twitter.servo.cache.CacheWrapper -import com.twitter.util.Future -import com.twitter.util.logging.Logger -import scala.util.control.NonFatal - -trait WriteLoggingCache[K, V] extends CacheWrapper[K, V] { - // Use getClass so we can see which implementation is actually failing. - private[this] lazy val logFailureLogger = Logger(getClass) - - def selectKey(k: K): Boolean - def select(k: K, v: V): Boolean - def log(action: String, k: K, v: Option[V]): Unit - - def safeLog(action: String, k: K, v: Option[V]): Unit = - try { - log(action, k, v) - } catch { - case NonFatal(e) => - // The exception occurred in logging, and we don't want to fail the - // request with the logging failure if this happens, so log it and carry - // on. - logFailureLogger.error("Logging cache write", e) - } - - override def add(k: K, v: V): Future[Boolean] = - // Call the selection function before doing the work. Since it's highly - // likely that the Future will succeed, it's cheaper to call the function - // before we make the call so that we can avoid creating the callback and - // attaching it to the Future if we would not log. - if (select(k, v)) { - underlyingCache.add(k, v).onSuccess(r => if (r) safeLog("add", k, Some(v))) - } else { - underlyingCache.add(k, v) - } - - override def checkAndSet(k: K, v: V, checksum: Checksum): Future[Boolean] = - if (select(k, v)) { - underlyingCache.checkAndSet(k, v, checksum).onSuccess(r => if (r) safeLog("cas", k, Some(v))) - } else { - underlyingCache.checkAndSet(k, v, checksum) - } - - override def set(k: K, v: V): Future[Unit] = - if (select(k, v)) { - underlyingCache.set(k, v).onSuccess(_ => safeLog("set", k, Some(v))) - } else { - underlyingCache.set(k, v) - } - - override def replace(k: K, v: V): Future[Boolean] = - if (select(k, v)) { - underlyingCache.replace(k, v).onSuccess(r => if (r) safeLog("replace", k, Some(v))) - } else { - underlyingCache.replace(k, v) - } - - override def delete(k: K): Future[Boolean] = - if (selectKey(k)) { - underlyingCache.delete(k).onSuccess(r => if (r) safeLog("delete", k, None)) - } else { - underlyingCache.delete(k) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/BUILD b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/BUILD deleted file mode 100644 index 1fb3cf249..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/BUILD +++ /dev/null @@ -1,38 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/org/apache/thrift:libthrift", - "core-app-services/failed_task:writer", - "core-app-services/lib:coreservices", - "finagle/finagle-core/src/main", - "finagle/finagle-mux/src/main/scala", - "finagle/finagle-stats", - "quill/capture", - "quill/core/src/main/thrift:thrift-scala", - "scrooge/scrooge-core/src/main/scala", - "tweetypie/servo/request/src/main/scala", - "tweetypie/servo/util", - "src/thrift/com/twitter/servo:servo-exception-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:delete_location_data-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:service-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:tweet-scala", - "tweetypie/server/src/main/scala/com/twitter/tweetypie", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/core", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/handler", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/store", - "tweetypie/server/src/main/thrift:compiled-scala", - "tweetypie/common/src/scala/com/twitter/tweetypie/additionalfields", - "tweetypie/common/src/scala/com/twitter/tweetypie/client_id", - "tweetypie/common/src/scala/com/twitter/tweetypie/context", - "tweetypie/common/src/scala/com/twitter/tweetypie/thriftscala", - "twitter-server-internal", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - "util/util-stats/src/main/scala", - ], -) diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/BUILD.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/BUILD.docx new file mode 100644 index 000000000..62ecdd1e8 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/BUILD.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ClientHandlingTweetService.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ClientHandlingTweetService.docx new file mode 100644 index 000000000..096efa91c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ClientHandlingTweetService.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ClientHandlingTweetService.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ClientHandlingTweetService.scala deleted file mode 100644 index f19245b60..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ClientHandlingTweetService.scala +++ /dev/null @@ -1,524 +0,0 @@ -/** Copyright 2012 Twitter, Inc. */ -package com.twitter.tweetypie.service - -import com.twitter.coreservices.StratoPublicApiRequestAttributionCounter -import com.twitter.finagle.CancelledRequestException -import com.twitter.finagle.context.Contexts -import com.twitter.finagle.context.Deadline -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.stats.DefaultStatsReceiver -import com.twitter.finagle.stats.Stat -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.servo.util.ExceptionCategorizer -import com.twitter.servo.util.MemoizedExceptionCounterFactory -import com.twitter.tweetypie.Future -import com.twitter.tweetypie.Gate -import com.twitter.tweetypie.Logger -import com.twitter.tweetypie.StatsReceiver -import com.twitter.tweetypie.ThriftTweetService -import com.twitter.tweetypie.TweetId -import com.twitter.tweetypie.client_id.ClientIdHelper -import com.twitter.tweetypie.context.TweetypieContext -import com.twitter.tweetypie.core.OverCapacity -import com.twitter.tweetypie.serverutil.ExceptionCounter -import com.twitter.tweetypie.thriftscala._ -import com.twitter.util.Promise - -/** - * A TweetService that takes care of the handling of requests from - * external services. In particular, this wrapper doesn't have any - * logic for handling requests itself. It just serves as a gateway for - * requests and responses, making sure that the underlying tweet - * service only sees requests it should handle and that the external - * clients get clean responses. - * - * - Ensures that exceptions are propagated cleanly - * - Sheds traffic if necessary - * - Authenticates clients - * - Records stats about clients - * - * For each endpoint, we record both client-specific and total metrics for number of requests, - * successes, exceptions, and latency. The stats names follow the patterns: - * - .//requests - * - .//success - * - .//client_errors - * - .//server_errors - * - .//exceptions - * - .//exceptions/ - * - .///requests - * - .///success - * - .///exceptions - * - .///exceptions/ - */ -class ClientHandlingTweetService( - underlying: ThriftTweetService, - stats: StatsReceiver, - loadShedEligible: Gate[String], - shedReadTrafficVoluntarily: Gate[Unit], - requestAuthorizer: ClientRequestAuthorizer, - getTweetsAuthorizer: MethodAuthorizer[GetTweetsRequest], - getTweetFieldsAuthorizer: MethodAuthorizer[GetTweetFieldsRequest], - requestSizeAuthorizer: MethodAuthorizer[Int], - clientIdHelper: ClientIdHelper) - extends ThriftTweetService { - import RescueExceptions._ - - private val log = Logger("com.twitter.tweetypie.service.TweetService") - - private[this] val Requests = "requests" - private[this] val Success = "success" - private[this] val Latency = "latency_ms" - - private[this] val StratoStatsCounter = new StratoPublicApiRequestAttributionCounter( - DefaultStatsReceiver - ) - private[this] val clientServerCategorizer = - ExceptionCategorizer.simple { - _ match { - case _: ClientError | _: AccessDenied => "client_errors" - case _ => "server_errors" - } - } - - private[this] val preServoExceptionCountersWithClientId = - new MemoizedExceptionCounterFactory(stats) - private[this] val preServoExceptionCounters = - new MemoizedExceptionCounterFactory(stats, categorizer = ExceptionCounter.defaultCategorizer) - private[this] val postServoExceptionCounters = - new MemoizedExceptionCounterFactory(stats, categorizer = clientServerCategorizer) - - private def clientId: String = - clientIdHelper.effectiveClientId.getOrElse(ClientIdHelper.UnknownClientId) - private def clientIdRoot: String = - clientIdHelper.effectiveClientIdRoot.getOrElse(ClientIdHelper.UnknownClientId) - - private[this] val futureOverCapacityException = - Future.exception(OverCapacity("Request rejected due to load shedding.")) - - private[this] def ifNotOverCapacityRead[T]( - methodStats: StatsReceiver, - requestSize: Long - )( - f: => Future[T] - ): Future[T] = { - val couldShed = loadShedEligible(clientId) - val doShed = couldShed && shedReadTrafficVoluntarily() - - methodStats.stat("loadshed_incoming_requests").add(requestSize) - if (couldShed) { - methodStats.stat("loadshed_eligible_requests").add(requestSize) - } else { - methodStats.stat("loadshed_ineligible_requests").add(requestSize) - } - - if (doShed) { - methodStats.stat("loadshed_rejected_requests").add(requestSize) - futureOverCapacityException - } else { - f - } - } - - private def maybeTimeFuture[A](maybeStat: Option[Stat])(f: => Future[A]) = - maybeStat match { - case Some(stat) => Stat.timeFuture(stat)(f) - case None => f - } - - /** - * Perform the action, increment the appropriate counters, and clean up the exceptions to servo exceptions - * - * This method also masks all interrupts to prevent request cancellation on hangup. - */ - private[this] def trackS[T]( - name: String, - requestInfo: Any, - extraStatPrefix: Option[String] = None, - requestSize: Option[Long] = None - )( - action: StatsReceiver => Future[T] - ): Future[T] = { - val methodStats = stats.scope(name) - val clientStats = methodStats.scope(clientIdRoot) - val cancelledCounter = methodStats.counter("cancelled") - - /** - * Returns an identical future except that it ignores interrupts and increments a counter - * when a request is cancelled. This is [[Future]].masked but with a counter. - */ - def maskedWithStats[A](f: Future[A]): Future[A] = { - val p = Promise[A]() - p.setInterruptHandler { - case _: ClientDiscardedRequestException | _: CancelledRequestException => - cancelledCounter.incr() - } - f.proxyTo(p) - p - } - - maskedWithStats( - requestAuthorizer(name, clientIdHelper.effectiveClientId) - .flatMap { _ => - methodStats.counter(Requests).incr() - extraStatPrefix.foreach(p => methodStats.counter(p, Requests).incr()) - clientStats.counter(Requests).incr() - StratoStatsCounter.recordStats(name, "tweets", requestSize.getOrElse(1L)) - - Stat.timeFuture(methodStats.stat(Latency)) { - Stat.timeFuture(clientStats.stat(Latency)) { - maybeTimeFuture(extraStatPrefix.map(p => methodStats.stat(p, Latency))) { - TweetypieContext.Local.trackStats(stats, methodStats, clientStats) - - // Remove the deadline for backend requests when we mask client cancellations so - // that side-effects are applied to all backend services even after client timeouts. - // Wrap and then flatten an extra layer of Future to capture any thrown exceptions. - Future(Contexts.broadcast.letClear(Deadline)(action(methodStats))).flatten - } - } - } - } - ).onSuccess { _ => - methodStats.counter(Success).incr() - extraStatPrefix.foreach(p => methodStats.counter(p, Success).incr()) - clientStats.counter(Success).incr() - } - .onFailure { e => - preServoExceptionCounters(name)(e) - preServoExceptionCountersWithClientId(name, clientIdRoot)(e) - } - .rescue(rescueToServoFailure(name, clientId)) - .onFailure { e => - postServoExceptionCounters(name)(e) - logFailure(e, requestInfo) - } - } - - def track[T]( - name: String, - requestInfo: Any, - extraStatPrefix: Option[String] = None, - requestSize: Option[Long] = None - )( - action: => Future[T] - ): Future[T] = { - trackS(name, requestInfo, extraStatPrefix, requestSize) { _: StatsReceiver => action } - } - - private def logFailure(ex: Throwable, requestInfo: Any): Unit = - log.warn(s"Returning failure response: $ex\n failed request info: $requestInfo") - - object RequestWidthPrefix { - private def prefix(width: Int) = { - val bucketMin = - width match { - case c if c < 10 => "0_9" - case c if c < 100 => "10_99" - case _ => "100_plus" - } - s"width_$bucketMin" - } - - def forGetTweetsRequest(r: GetTweetsRequest): String = prefix(r.tweetIds.size) - def forGetTweetFieldsRequest(r: GetTweetFieldsRequest): String = prefix(r.tweetIds.size) - } - - object WithMediaPrefix { - def forPostTweetRequest(r: PostTweetRequest): String = - if (r.mediaUploadIds.exists(_.nonEmpty)) - "with_media" - else - "without_media" - } - - override def getTweets(request: GetTweetsRequest): Future[Seq[GetTweetResult]] = - trackS( - "get_tweets", - request, - Some(RequestWidthPrefix.forGetTweetsRequest(request)), - Some(request.tweetIds.size) - ) { stats => - getTweetsAuthorizer(request, clientId).flatMap { _ => - ifNotOverCapacityRead(stats, request.tweetIds.length) { - underlying.getTweets(request) - } - } - } - - override def getTweetFields(request: GetTweetFieldsRequest): Future[Seq[GetTweetFieldsResult]] = - trackS( - "get_tweet_fields", - request, - Some(RequestWidthPrefix.forGetTweetFieldsRequest(request)), - Some(request.tweetIds.size) - ) { stats => - getTweetFieldsAuthorizer(request, clientId).flatMap { _ => - ifNotOverCapacityRead(stats, request.tweetIds.length) { - underlying.getTweetFields(request) - } - } - } - - override def replicatedGetTweets(request: GetTweetsRequest): Future[Unit] = - track("replicated_get_tweets", request, requestSize = Some(request.tweetIds.size)) { - underlying.replicatedGetTweets(request).rescue { - case e: Throwable => Future.Unit // do not need deferredrpc to retry on exceptions - } - } - - override def replicatedGetTweetFields(request: GetTweetFieldsRequest): Future[Unit] = - track("replicated_get_tweet_fields", request, requestSize = Some(request.tweetIds.size)) { - underlying.replicatedGetTweetFields(request).rescue { - case e: Throwable => Future.Unit // do not need deferredrpc to retry on exceptions - } - } - - override def getTweetCounts(request: GetTweetCountsRequest): Future[Seq[GetTweetCountsResult]] = - trackS("get_tweet_counts", request, requestSize = Some(request.tweetIds.size)) { stats => - ifNotOverCapacityRead(stats, request.tweetIds.length) { - requestSizeAuthorizer(request.tweetIds.size, clientId).flatMap { _ => - underlying.getTweetCounts(request) - } - } - } - - override def replicatedGetTweetCounts(request: GetTweetCountsRequest): Future[Unit] = - track("replicated_get_tweet_counts", request, requestSize = Some(request.tweetIds.size)) { - underlying.replicatedGetTweetCounts(request).rescue { - case e: Throwable => Future.Unit // do not need deferredrpc to retry on exceptions - } - } - - override def postTweet(request: PostTweetRequest): Future[PostTweetResult] = - track("post_tweet", request, Some(WithMediaPrefix.forPostTweetRequest(request))) { - underlying.postTweet(request) - } - - override def postRetweet(request: RetweetRequest): Future[PostTweetResult] = - track("post_retweet", request) { - underlying.postRetweet(request) - } - - override def setAdditionalFields(request: SetAdditionalFieldsRequest): Future[Unit] = - track("set_additional_fields", request) { - underlying.setAdditionalFields(request) - } - - override def deleteAdditionalFields(request: DeleteAdditionalFieldsRequest): Future[Unit] = - track("delete_additional_fields", request, requestSize = Some(request.tweetIds.size)) { - requestSizeAuthorizer(request.tweetIds.size, clientId).flatMap { _ => - underlying.deleteAdditionalFields(request) - } - } - - override def asyncSetAdditionalFields(request: AsyncSetAdditionalFieldsRequest): Future[Unit] = - track("async_set_additional_fields", request) { - underlying.asyncSetAdditionalFields(request) - } - - override def asyncDeleteAdditionalFields( - request: AsyncDeleteAdditionalFieldsRequest - ): Future[Unit] = - track("async_delete_additional_fields", request) { - underlying.asyncDeleteAdditionalFields(request) - } - - override def replicatedUndeleteTweet2(request: ReplicatedUndeleteTweet2Request): Future[Unit] = - track("replicated_undelete_tweet2", request) { underlying.replicatedUndeleteTweet2(request) } - - override def replicatedInsertTweet2(request: ReplicatedInsertTweet2Request): Future[Unit] = - track("replicated_insert_tweet2", request) { underlying.replicatedInsertTweet2(request) } - - override def asyncInsert(request: AsyncInsertRequest): Future[Unit] = - track("async_insert", request) { underlying.asyncInsert(request) } - - override def updatePossiblySensitiveTweet( - request: UpdatePossiblySensitiveTweetRequest - ): Future[Unit] = - track("update_possibly_sensitive_tweet", request) { - underlying.updatePossiblySensitiveTweet(request) - } - - override def asyncUpdatePossiblySensitiveTweet( - request: AsyncUpdatePossiblySensitiveTweetRequest - ): Future[Unit] = - track("async_update_possibly_sensitive_tweet", request) { - underlying.asyncUpdatePossiblySensitiveTweet(request) - } - - override def replicatedUpdatePossiblySensitiveTweet(tweet: Tweet): Future[Unit] = - track("replicated_update_possibly_sensitive_tweet", tweet) { - underlying.replicatedUpdatePossiblySensitiveTweet(tweet) - } - - override def undeleteTweet(request: UndeleteTweetRequest): Future[UndeleteTweetResponse] = - track("undelete_tweet", request) { - underlying.undeleteTweet(request) - } - - override def asyncUndeleteTweet(request: AsyncUndeleteTweetRequest): Future[Unit] = - track("async_undelete_tweet", request) { - underlying.asyncUndeleteTweet(request) - } - - override def unretweet(request: UnretweetRequest): Future[UnretweetResult] = - track("unretweet", request) { - underlying.unretweet(request) - } - - override def eraseUserTweets(request: EraseUserTweetsRequest): Future[Unit] = - track("erase_user_tweets", request) { - underlying.eraseUserTweets(request) - } - - override def asyncEraseUserTweets(request: AsyncEraseUserTweetsRequest): Future[Unit] = - track("async_erase_user_tweets", request) { - underlying.asyncEraseUserTweets(request) - } - - override def asyncDelete(request: AsyncDeleteRequest): Future[Unit] = - track("async_delete", request) { underlying.asyncDelete(request) } - - override def deleteTweets(request: DeleteTweetsRequest): Future[Seq[DeleteTweetResult]] = - track("delete_tweets", request, requestSize = Some(request.tweetIds.size)) { - requestSizeAuthorizer(request.tweetIds.size, clientId).flatMap { _ => - underlying.deleteTweets(request) - } - } - - override def cascadedDeleteTweet(request: CascadedDeleteTweetRequest): Future[Unit] = - track("cascaded_delete_tweet", request) { underlying.cascadedDeleteTweet(request) } - - override def replicatedDeleteTweet2(request: ReplicatedDeleteTweet2Request): Future[Unit] = - track("replicated_delete_tweet2", request) { underlying.replicatedDeleteTweet2(request) } - - override def incrTweetFavCount(request: IncrTweetFavCountRequest): Future[Unit] = - track("incr_tweet_fav_count", request) { underlying.incrTweetFavCount(request) } - - override def asyncIncrFavCount(request: AsyncIncrFavCountRequest): Future[Unit] = - track("async_incr_fav_count", request) { underlying.asyncIncrFavCount(request) } - - override def replicatedIncrFavCount(tweetId: TweetId, delta: Int): Future[Unit] = - track("replicated_incr_fav_count", tweetId) { - underlying.replicatedIncrFavCount(tweetId, delta) - } - - override def incrTweetBookmarkCount(request: IncrTweetBookmarkCountRequest): Future[Unit] = - track("incr_tweet_bookmark_count", request) { underlying.incrTweetBookmarkCount(request) } - - override def asyncIncrBookmarkCount(request: AsyncIncrBookmarkCountRequest): Future[Unit] = - track("async_incr_bookmark_count", request) { underlying.asyncIncrBookmarkCount(request) } - - override def replicatedIncrBookmarkCount(tweetId: TweetId, delta: Int): Future[Unit] = - track("replicated_incr_bookmark_count", tweetId) { - underlying.replicatedIncrBookmarkCount(tweetId, delta) - } - - override def replicatedSetAdditionalFields(request: SetAdditionalFieldsRequest): Future[Unit] = - track("replicated_set_additional_fields", request) { - underlying.replicatedSetAdditionalFields(request) - } - - def setRetweetVisibility(request: SetRetweetVisibilityRequest): Future[Unit] = { - track("set_retweet_visibility", request) { - underlying.setRetweetVisibility(request) - } - } - - def asyncSetRetweetVisibility(request: AsyncSetRetweetVisibilityRequest): Future[Unit] = { - track("async_set_retweet_visibility", request) { - underlying.asyncSetRetweetVisibility(request) - } - } - - override def replicatedSetRetweetVisibility( - request: ReplicatedSetRetweetVisibilityRequest - ): Future[Unit] = - track("replicated_set_retweet_visibility", request) { - underlying.replicatedSetRetweetVisibility(request) - } - - override def replicatedDeleteAdditionalFields( - request: ReplicatedDeleteAdditionalFieldsRequest - ): Future[Unit] = - track("replicated_delete_additional_fields", request) { - underlying.replicatedDeleteAdditionalFields(request) - } - - override def replicatedTakedown(tweet: Tweet): Future[Unit] = - track("replicated_takedown", tweet) { underlying.replicatedTakedown(tweet) } - - override def scrubGeoUpdateUserTimestamp(request: DeleteLocationData): Future[Unit] = - track("scrub_geo_update_user_timestamp", request) { - underlying.scrubGeoUpdateUserTimestamp(request) - } - - override def scrubGeo(request: GeoScrub): Future[Unit] = - track("scrub_geo", request, requestSize = Some(request.statusIds.size)) { - requestSizeAuthorizer(request.statusIds.size, clientId).flatMap { _ => - underlying.scrubGeo(request) - } - } - - override def replicatedScrubGeo(tweetIds: Seq[TweetId]): Future[Unit] = - track("replicated_scrub_geo", tweetIds) { underlying.replicatedScrubGeo(tweetIds) } - - override def deleteLocationData(request: DeleteLocationDataRequest): Future[Unit] = - track("delete_location_data", request) { - underlying.deleteLocationData(request) - } - - override def flush(request: FlushRequest): Future[Unit] = - track("flush", request, requestSize = Some(request.tweetIds.size)) { - requestSizeAuthorizer(request.tweetIds.size, clientId).flatMap { _ => - underlying.flush(request) - } - } - - override def takedown(request: TakedownRequest): Future[Unit] = - track("takedown", request) { underlying.takedown(request) } - - override def asyncTakedown(request: AsyncTakedownRequest): Future[Unit] = - track("async_takedown", request) { - underlying.asyncTakedown(request) - } - - override def setTweetUserTakedown(request: SetTweetUserTakedownRequest): Future[Unit] = - track("set_tweet_user_takedown", request) { underlying.setTweetUserTakedown(request) } - - override def quotedTweetDelete(request: QuotedTweetDeleteRequest): Future[Unit] = - track("quoted_tweet_delete", request) { - underlying.quotedTweetDelete(request) - } - - override def quotedTweetTakedown(request: QuotedTweetTakedownRequest): Future[Unit] = - track("quoted_tweet_takedown", request) { - underlying.quotedTweetTakedown(request) - } - - override def getDeletedTweets( - request: GetDeletedTweetsRequest - ): Future[Seq[GetDeletedTweetResult]] = - track("get_deleted_tweets", request, requestSize = Some(request.tweetIds.size)) { - requestSizeAuthorizer(request.tweetIds.size, clientId).flatMap { _ => - underlying.getDeletedTweets(request) - } - } - - override def getStoredTweets( - request: GetStoredTweetsRequest - ): Future[Seq[GetStoredTweetsResult]] = { - track("get_stored_tweets", request, requestSize = Some(request.tweetIds.size)) { - requestSizeAuthorizer(request.tweetIds.size, clientId).flatMap { _ => - underlying.getStoredTweets(request) - } - } - } - - override def getStoredTweetsByUser( - request: GetStoredTweetsByUserRequest - ): Future[GetStoredTweetsByUserResult] = { - track("get_stored_tweets_by_user", request) { - underlying.getStoredTweetsByUser(request) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/DispatchingTweetService.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/DispatchingTweetService.docx new file mode 100644 index 000000000..b07e68411 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/DispatchingTweetService.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/DispatchingTweetService.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/DispatchingTweetService.scala deleted file mode 100644 index f148fb25a..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/DispatchingTweetService.scala +++ /dev/null @@ -1,376 +0,0 @@ -/** Copyright 2010 Twitter, Inc. */ -package com.twitter.tweetypie -package service - -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.servo.exception.thriftscala.ClientErrorCause -import com.twitter.tweetypie.additionalfields.AdditionalFields -import com.twitter.tweetypie.client_id.ClientIdHelper -import com.twitter.tweetypie.handler._ -import com.twitter.tweetypie.store._ -import com.twitter.tweetypie.thriftscala._ -import com.twitter.util.Future - -/** - * Implementation of the TweetService which dispatches requests to underlying - * handlers and stores. - */ -class DispatchingTweetService( - asyncDeleteAdditionalFieldsBuilder: AsyncDeleteAdditionalFieldsBuilder.Type, - asyncSetAdditionalFieldsBuilder: AsyncSetAdditionalFieldsBuilder.Type, - deleteAdditionalFieldsBuilder: DeleteAdditionalFieldsBuilder.Type, - deleteLocationDataHandler: DeleteLocationDataHandler.Type, - deletePathHandler: TweetDeletePathHandler, - eraseUserTweetsHandler: EraseUserTweetsHandler, - getDeletedTweetsHandler: GetDeletedTweetsHandler.Type, - getStoredTweetsHandler: GetStoredTweetsHandler.Type, - getStoredTweetsByUserHandler: GetStoredTweetsByUserHandler.Type, - getTweetCountsHandler: GetTweetCountsHandler.Type, - getTweetsHandler: GetTweetsHandler.Type, - getTweetFieldsHandler: GetTweetFieldsHandler.Type, - postTweetHandler: PostTweet.Type[PostTweetRequest], - postRetweetHandler: PostTweet.Type[RetweetRequest], - quotedTweetDeleteBuilder: QuotedTweetDeleteEventBuilder.Type, - quotedTweetTakedownBuilder: QuotedTweetTakedownEventBuilder.Type, - scrubGeoScrubTweetsBuilder: ScrubGeoEventBuilder.ScrubTweets.Type, - scrubGeoUpdateUserTimestampBuilder: ScrubGeoEventBuilder.UpdateUserTimestamp.Type, - setAdditionalFieldsBuilder: SetAdditionalFieldsBuilder.Type, - setRetweetVisibilityHandler: SetRetweetVisibilityHandler.Type, - statsReceiver: StatsReceiver, - takedownHandler: TakedownHandler.Type, - tweetStore: TotalTweetStore, - undeleteTweetHandler: UndeleteTweetHandler.Type, - unretweetHandler: UnretweetHandler.Type, - updatePossiblySensitiveTweetHandler: UpdatePossiblySensitiveTweetHandler.Type, - userTakedownHandler: UserTakedownHandler.Type, - clientIdHelper: ClientIdHelper) - extends ThriftTweetService { - import AdditionalFields._ - - // Incoming reads - - override def getTweets(request: GetTweetsRequest): Future[Seq[GetTweetResult]] = - getTweetsHandler(request) - - override def getTweetFields(request: GetTweetFieldsRequest): Future[Seq[GetTweetFieldsResult]] = - getTweetFieldsHandler(request) - - override def getTweetCounts(request: GetTweetCountsRequest): Future[Seq[GetTweetCountsResult]] = - getTweetCountsHandler(request) - - // Incoming deletes - - override def cascadedDeleteTweet(request: CascadedDeleteTweetRequest): Future[Unit] = - deletePathHandler.cascadedDeleteTweet(request) - - override def deleteTweets(request: DeleteTweetsRequest): Future[Seq[DeleteTweetResult]] = - deletePathHandler.deleteTweets(request) - - // Incoming writes - - override def postTweet(request: PostTweetRequest): Future[PostTweetResult] = - postTweetHandler(request) - - override def postRetweet(request: RetweetRequest): Future[PostTweetResult] = - postRetweetHandler(request) - - override def setAdditionalFields(request: SetAdditionalFieldsRequest): Future[Unit] = { - val setFields = AdditionalFields.nonEmptyAdditionalFieldIds(request.additionalFields) - if (setFields.isEmpty) { - Future.exception( - ClientError( - ClientErrorCause.BadRequest, - s"${SetAdditionalFieldsRequest.AdditionalFieldsField.name} is empty, there must be at least one field to set" - ) - ) - } else { - - unsettableAdditionalFieldIds(request.additionalFields) match { - case Nil => - setAdditionalFieldsBuilder(request).flatMap(tweetStore.setAdditionalFields) - case unsettableFieldIds => - Future.exception( - ClientError( - ClientErrorCause.BadRequest, - unsettableAdditionalFieldIdsErrorMessage(unsettableFieldIds) - ) - ) - } - } - } - - override def deleteAdditionalFields(request: DeleteAdditionalFieldsRequest): Future[Unit] = - if (request.tweetIds.isEmpty || request.fieldIds.isEmpty) { - Future.exception( - ClientError(ClientErrorCause.BadRequest, "request contains empty tweet ids or field ids") - ) - } else if (request.fieldIds.exists(!isAdditionalFieldId(_))) { - Future.exception( - ClientError(ClientErrorCause.BadRequest, "cannot delete non-additional fields") - ) - } else { - deleteAdditionalFieldsBuilder(request).flatMap { events => - Future.join(events.map(tweetStore.deleteAdditionalFields)) - } - } - - override def asyncInsert(request: AsyncInsertRequest): Future[Unit] = - AsyncInsertTweet.Event.fromAsyncRequest(request) match { - case TweetStoreEventOrRetry.First(e) => tweetStore.asyncInsertTweet(e) - case TweetStoreEventOrRetry.Retry(e) => tweetStore.retryAsyncInsertTweet(e) - } - - override def asyncSetAdditionalFields(request: AsyncSetAdditionalFieldsRequest): Future[Unit] = - asyncSetAdditionalFieldsBuilder(request).map { - case TweetStoreEventOrRetry.First(e) => tweetStore.asyncSetAdditionalFields(e) - case TweetStoreEventOrRetry.Retry(e) => tweetStore.retryAsyncSetAdditionalFields(e) - } - - /** - * Set if a retweet should be included in its source tweet's retweet count. - * - * This is called by our RetweetVisibility daemon when a user enter/exit - * suspended or read-only state and all their retweets visibility need to - * be modified. - * - * @see [[SetRetweetVisibilityHandler]] for more implementation details - */ - override def setRetweetVisibility(request: SetRetweetVisibilityRequest): Future[Unit] = - setRetweetVisibilityHandler(request) - - override def asyncSetRetweetVisibility(request: AsyncSetRetweetVisibilityRequest): Future[Unit] = - AsyncSetRetweetVisibility.Event.fromAsyncRequest(request) match { - case TweetStoreEventOrRetry.First(e) => tweetStore.asyncSetRetweetVisibility(e) - case TweetStoreEventOrRetry.Retry(e) => tweetStore.retryAsyncSetRetweetVisibility(e) - } - - /** - * When a tweet has been successfully undeleted from storage in Manhattan this endpoint will - * enqueue requests to three related endpoints via deferredRPC: - * - * 1. asyncUndeleteTweet: Asynchronously handle aspects of the undelete not required for the response. - * 2. replicatedUndeleteTweet2: Send the undeleted tweet to other clusters for cache caching. - * - * @see [[UndeleteTweetHandler]] for the core undelete implementation - */ - override def undeleteTweet(request: UndeleteTweetRequest): Future[UndeleteTweetResponse] = - undeleteTweetHandler(request) - - /** - * The async method that undeleteTweet calls to handle notifiying other services of the undelete - * See [[TweetStores.asyncUndeleteTweetStore]] for all the stores that handle this event. - */ - override def asyncUndeleteTweet(request: AsyncUndeleteTweetRequest): Future[Unit] = - AsyncUndeleteTweet.Event.fromAsyncRequest(request) match { - case TweetStoreEventOrRetry.First(e) => tweetStore.asyncUndeleteTweet(e) - case TweetStoreEventOrRetry.Retry(e) => tweetStore.retryAsyncUndeleteTweet(e) - } - - override def getDeletedTweets( - request: GetDeletedTweetsRequest - ): Future[Seq[GetDeletedTweetResult]] = - getDeletedTweetsHandler(request) - - /** - * Triggers the deletion of all of a users tweets. Used by Gizmoduck when erasing a user - * after they have been deactived for some number of days. - */ - override def eraseUserTweets(request: EraseUserTweetsRequest): Future[Unit] = - eraseUserTweetsHandler.eraseUserTweetsRequest(request) - - override def asyncEraseUserTweets(request: AsyncEraseUserTweetsRequest): Future[Unit] = - eraseUserTweetsHandler.asyncEraseUserTweetsRequest(request) - - override def asyncDelete(request: AsyncDeleteRequest): Future[Unit] = - AsyncDeleteTweet.Event.fromAsyncRequest(request) match { - case TweetStoreEventOrRetry.First(e) => tweetStore.asyncDeleteTweet(e) - case TweetStoreEventOrRetry.Retry(e) => tweetStore.retryAsyncDeleteTweet(e) - } - - /* - * unretweet a tweet. - * - * There are two ways to unretweet: - * - call deleteTweets() with the retweetId - * - call unretweet() with the retweeter userId and sourceTweetId - * - * This is useful if you want to be able to undo a retweet without having to - * keep track of a retweetId - * - * Returns DeleteTweetResult for any deleted retweets. - */ - override def unretweet(request: UnretweetRequest): Future[UnretweetResult] = - unretweetHandler(request) - - override def asyncDeleteAdditionalFields( - request: AsyncDeleteAdditionalFieldsRequest - ): Future[Unit] = - asyncDeleteAdditionalFieldsBuilder(request).map { - case TweetStoreEventOrRetry.First(e) => tweetStore.asyncDeleteAdditionalFields(e) - case TweetStoreEventOrRetry.Retry(e) => tweetStore.retryAsyncDeleteAdditionalFields(e) - } - - override def incrTweetFavCount(request: IncrTweetFavCountRequest): Future[Unit] = - tweetStore.incrFavCount(IncrFavCount.Event(request.tweetId, request.delta, Time.now)) - - override def asyncIncrFavCount(request: AsyncIncrFavCountRequest): Future[Unit] = - tweetStore.asyncIncrFavCount(AsyncIncrFavCount.Event(request.tweetId, request.delta, Time.now)) - - override def incrTweetBookmarkCount(request: IncrTweetBookmarkCountRequest): Future[Unit] = - tweetStore.incrBookmarkCount(IncrBookmarkCount.Event(request.tweetId, request.delta, Time.now)) - - override def asyncIncrBookmarkCount(request: AsyncIncrBookmarkCountRequest): Future[Unit] = - tweetStore.asyncIncrBookmarkCount( - AsyncIncrBookmarkCount.Event(request.tweetId, request.delta, Time.now)) - - override def scrubGeoUpdateUserTimestamp(request: DeleteLocationData): Future[Unit] = - scrubGeoUpdateUserTimestampBuilder(request).flatMap(tweetStore.scrubGeoUpdateUserTimestamp) - - override def deleteLocationData(request: DeleteLocationDataRequest): Future[Unit] = - deleteLocationDataHandler(request) - - override def scrubGeo(request: GeoScrub): Future[Unit] = - scrubGeoScrubTweetsBuilder(request).flatMap(tweetStore.scrubGeo) - - override def takedown(request: TakedownRequest): Future[Unit] = - takedownHandler(request) - - override def quotedTweetDelete(request: QuotedTweetDeleteRequest): Future[Unit] = - quotedTweetDeleteBuilder(request).flatMap { - case Some(event) => tweetStore.quotedTweetDelete(event) - case None => Future.Unit - } - - override def quotedTweetTakedown(request: QuotedTweetTakedownRequest): Future[Unit] = - quotedTweetTakedownBuilder(request).flatMap { - case Some(event) => tweetStore.quotedTweetTakedown(event) - case None => Future.Unit - } - - override def asyncTakedown(request: AsyncTakedownRequest): Future[Unit] = - AsyncTakedown.Event.fromAsyncRequest(request) match { - case TweetStoreEventOrRetry.First(e) => tweetStore.asyncTakedown(e) - case TweetStoreEventOrRetry.Retry(e) => tweetStore.retryAsyncTakedown(e) - } - - override def setTweetUserTakedown(request: SetTweetUserTakedownRequest): Future[Unit] = - userTakedownHandler(request) - - override def asyncUpdatePossiblySensitiveTweet( - request: AsyncUpdatePossiblySensitiveTweetRequest - ): Future[Unit] = { - AsyncUpdatePossiblySensitiveTweet.Event.fromAsyncRequest(request) match { - case TweetStoreEventOrRetry.First(event) => - tweetStore.asyncUpdatePossiblySensitiveTweet(event) - case TweetStoreEventOrRetry.Retry(event) => - tweetStore.retryAsyncUpdatePossiblySensitiveTweet(event) - } - } - - override def flush(request: FlushRequest): Future[Unit] = { - // The logged "previous Tweet" value is intended to be used when interactively debugging an - // issue and an engineer flushes the tweet manually, e.g. from tweetypie.cmdline console. - // Don't log automated flushes originating from tweetypie-daemons to cut down noise. - val logExisting = !clientIdHelper.effectiveClientIdRoot.exists(_ == "tweetypie-daemons") - tweetStore.flush( - Flush.Event(request.tweetIds, request.flushTweets, request.flushCounts, logExisting) - ) - } - - // Incoming replication events - - override def replicatedGetTweetCounts(request: GetTweetCountsRequest): Future[Unit] = - getTweetCounts(request).unit - - override def replicatedGetTweetFields(request: GetTweetFieldsRequest): Future[Unit] = - getTweetFields(request).unit - - override def replicatedGetTweets(request: GetTweetsRequest): Future[Unit] = - getTweets(request).unit - - override def replicatedInsertTweet2(request: ReplicatedInsertTweet2Request): Future[Unit] = - tweetStore.replicatedInsertTweet( - ReplicatedInsertTweet - .Event( - request.cachedTweet.tweet, - request.cachedTweet, - request.quoterHasAlreadyQuotedTweet.getOrElse(false), - request.initialTweetUpdateRequest - ) - ) - - override def replicatedDeleteTweet2(request: ReplicatedDeleteTweet2Request): Future[Unit] = - tweetStore.replicatedDeleteTweet( - ReplicatedDeleteTweet.Event( - tweet = request.tweet, - isErasure = request.isErasure, - isBounceDelete = request.isBounceDelete, - isLastQuoteOfQuoter = request.isLastQuoteOfQuoter.getOrElse(false) - ) - ) - - override def replicatedIncrFavCount(tweetId: TweetId, delta: Int): Future[Unit] = - tweetStore.replicatedIncrFavCount(ReplicatedIncrFavCount.Event(tweetId, delta)) - - override def replicatedIncrBookmarkCount(tweetId: TweetId, delta: Int): Future[Unit] = - tweetStore.replicatedIncrBookmarkCount(ReplicatedIncrBookmarkCount.Event(tweetId, delta)) - - override def replicatedScrubGeo(tweetIds: Seq[TweetId]): Future[Unit] = - tweetStore.replicatedScrubGeo(ReplicatedScrubGeo.Event(tweetIds)) - - override def replicatedSetAdditionalFields(request: SetAdditionalFieldsRequest): Future[Unit] = - tweetStore.replicatedSetAdditionalFields( - ReplicatedSetAdditionalFields.Event(request.additionalFields) - ) - - override def replicatedSetRetweetVisibility( - request: ReplicatedSetRetweetVisibilityRequest - ): Future[Unit] = - tweetStore.replicatedSetRetweetVisibility( - ReplicatedSetRetweetVisibility.Event(request.srcId, request.visible) - ) - - override def replicatedDeleteAdditionalFields( - request: ReplicatedDeleteAdditionalFieldsRequest - ): Future[Unit] = - Future.join( - request.fieldsMap.map { - case (tweetId, fieldIds) => - tweetStore.replicatedDeleteAdditionalFields( - ReplicatedDeleteAdditionalFields.Event(tweetId, fieldIds) - ) - }.toSeq - ) - - override def replicatedUndeleteTweet2(request: ReplicatedUndeleteTweet2Request): Future[Unit] = - tweetStore.replicatedUndeleteTweet( - ReplicatedUndeleteTweet - .Event( - request.cachedTweet.tweet, - request.cachedTweet, - request.quoterHasAlreadyQuotedTweet.getOrElse(false) - )) - - override def replicatedTakedown(tweet: Tweet): Future[Unit] = - tweetStore.replicatedTakedown(ReplicatedTakedown.Event(tweet)) - - override def updatePossiblySensitiveTweet( - request: UpdatePossiblySensitiveTweetRequest - ): Future[Unit] = - updatePossiblySensitiveTweetHandler(request) - - override def replicatedUpdatePossiblySensitiveTweet(tweet: Tweet): Future[Unit] = - tweetStore.replicatedUpdatePossiblySensitiveTweet( - ReplicatedUpdatePossiblySensitiveTweet.Event(tweet) - ) - - override def getStoredTweets( - request: GetStoredTweetsRequest - ): Future[Seq[GetStoredTweetsResult]] = - getStoredTweetsHandler(request) - - override def getStoredTweetsByUser( - request: GetStoredTweetsByUserRequest - ): Future[GetStoredTweetsByUserResult] = - getStoredTweetsByUserHandler(request) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/FailureLoggingTweetService.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/FailureLoggingTweetService.docx new file mode 100644 index 000000000..1c983b797 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/FailureLoggingTweetService.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/FailureLoggingTweetService.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/FailureLoggingTweetService.scala deleted file mode 100644 index c1dd98151..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/FailureLoggingTweetService.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.tweetypie -package service - -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.coreservices.failed_task.writer.FailedTaskWriter -import com.twitter.scrooge.ThriftException -import com.twitter.scrooge.ThriftStruct -import com.twitter.scrooge.ThriftStructCodec -import com.twitter.tweetypie.serverutil.BoringStackTrace -import com.twitter.tweetypie.thriftscala._ -import scala.util.control.NoStackTrace - -object FailureLoggingTweetService { - - /** - * Defines the universe of exception types for which we should scribe - * the failure. - */ - private def shouldWrite(t: Throwable): Boolean = - t match { - case _: ThriftException => true - case _: PostTweetFailure => true - case _ => !BoringStackTrace.isBoring(t) - } - - /** - * Holds failure information from a failing PostTweetResult. - * - * FailedTaskWriter logs an exception with the failed request, so we - * need to package up any failure that we want to log into an - * exception. - */ - private class PostTweetFailure(state: TweetCreateState, reason: Option[String]) - extends Exception - with NoStackTrace { - override def toString: String = s"PostTweetFailure($state, $reason)" - } -} - -/** - * Wraps a tweet service with scribing of failed requests in order to - * enable analysis of failures for diagnosing problems. - */ -class FailureLoggingTweetService( - failedTaskWriter: FailedTaskWriter[Array[Byte]], - protected val underlying: ThriftTweetService) - extends TweetServiceProxy { - import FailureLoggingTweetService._ - - private[this] object writers { - private[this] def writer[T <: ThriftStruct]( - name: String, - codec: ThriftStructCodec[T] - ): (T, Throwable) => Future[Unit] = { - val taskWriter = failedTaskWriter(name, BinaryScalaCodec(codec).apply) - - (t, exc) => - Future.when(shouldWrite(exc)) { - taskWriter.writeFailure(t, exc) - } - } - - val postTweet: (PostTweetRequest, Throwable) => Future[Unit] = - writer("post_tweet", PostTweetRequest) - } - - override def postTweet(request: PostTweetRequest): Future[PostTweetResult] = - underlying.postTweet(request).respond { - // Log requests for states other than OK to enable debugging creation failures - case Return(res) if res.state != TweetCreateState.Ok => - writers.postTweet(request, new PostTweetFailure(res.state, res.failureReason)) - case Throw(exc) => - writers.postTweet(request, exc) - case _ => - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/MethodAuthorizer.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/MethodAuthorizer.docx new file mode 100644 index 000000000..65480f897 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/MethodAuthorizer.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/MethodAuthorizer.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/MethodAuthorizer.scala deleted file mode 100644 index 8b1d2e1db..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/MethodAuthorizer.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.tweetypie -package service - -/** - * An authorizer for determining if a request to a - * method should be rejected. - * - * This class is in the spirit of servo.request.ClientRequestAuthorizer. - * The difference is ClientRequestAuthorizer only operates - * on two pieces of information, clientId and a method name. - * - * This class can be used to create a more complex authorizer that - * operates on the specifics of a request. e.g, an - * authorizer that disallows certain clients from passing - * certain optional flags. - * - * Note: With some work, ClientRequestAuthorizer could be - * generalized to support cases like this. If we end up making - * more method authorizers it might be worth it to - * go that route. - */ -abstract class MethodAuthorizer[T]() { - def apply(request: T, clientId: String): Future[Unit] - - /** - * Created decidered MethodAuthorizer - * if the decider is off it will execute - * MethodAuthorizer.unit, which always succeeds. - */ - def enabledBy(decider: Gate[Unit]): MethodAuthorizer[T] = - MethodAuthorizer.select(decider, this, MethodAuthorizer.unit) - - /** - * Transform this MethodAuthorizer[T] into a MethodAuthorizer[A] - * by providing a function from A => T - */ - def contramap[A](f: A => T): MethodAuthorizer[A] = - MethodAuthorizer[A] { (request, clientId) => this(f(request), clientId) } -} - -object MethodAuthorizer { - - /** - * @param f an authorization function that returns - * Future.Unit if the request is authorized, and Future.exception() - * if the request is not authorized. - * - * @return An instance of MethodAuthorizer with an apply method - * that returns f - */ - def apply[T](f: (T, String) => Future[Unit]): MethodAuthorizer[T] = - new MethodAuthorizer[T]() { - def apply(request: T, clientId: String): Future[Unit] = f(request, clientId) - } - - /** - * @param authorizers A seq of MethodAuthorizers to be - * composed into one. - * @return A MethodAuthorizer that sequentially executes - * all of the authorizers - */ - def all[T](authorizers: Seq[MethodAuthorizer[T]]): MethodAuthorizer[T] = - MethodAuthorizer { (request, clientId) => - authorizers.foldLeft(Future.Unit) { - case (f, authorize) => f.before(authorize(request, clientId)) - } - } - - /** - * @return A MethodAuthorizer that always returns Future.Unit - * Useful if you need to decider off your MethodAuthorizer - * and replace it with one that always passes. - */ - def unit[T]: MethodAuthorizer[T] = MethodAuthorizer { (request, client) => Future.Unit } - - /** - * @return A MethodAuthorizer that switches between two provided - * MethodAuthorizers depending on a decider. - */ - def select[T]( - decider: Gate[Unit], - ifTrue: MethodAuthorizer[T], - ifFalse: MethodAuthorizer[T] - ): MethodAuthorizer[T] = - MethodAuthorizer { (request, client) => - decider.pick( - ifTrue(request, client), - ifFalse(request, client) - ) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ObservedTweetService.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ObservedTweetService.docx new file mode 100644 index 000000000..b229f7749 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ObservedTweetService.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ObservedTweetService.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ObservedTweetService.scala deleted file mode 100644 index d0337076a..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ObservedTweetService.scala +++ /dev/null @@ -1,422 +0,0 @@ -package com.twitter.tweetypie -package service - -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.servo.util.SynchronizedHashMap -import com.twitter.tweetypie.client_id.ClientIdHelper -import com.twitter.tweetypie.service.observer._ -import com.twitter.tweetypie.thriftscala._ -import com.twitter.finagle.tracing.Trace - -/** - * Wraps an underlying TweetService, observing requests and results. - */ -class ObservedTweetService( - protected val underlying: ThriftTweetService, - stats: StatsReceiver, - clientIdHelper: ClientIdHelper) - extends TweetServiceProxy { - - private[this] val asyncEventOrRetryScope = stats.scope("async_event_or_retry") - private[this] val deleteFieldsScope = stats.scope("delete_additional_fields") - private[this] val deleteTweetsScope = stats.scope("delete_tweets") - private[this] val getDeletedTweetsScope = stats.scope("get_deleted_tweets") - private[this] val getTweetCountsScope = stats.scope("get_tweet_counts") - private[this] val getTweetsScope = stats.scope("get_tweets") - private[this] val getTweetFieldsScope = stats.scope("get_tweet_fields") - private[this] val postTweetScope = stats.scope("post_tweet") - private[this] val replicatedInsertTweet2Scope = stats.scope("replicated_insert_tweet2") - private[this] val retweetScope = stats.scope("post_retweet") - private[this] val scrubGeoScope = stats.scope("scrub_geo") - private[this] val setFieldsScope = stats.scope("set_additional_fields") - private[this] val setRetweetVisibilityScope = stats.scope("set_retweet_visibility") - private[this] val getStoredTweetsScope = stats.scope("get_stored_tweets") - private[this] val getStoredTweetsByUserScope = stats.scope("get_stored_tweets_by_user") - - private[this] val defaultGetTweetsRequestOptions = GetTweetOptions() - - /** Increments the appropriate write success/failure counter */ - private[this] val observeWriteResult: Effect[Try[_]] = { - withAndWithoutClientId(stats) { (stats, _) => - val successCounter = stats.counter("write_successes") - val failureCounter = stats.counter("write_failures") - val clientErrorCounter = stats.counter("write_client_errors") - Effect[Try[_]] { - case Return(_) => successCounter.incr() - case Throw(ClientError(_, _)) | Throw(AccessDenied(_, _)) => clientErrorCounter.incr() - case Throw(_) => failureCounter.incr() - } - } - } - - /** Increments the tweet_creates counter on future success. */ - private[this] val observeTweetWriteSuccess: Effect[Any] = { - withAndWithoutClientId(stats) { (stats, _) => - val counter = stats.counter("tweet_writes") - Effect[Any] { _ => counter.incr() } - } - } - - private[this] val observeGetTweetsRequest = - withAndWithoutClientId(getTweetsScope) { - GetTweetsObserver.observeRequest - } - - private[this] val observeGetTweetFieldsRequest = - withAndWithoutClientId(getTweetFieldsScope) { - GetTweetFieldsObserver.observeRequest - } - - private[this] val observeGetTweetCountsRequest = - withAndWithoutClientId(getTweetCountsScope) { (s, _) => - GetTweetCountsObserver.observeRequest(s) - } - - private[this] val observeRetweetRequest: Effect[RetweetRequest] = - withAndWithoutClientId(retweetScope) { (s, _) => Observer.observeRetweetRequest(s) } - - private[this] val observeDeleteTweetsRequest = - withAndWithoutClientId(deleteTweetsScope) { (s, _) => Observer.observeDeleteTweetsRequest(s) } - - private[this] val observeSetFieldsRequest: Effect[SetAdditionalFieldsRequest] = - withAndWithoutClientId(setFieldsScope) { (s, _) => Observer.observeSetFieldsRequest(s) } - - private[this] val observeSetRetweetVisibilityRequest: Effect[SetRetweetVisibilityRequest] = - withAndWithoutClientId(setRetweetVisibilityScope) { (s, _) => - Observer.observeSetRetweetVisibilityRequest(s) - } - - private[this] val observeDeleteFieldsRequest: Effect[DeleteAdditionalFieldsRequest] = - withAndWithoutClientId(deleteFieldsScope) { (s, _) => Observer.observeDeleteFieldsRequest(s) } - - private[this] val observePostTweetAdditionals: Effect[Tweet] = - withAndWithoutClientId(postTweetScope) { (s, _) => Observer.observeAdditionalFields(s) } - - private[this] val observePostTweetRequest: Effect[PostTweetRequest] = - withAndWithoutClientId(postTweetScope) { (s, _) => PostTweetObserver.observerRequest(s) } - - private[this] val observeGetTweetResults = - withAndWithoutClientId(getTweetsScope) { - GetTweetsObserver.observeResults - } - - private[this] val observeGetTweetFieldsResults: Effect[Seq[GetTweetFieldsResult]] = - GetTweetFieldsObserver.observeResults(getTweetFieldsScope) - - private[this] val observeTweetCountsResults = - GetTweetCountsObserver.observeResults(getTweetCountsScope) - - private[this] val observeScrubGeoRequest = - Observer.observeScrubGeo(scrubGeoScope) - - private[this] val observeRetweetResponse = - PostTweetObserver.observeResults(retweetScope, byClient = false) - - private[this] val observePostTweetResponse = - PostTweetObserver.observeResults(postTweetScope, byClient = false) - - private[this] val observeAsyncInsertRequest = - Observer.observeAsyncInsertRequest(asyncEventOrRetryScope) - - private[this] val observeAsyncSetAdditionalFieldsRequest = - Observer.observeAsyncSetAdditionalFieldsRequest(asyncEventOrRetryScope) - - private[this] val observeAsyncSetRetweetVisibilityRequest = - Observer.observeAsyncSetRetweetVisibilityRequest(asyncEventOrRetryScope) - - private[this] val observeAsyncUndeleteTweetRequest = - Observer.observeAsyncUndeleteTweetRequest(asyncEventOrRetryScope) - - private[this] val observeAsyncDeleteTweetRequest = - Observer.observeAsyncDeleteTweetRequest(asyncEventOrRetryScope) - - private[this] val observeAsyncDeleteAdditionalFieldsRequest = - Observer.observeAsyncDeleteAdditionalFieldsRequest(asyncEventOrRetryScope) - - private[this] val observeAsyncTakedownRequest = - Observer.observeAsyncTakedownRequest(asyncEventOrRetryScope) - - private[this] val observeAsyncUpdatePossiblySensitiveTweetRequest = - Observer.observeAsyncUpdatePossiblySensitiveTweetRequest(asyncEventOrRetryScope) - - private[this] val observedReplicatedInsertTweet2Request = - Observer.observeReplicatedInsertTweetRequest(replicatedInsertTweet2Scope) - - private[this] val observeGetTweetFieldsResultState: Effect[GetTweetFieldsObserver.Type] = - withAndWithoutClientId(getTweetFieldsScope) { (statsReceiver, _) => - GetTweetFieldsObserver.observeExchange(statsReceiver) - } - - private[this] val observeGetTweetsResultState: Effect[GetTweetsObserver.Type] = - withAndWithoutClientId(getTweetsScope) { (statsReceiver, _) => - GetTweetsObserver.observeExchange(statsReceiver) - } - - private[this] val observeGetTweetCountsResultState: Effect[GetTweetCountsObserver.Type] = - withAndWithoutClientId(getTweetCountsScope) { (statsReceiver, _) => - GetTweetCountsObserver.observeExchange(statsReceiver) - } - - private[this] val observeGetDeletedTweetsResultState: Effect[GetDeletedTweetsObserver.Type] = - withAndWithoutClientId(getDeletedTweetsScope) { (statsReceiver, _) => - GetDeletedTweetsObserver.observeExchange(statsReceiver) - } - - private[this] val observeGetStoredTweetsRequest: Effect[GetStoredTweetsRequest] = - GetStoredTweetsObserver.observeRequest(getStoredTweetsScope) - - private[this] val observeGetStoredTweetsResult: Effect[Seq[GetStoredTweetsResult]] = - GetStoredTweetsObserver.observeResult(getStoredTweetsScope) - - private[this] val observeGetStoredTweetsResultState: Effect[GetStoredTweetsObserver.Type] = - GetStoredTweetsObserver.observeExchange(getStoredTweetsScope) - - private[this] val observeGetStoredTweetsByUserRequest: Effect[GetStoredTweetsByUserRequest] = - GetStoredTweetsByUserObserver.observeRequest(getStoredTweetsByUserScope) - - private[this] val observeGetStoredTweetsByUserResult: Effect[GetStoredTweetsByUserResult] = - GetStoredTweetsByUserObserver.observeResult(getStoredTweetsByUserScope) - - private[this] val observeGetStoredTweetsByUserResultState: Effect[ - GetStoredTweetsByUserObserver.Type - ] = - GetStoredTweetsByUserObserver.observeExchange(getStoredTweetsByUserScope) - - override def getTweets(request: GetTweetsRequest): Future[Seq[GetTweetResult]] = { - val actualRequest = - if (request.options.nonEmpty) request - else request.copy(options = Some(defaultGetTweetsRequestOptions)) - observeGetTweetsRequest(actualRequest) - Trace.recordBinary("query_width", request.tweetIds.length) - super - .getTweets(request) - .onSuccess(observeGetTweetResults) - .respond(response => observeGetTweetsResultState((request, response))) - } - - override def getTweetFields(request: GetTweetFieldsRequest): Future[Seq[GetTweetFieldsResult]] = { - observeGetTweetFieldsRequest(request) - Trace.recordBinary("query_width", request.tweetIds.length) - super - .getTweetFields(request) - .onSuccess(observeGetTweetFieldsResults) - .respond(response => observeGetTweetFieldsResultState((request, response))) - } - - override def getTweetCounts(request: GetTweetCountsRequest): Future[Seq[GetTweetCountsResult]] = { - observeGetTweetCountsRequest(request) - Trace.recordBinary("query_width", request.tweetIds.length) - super - .getTweetCounts(request) - .onSuccess(observeTweetCountsResults) - .respond(response => observeGetTweetCountsResultState((request, response))) - } - - override def getDeletedTweets( - request: GetDeletedTweetsRequest - ): Future[Seq[GetDeletedTweetResult]] = { - Trace.recordBinary("query_width", request.tweetIds.length) - super - .getDeletedTweets(request) - .respond(response => observeGetDeletedTweetsResultState((request, response))) - } - - override def postTweet(request: PostTweetRequest): Future[PostTweetResult] = { - observePostTweetRequest(request) - request.additionalFields.foreach(observePostTweetAdditionals) - super - .postTweet(request) - .onSuccess(observePostTweetResponse) - .onSuccess(observeTweetWriteSuccess) - .respond(observeWriteResult) - } - - override def postRetweet(request: RetweetRequest): Future[PostTweetResult] = { - observeRetweetRequest(request) - super - .postRetweet(request) - .onSuccess(observeRetweetResponse) - .onSuccess(observeTweetWriteSuccess) - .respond(observeWriteResult) - } - - override def setAdditionalFields(request: SetAdditionalFieldsRequest): Future[Unit] = { - observeSetFieldsRequest(request) - super - .setAdditionalFields(request) - .respond(observeWriteResult) - } - - override def setRetweetVisibility(request: SetRetweetVisibilityRequest): Future[Unit] = { - observeSetRetweetVisibilityRequest(request) - super - .setRetweetVisibility(request) - .respond(observeWriteResult) - } - - override def deleteAdditionalFields(request: DeleteAdditionalFieldsRequest): Future[Unit] = { - observeDeleteFieldsRequest(request) - super - .deleteAdditionalFields(request) - .respond(observeWriteResult) - } - - override def updatePossiblySensitiveTweet( - request: UpdatePossiblySensitiveTweetRequest - ): Future[Unit] = - super - .updatePossiblySensitiveTweet(request) - .respond(observeWriteResult) - - override def deleteLocationData(request: DeleteLocationDataRequest): Future[Unit] = - super - .deleteLocationData(request) - .respond(observeWriteResult) - - override def scrubGeo(geoScrub: GeoScrub): Future[Unit] = { - observeScrubGeoRequest(geoScrub) - super - .scrubGeo(geoScrub) - .respond(observeWriteResult) - } - - override def scrubGeoUpdateUserTimestamp(request: DeleteLocationData): Future[Unit] = - super.scrubGeoUpdateUserTimestamp(request).respond(observeWriteResult) - - override def takedown(request: TakedownRequest): Future[Unit] = - super - .takedown(request) - .respond(observeWriteResult) - - override def setTweetUserTakedown(request: SetTweetUserTakedownRequest): Future[Unit] = - super - .setTweetUserTakedown(request) - .respond(observeWriteResult) - - override def incrTweetFavCount(request: IncrTweetFavCountRequest): Future[Unit] = - super - .incrTweetFavCount(request) - .respond(observeWriteResult) - - override def incrTweetBookmarkCount(request: IncrTweetBookmarkCountRequest): Future[Unit] = - super - .incrTweetBookmarkCount(request) - .respond(observeWriteResult) - - override def deleteTweets(request: DeleteTweetsRequest): Future[Seq[DeleteTweetResult]] = { - observeDeleteTweetsRequest(request) - super - .deleteTweets(request) - .respond(observeWriteResult) - } - - override def cascadedDeleteTweet(request: CascadedDeleteTweetRequest): Future[Unit] = - super - .cascadedDeleteTweet(request) - .respond(observeWriteResult) - - override def asyncInsert(request: AsyncInsertRequest): Future[Unit] = { - observeAsyncInsertRequest(request) - super - .asyncInsert(request) - .respond(observeWriteResult) - } - - override def asyncSetAdditionalFields(request: AsyncSetAdditionalFieldsRequest): Future[Unit] = { - observeAsyncSetAdditionalFieldsRequest(request) - super - .asyncSetAdditionalFields(request) - .respond(observeWriteResult) - } - - override def asyncSetRetweetVisibility( - request: AsyncSetRetweetVisibilityRequest - ): Future[Unit] = { - observeAsyncSetRetweetVisibilityRequest(request) - super - .asyncSetRetweetVisibility(request) - .respond(observeWriteResult) - } - - override def asyncUndeleteTweet(request: AsyncUndeleteTweetRequest): Future[Unit] = { - observeAsyncUndeleteTweetRequest(request) - super - .asyncUndeleteTweet(request) - .respond(observeWriteResult) - } - - override def asyncDelete(request: AsyncDeleteRequest): Future[Unit] = { - observeAsyncDeleteTweetRequest(request) - super - .asyncDelete(request) - .respond(observeWriteResult) - } - - override def asyncDeleteAdditionalFields( - request: AsyncDeleteAdditionalFieldsRequest - ): Future[Unit] = { - observeAsyncDeleteAdditionalFieldsRequest(request) - super - .asyncDeleteAdditionalFields(request) - .respond(observeWriteResult) - } - - override def asyncTakedown(request: AsyncTakedownRequest): Future[Unit] = { - observeAsyncTakedownRequest(request) - super - .asyncTakedown(request) - .respond(observeWriteResult) - } - - override def asyncUpdatePossiblySensitiveTweet( - request: AsyncUpdatePossiblySensitiveTweetRequest - ): Future[Unit] = { - observeAsyncUpdatePossiblySensitiveTweetRequest(request) - super - .asyncUpdatePossiblySensitiveTweet(request) - .respond(observeWriteResult) - } - - override def replicatedInsertTweet2(request: ReplicatedInsertTweet2Request): Future[Unit] = { - observedReplicatedInsertTweet2Request(request.cachedTweet.tweet) - super.replicatedInsertTweet2(request) - } - - override def getStoredTweets( - request: GetStoredTweetsRequest - ): Future[Seq[GetStoredTweetsResult]] = { - observeGetStoredTweetsRequest(request) - super - .getStoredTweets(request) - .onSuccess(observeGetStoredTweetsResult) - .respond(response => observeGetStoredTweetsResultState((request, response))) - } - - override def getStoredTweetsByUser( - request: GetStoredTweetsByUserRequest - ): Future[GetStoredTweetsByUserResult] = { - observeGetStoredTweetsByUserRequest(request) - super - .getStoredTweetsByUser(request) - .onSuccess(observeGetStoredTweetsByUserResult) - .respond(response => observeGetStoredTweetsByUserResultState((request, response))) - } - - private def withAndWithoutClientId[A]( - stats: StatsReceiver - )( - f: (StatsReceiver, Boolean) => Effect[A] - ) = - f(stats, false).also(withClientId(stats)(f)) - - private def withClientId[A](stats: StatsReceiver)(f: (StatsReceiver, Boolean) => Effect[A]) = { - val map = new SynchronizedHashMap[String, Effect[A]] - - Effect[A] { value => - clientIdHelper.effectiveClientIdRoot.foreach { clientId => - val clientObserver = map.getOrElseUpdate(clientId, f(stats.scope(clientId), true)) - clientObserver(value) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/QuillTweetService.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/QuillTweetService.docx new file mode 100644 index 000000000..21bd2fd9e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/QuillTweetService.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/QuillTweetService.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/QuillTweetService.scala deleted file mode 100644 index 69b9481be..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/QuillTweetService.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.tweetypie -package service - -import com.twitter.quill.capture.QuillCapture -import com.twitter.tweetypie.thriftscala._ -import org.apache.thrift.transport.TMemoryBuffer -import com.twitter.finagle.thrift.Protocols -import com.twitter.quill.capture.Payloads -import com.twitter.tweetypie.service.QuillTweetService.createThriftBinaryRequest -import org.apache.thrift.protocol.TMessage -import org.apache.thrift.protocol.TMessageType -import org.apache.thrift.protocol.TProtocol - -object QuillTweetService { - // Construct the byte stream for a binary thrift request - def createThriftBinaryRequest(method_name: String, write_args: TProtocol => Unit): Array[Byte] = { - val buf = new TMemoryBuffer(512) - val oprot = Protocols.binaryFactory().getProtocol(buf) - - oprot.writeMessageBegin(new TMessage(method_name, TMessageType.CALL, 0)) - write_args(oprot) - oprot.writeMessageEnd() - - // Return bytes - java.util.Arrays.copyOfRange(buf.getArray, 0, buf.length) - } -} - -/** - * Wraps an underlying TweetService, logging some requests. - */ -class QuillTweetService(quillCapture: QuillCapture, protected val underlying: ThriftTweetService) - extends TweetServiceProxy { - - override def postTweet(request: PostTweetRequest): Future[PostTweetResult] = { - val requestBytes = createThriftBinaryRequest( - TweetService.PostTweet.name, - TweetService.PostTweet.Args(request).write) - quillCapture.storeServerRecv(Payloads.fromThriftMessageBytes(requestBytes)) - underlying.postTweet(request) - } - - override def deleteTweets(request: DeleteTweetsRequest): Future[Seq[DeleteTweetResult]] = { - val requestBytes = createThriftBinaryRequest( - TweetService.DeleteTweets.name, - TweetService.DeleteTweets.Args(request).write) - quillCapture.storeServerRecv(Payloads.fromThriftMessageBytes(requestBytes)) - underlying.deleteTweets(request) - } - - override def postRetweet(request: RetweetRequest): Future[PostTweetResult] = { - val requestBytes = createThriftBinaryRequest( - TweetService.PostRetweet.name, - TweetService.PostRetweet.Args(request).write) - quillCapture.storeServerRecv(Payloads.fromThriftMessageBytes(requestBytes)) - underlying.postRetweet(request) - } - - override def unretweet(request: UnretweetRequest): Future[UnretweetResult] = { - val requestBytes = createThriftBinaryRequest( - TweetService.Unretweet.name, - TweetService.Unretweet.Args(request).write) - quillCapture.storeServerRecv(Payloads.fromThriftMessageBytes(requestBytes)) - underlying.unretweet(request) - } - - override def cascadedDeleteTweet(request: CascadedDeleteTweetRequest): Future[Unit] = { - val requestBytes = createThriftBinaryRequest( - TweetServiceInternal.CascadedDeleteTweet.name, - TweetServiceInternal.CascadedDeleteTweet.Args(request).write) - quillCapture.storeServerRecv(Payloads.fromThriftMessageBytes(requestBytes)) - underlying.cascadedDeleteTweet(request) - } - -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ReplicatingTweetService.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ReplicatingTweetService.docx new file mode 100644 index 000000000..97391354a Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ReplicatingTweetService.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ReplicatingTweetService.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ReplicatingTweetService.scala deleted file mode 100644 index d10170232..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/ReplicatingTweetService.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.tweetypie -package service - -import com.twitter.tweetypie.thriftscala._ -import com.twitter.servo.forked.Forked -import com.twitter.tweetypie.service.ReplicatingTweetService.GatedReplicationClient - -/** - * Wraps an underlying ThriftTweetService, transforming external requests to replicated requests. - */ -object ReplicatingTweetService { - // Can be used to associate replication client with a gate that determines - // if a replication request should be performed. - case class GatedReplicationClient(client: ThriftTweetService, gate: Gate[Unit]) { - def execute(executor: Forked.Executor, action: ThriftTweetService => Unit): Unit = { - if (gate()) executor { () => action(client) } - } - } -} - -class ReplicatingTweetService( - protected val underlying: ThriftTweetService, - replicationTargets: Seq[GatedReplicationClient], - executor: Forked.Executor, -) extends TweetServiceProxy { - private[this] def replicateRead(action: ThriftTweetService => Unit): Unit = - replicationTargets.foreach(_.execute(executor, action)) - - override def getTweetCounts(request: GetTweetCountsRequest): Future[Seq[GetTweetCountsResult]] = { - replicateRead(_.replicatedGetTweetCounts(request)) - underlying.getTweetCounts(request) - } - - override def getTweetFields(request: GetTweetFieldsRequest): Future[Seq[GetTweetFieldsResult]] = { - if (!request.options.doNotCache) { - replicateRead(_.replicatedGetTweetFields(request)) - } - underlying.getTweetFields(request) - } - - override def getTweets(request: GetTweetsRequest): Future[Seq[GetTweetResult]] = { - if (!request.options.exists(_.doNotCache)) { - replicateRead(_.replicatedGetTweets(request)) - } - underlying.getTweets(request) - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/RescueExceptions.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/RescueExceptions.docx new file mode 100644 index 000000000..ae050663e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/RescueExceptions.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/RescueExceptions.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/RescueExceptions.scala deleted file mode 100644 index 9ae769f2b..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/RescueExceptions.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.tweetypie -package service - -import com.twitter.finagle.IndividualRequestTimeoutException -import com.twitter.servo.exception.thriftscala._ -import com.twitter.tweetypie.core.OverCapacity -import com.twitter.tweetypie.core.RateLimited -import com.twitter.tweetypie.core.TweetHydrationError -import com.twitter.tweetypie.core.UpstreamFailure -import com.twitter.tweetypie.thriftscala._ -import com.twitter.util.TimeoutException - -object RescueExceptions { - private val log = Logger("com.twitter.tweetypie.service.TweetService") - - /** - * rescue to servo exceptions - */ - def rescueToServoFailure( - name: String, - clientId: String - ): PartialFunction[Throwable, Future[Nothing]] = { - translateToServoFailure(formatError(name, clientId, _)).andThen(Future.exception) - } - - private def translateToServoFailure( - toMsg: String => String - ): PartialFunction[Throwable, Throwable] = { - case e: AccessDenied if suspendedOrDeactivated(e) => - e.copy(message = toMsg(e.message)) - case e: ClientError => - e.copy(message = toMsg(e.message)) - case e: UnauthorizedException => - ClientError(ClientErrorCause.Unauthorized, toMsg(e.msg)) - case e: AccessDenied => - ClientError(ClientErrorCause.Unauthorized, toMsg(e.message)) - case e: RateLimited => - ClientError(ClientErrorCause.RateLimited, toMsg(e.message)) - case e: ServerError => - e.copy(message = toMsg(e.message)) - case e: TimeoutException => - ServerError(ServerErrorCause.RequestTimeout, toMsg(e.toString)) - case e: IndividualRequestTimeoutException => - ServerError(ServerErrorCause.RequestTimeout, toMsg(e.toString)) - case e: UpstreamFailure => - ServerError(ServerErrorCause.DependencyError, toMsg(e.toString)) - case e: OverCapacity => - ServerError(ServerErrorCause.ServiceUnavailable, toMsg(e.message)) - case e: TweetHydrationError => - ServerError(ServerErrorCause.DependencyError, toMsg(e.toString)) - case e => - log.warn("caught unexpected exception", e) - ServerError(ServerErrorCause.InternalServerError, toMsg(e.toString)) - } - - private def suspendedOrDeactivated(e: AccessDenied): Boolean = - e.errorCause.exists { c => - c == AccessDeniedCause.UserDeactivated || c == AccessDeniedCause.UserSuspended - } - - private def formatError(name: String, clientId: String, msg: String): String = - s"($clientId, $name) $msg" -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceProxy.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceProxy.docx new file mode 100644 index 000000000..f1080515c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceProxy.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceProxy.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceProxy.scala deleted file mode 100644 index a167ecb43..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceProxy.scala +++ /dev/null @@ -1,146 +0,0 @@ -/** Copyright 2012 Twitter, Inc. */ -package com.twitter.tweetypie -package service - -import com.twitter.finagle.thrift.ClientId -import com.twitter.tweetypie.thriftscala.{TweetServiceProxy => BaseTweetServiceProxy, _} - -/** - * A trait for TweetService implementations that wrap an underlying TweetService and need to modify - * only some of the methods. - * - * This proxy is the same as [[com.twitter.tweetypie.thriftscala.TweetServiceProxy]], except it also - * extends [[com.twitter.tweetypie.thriftscala.TweetServiceInternal]] which gives us access to all - * of the async* methods. - */ -trait TweetServiceProxy extends BaseTweetServiceProxy with ThriftTweetService { - protected override def underlying: ThriftTweetService - - override def replicatedGetTweetCounts(request: GetTweetCountsRequest): Future[Unit] = - wrap(underlying.replicatedGetTweetCounts(request)) - - override def replicatedGetTweetFields(request: GetTweetFieldsRequest): Future[Unit] = - wrap(underlying.replicatedGetTweetFields(request)) - - override def replicatedGetTweets(request: GetTweetsRequest): Future[Unit] = - wrap(underlying.replicatedGetTweets(request)) - - override def asyncSetAdditionalFields(request: AsyncSetAdditionalFieldsRequest): Future[Unit] = - wrap(underlying.asyncSetAdditionalFields(request)) - - override def asyncDeleteAdditionalFields( - request: AsyncDeleteAdditionalFieldsRequest - ): Future[Unit] = - wrap(underlying.asyncDeleteAdditionalFields(request)) - - override def cascadedDeleteTweet(request: CascadedDeleteTweetRequest): Future[Unit] = - wrap(underlying.cascadedDeleteTweet(request)) - - override def asyncInsert(request: AsyncInsertRequest): Future[Unit] = - wrap(underlying.asyncInsert(request)) - - override def replicatedUpdatePossiblySensitiveTweet(tweet: Tweet): Future[Unit] = - wrap(underlying.replicatedUpdatePossiblySensitiveTweet(tweet)) - - override def asyncUpdatePossiblySensitiveTweet( - request: AsyncUpdatePossiblySensitiveTweetRequest - ): Future[Unit] = - wrap(underlying.asyncUpdatePossiblySensitiveTweet(request)) - - override def asyncUndeleteTweet(request: AsyncUndeleteTweetRequest): Future[Unit] = - wrap(underlying.asyncUndeleteTweet(request)) - - override def eraseUserTweets(request: EraseUserTweetsRequest): Future[Unit] = - wrap(underlying.eraseUserTweets(request)) - - override def asyncEraseUserTweets(request: AsyncEraseUserTweetsRequest): Future[Unit] = - wrap(underlying.asyncEraseUserTweets(request)) - - override def asyncDelete(request: AsyncDeleteRequest): Future[Unit] = - wrap(underlying.asyncDelete(request)) - - override def asyncIncrFavCount(request: AsyncIncrFavCountRequest): Future[Unit] = - wrap(underlying.asyncIncrFavCount(request)) - - override def asyncIncrBookmarkCount(request: AsyncIncrBookmarkCountRequest): Future[Unit] = - wrap(underlying.asyncIncrBookmarkCount(request)) - - override def scrubGeoUpdateUserTimestamp(request: DeleteLocationData): Future[Unit] = - wrap(underlying.scrubGeoUpdateUserTimestamp(request)) - - override def asyncSetRetweetVisibility(request: AsyncSetRetweetVisibilityRequest): Future[Unit] = - wrap(underlying.asyncSetRetweetVisibility(request)) - - override def setRetweetVisibility(request: SetRetweetVisibilityRequest): Future[Unit] = - wrap(underlying.setRetweetVisibility(request)) - - override def asyncTakedown(request: AsyncTakedownRequest): Future[Unit] = - wrap(underlying.asyncTakedown(request)) - - override def setTweetUserTakedown(request: SetTweetUserTakedownRequest): Future[Unit] = - wrap(underlying.setTweetUserTakedown(request)) - - override def replicatedUndeleteTweet2(request: ReplicatedUndeleteTweet2Request): Future[Unit] = - wrap(underlying.replicatedUndeleteTweet2(request)) - - override def replicatedInsertTweet2(request: ReplicatedInsertTweet2Request): Future[Unit] = - wrap(underlying.replicatedInsertTweet2(request)) - - override def replicatedDeleteTweet2(request: ReplicatedDeleteTweet2Request): Future[Unit] = - wrap(underlying.replicatedDeleteTweet2(request)) - - override def replicatedIncrFavCount(tweetId: TweetId, delta: Int): Future[Unit] = - wrap(underlying.replicatedIncrFavCount(tweetId, delta)) - - override def replicatedIncrBookmarkCount(tweetId: TweetId, delta: Int): Future[Unit] = - wrap(underlying.replicatedIncrBookmarkCount(tweetId, delta)) - - override def replicatedSetRetweetVisibility( - request: ReplicatedSetRetweetVisibilityRequest - ): Future[Unit] = - wrap(underlying.replicatedSetRetweetVisibility(request)) - - override def replicatedScrubGeo(tweetIds: Seq[TweetId]): Future[Unit] = - wrap(underlying.replicatedScrubGeo(tweetIds)) - - override def replicatedSetAdditionalFields(request: SetAdditionalFieldsRequest): Future[Unit] = - wrap(underlying.replicatedSetAdditionalFields(request)) - - override def replicatedDeleteAdditionalFields( - request: ReplicatedDeleteAdditionalFieldsRequest - ): Future[Unit] = - wrap(underlying.replicatedDeleteAdditionalFields(request)) - - override def replicatedTakedown(tweet: Tweet): Future[Unit] = - wrap(underlying.replicatedTakedown(tweet)) - - override def quotedTweetDelete(request: QuotedTweetDeleteRequest): Future[Unit] = - wrap(underlying.quotedTweetDelete(request)) - - override def quotedTweetTakedown(request: QuotedTweetTakedownRequest): Future[Unit] = - wrap(underlying.quotedTweetTakedown(request)) - - override def getStoredTweets( - request: GetStoredTweetsRequest - ): Future[Seq[GetStoredTweetsResult]] = - wrap(underlying.getStoredTweets(request)) - - override def getStoredTweetsByUser( - request: GetStoredTweetsByUserRequest - ): Future[GetStoredTweetsByUserResult] = - wrap(underlying.getStoredTweetsByUser(request)) -} - -/** - * A TweetServiceProxy with a mutable underlying field. - */ -class MutableTweetServiceProxy(var underlying: ThriftTweetService) extends TweetServiceProxy - -/** - * A TweetServiceProxy that sets the ClientId context before executing the method. - */ -class ClientIdSettingTweetServiceProxy(clientId: ClientId, val underlying: ThriftTweetService) - extends TweetServiceProxy { - override def wrap[A](f: => Future[A]): Future[A] = - clientId.asCurrent(f) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceWarmer.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceWarmer.docx new file mode 100644 index 000000000..0e7b23b85 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceWarmer.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceWarmer.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceWarmer.scala deleted file mode 100644 index 79e97519c..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/TweetServiceWarmer.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.tweetypie -package service - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.thrift.ClientId -import com.twitter.tweetypie.thriftscala._ -import com.twitter.util.Await -import scala.util.control.NonFatal - -/** - * Settings for the artificial tweet fetching requests that are sent to warmup the - * server before authentic requests are processed. - */ -case class WarmupQueriesSettings( - realTweetRequestCycles: Int = 100, - requestTimeout: Duration = 3.seconds, - clientId: ClientId = ClientId("tweetypie.warmup"), - requestTimeRange: Duration = 10.minutes, - maxConcurrency: Int = 20) - -object TweetServiceWarmer { - - /** - * Load info from perspective of TLS test account with short favorites timeline. - */ - val ForUserId = 3511687034L // @mikestltestact1 -} - -/** - * Generates requests to getTweets for the purpose of warming up the code paths used - * in fetching tweets. - */ -class TweetServiceWarmer( - warmupSettings: WarmupQueriesSettings, - requestOptions: GetTweetOptions = GetTweetOptions(includePlaces = true, - includeRetweetCount = true, includeReplyCount = true, includeFavoriteCount = true, - includeCards = true, cardsPlatformKey = Some("iPhone-13"), includePerspectivals = true, - includeQuotedTweet = true, forUserId = Some(TweetServiceWarmer.ForUserId))) - extends (ThriftTweetService => Unit) { - import warmupSettings._ - - private val realTweetIds = - Seq( - 20L, // just setting up my twttr - 456190426412617728L, // protected user tweet - 455477977715707904L, // suspended user tweet - 440322224407314432L, // ellen oscar selfie - 372173241290612736L, // gaga mentions 1d - 456965485179838464L, // media tagged tweet - 525421442918121473L, // tweet with card - 527214829807759360L, // tweet with annotation - 472788687571677184L // tweet with quote tweet - ) - - private val log = Logger(getClass) - - /** - * Executes the warmup queries, waiting for them to complete or until - * the warmupTimeout occurs. - */ - def apply(service: ThriftTweetService): Unit = { - val warmupStart = Time.now - log.info("warming up...") - warmup(service) - val warmupDuration = Time.now.since(warmupStart) - log.info("warmup took " + warmupDuration) - } - - /** - * Executes the warmup queries, returning when all responses have completed or timed-out. - */ - private[this] def warmup(service: ThriftTweetService): Unit = - clientId.asCurrent { - val request = GetTweetsRequest(realTweetIds, options = Some(requestOptions)) - val requests = Seq.fill(realTweetRequestCycles)(request) - val requestGroups = requests.grouped(maxConcurrency) - - for (requests <- requestGroups) { - val responses = requests.map(service.getTweets(_)) - try { - Await.ready(Future.join(responses), requestTimeout) - } catch { - // Await.ready throws exceptions on timeouts and - // interruptions. This prevents those exceptions from - // bubbling up. - case NonFatal(_) => - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/BUILD b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/BUILD deleted file mode 100644 index 45c15cb77..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-core/src/main", - "tweetypie/servo/util/src/main/scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/thrift/com/twitter/escherbird:media-annotation-structs-scala", - "src/thrift/com/twitter/servo:servo-exception-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:service-scala", - "tweetypie/common/src/thrift/com/twitter/tweetypie:stored-tweet-info-scala", - "tweetypie/server/src/main/scala/com/twitter/tweetypie", - "tweetypie/server/src/main/scala/com/twitter/tweetypie/media", - "tweetypie/server/src/main/thrift:compiled-scala", - "tweetypie/common/src/scala/com/twitter/tweetypie/additionalfields", - "tweetypie/common/src/scala/com/twitter/tweetypie/tweettext", - "tweetypie/common/src/scala/com/twitter/tweetypie/util", - ], -) diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/BUILD.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/BUILD.docx new file mode 100644 index 000000000..dc3b062ea Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/BUILD.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetDeletedTweetsObserver.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetDeletedTweetsObserver.docx new file mode 100644 index 000000000..01fe6c34c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetDeletedTweetsObserver.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetDeletedTweetsObserver.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetDeletedTweetsObserver.scala deleted file mode 100644 index 1e86348b8..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetDeletedTweetsObserver.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.tweetypie.thriftscala.GetDeletedTweetResult -import com.twitter.tweetypie.thriftscala.GetDeletedTweetsRequest - -private[service] object GetDeletedTweetsObserver { - type Type = ObserveExchange[GetDeletedTweetsRequest, Seq[GetDeletedTweetResult]] - - def observeExchange(stats: StatsReceiver): Effect[Type] = { - val resultStateStats = ResultStateStats(stats) - - Effect { - case (request, response) => - response match { - case Return(_) | Throw(ClientError(_)) => - resultStateStats.success(request.tweetIds.size) - case Throw(_) => - resultStateStats.failed(request.tweetIds.size) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsByUserObserver.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsByUserObserver.docx new file mode 100644 index 000000000..8f195b35e Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsByUserObserver.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsByUserObserver.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsByUserObserver.scala deleted file mode 100644 index 5c16c68b2..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsByUserObserver.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.tweetypie.thriftscala.GetStoredTweetsByUserRequest -import com.twitter.tweetypie.thriftscala.GetStoredTweetsByUserResult - -private[service] object GetStoredTweetsByUserObserver extends StoredTweetsObserver { - - type Type = ObserveExchange[GetStoredTweetsByUserRequest, GetStoredTweetsByUserResult] - val firstTweetTimestamp: Long = 1142974200L - - def observeRequest(stats: StatsReceiver): Effect[GetStoredTweetsByUserRequest] = { - val optionsScope = stats.scope("options") - val bypassVisibilityFilteringCounter = optionsScope.counter("bypass_visibility_filtering") - val forUserIdCounter = optionsScope.counter("set_for_user_id") - val timeRangeStat = optionsScope.stat("time_range_seconds") - val cursorCounter = optionsScope.counter("cursor") - val startFromOldestCounter = optionsScope.counter("start_from_oldest") - val additionalFieldsScope = optionsScope.scope("additional_fields") - - Effect { request => - if (request.options.isDefined) { - val options = request.options.get - - if (options.bypassVisibilityFiltering) bypassVisibilityFilteringCounter.incr() - if (options.setForUserId) forUserIdCounter.incr() - if (options.cursor.isDefined) { - cursorCounter.incr() - } else { - // We only add a time range stat once, when there's no cursor in the request (i.e. this - // isn't a repeat request for a subsequent batch of results) - val startTimeSeconds: Long = - options.startTimeMsec.map(_ / 1000).getOrElse(firstTweetTimestamp) - val endTimeSeconds: Long = options.endTimeMsec.map(_ / 1000).getOrElse(Time.now.inSeconds) - timeRangeStat.add(endTimeSeconds - startTimeSeconds) - - // We use the startFromOldest parameter when the cursor isn't defined - if (options.startFromOldest) startFromOldestCounter.incr() - } - options.additionalFieldIds.foreach { id => - additionalFieldsScope.counter(id.toString).incr() - } - } - } - } - - def observeResult(stats: StatsReceiver): Effect[GetStoredTweetsByUserResult] = { - val resultScope = stats.scope("result") - - Effect { result => - observeStoredTweets(result.storedTweets, resultScope) - } - } - - def observeExchange(stats: StatsReceiver): Effect[Type] = { - val resultStateStats = ResultStateStats(stats) - - Effect { - case (request, response) => - response match { - case Return(_) => resultStateStats.success() - case Throw(_) => resultStateStats.failed() - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsObserver.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsObserver.docx new file mode 100644 index 000000000..18c3743d9 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsObserver.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsObserver.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsObserver.scala deleted file mode 100644 index f6021d06c..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetStoredTweetsObserver.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.tweetypie.thriftscala.GetStoredTweetsRequest -import com.twitter.tweetypie.thriftscala.GetStoredTweetsResult - -private[service] object GetStoredTweetsObserver extends StoredTweetsObserver { - type Type = ObserveExchange[GetStoredTweetsRequest, Seq[GetStoredTweetsResult]] - - def observeRequest(stats: StatsReceiver): Effect[GetStoredTweetsRequest] = { - val requestSizeStat = stats.stat("request_size") - - val optionsScope = stats.scope("options") - val bypassVisibilityFilteringCounter = optionsScope.counter("bypass_visibility_filtering") - val forUserIdCounter = optionsScope.counter("for_user_id") - val additionalFieldsScope = optionsScope.scope("additional_fields") - - Effect { request => - requestSizeStat.add(request.tweetIds.size) - - if (request.options.isDefined) { - val options = request.options.get - if (options.bypassVisibilityFiltering) bypassVisibilityFilteringCounter.incr() - if (options.forUserId.isDefined) forUserIdCounter.incr() - options.additionalFieldIds.foreach { id => - additionalFieldsScope.counter(id.toString).incr() - } - } - } - } - - def observeResult(stats: StatsReceiver): Effect[Seq[GetStoredTweetsResult]] = { - val resultScope = stats.scope("result") - - Effect { result => - observeStoredTweets(result.map(_.storedTweet), resultScope) - } - } - - def observeExchange(stats: StatsReceiver): Effect[Type] = { - val resultStateStats = ResultStateStats(stats) - - Effect { - case (request, response) => - response match { - case Return(_) => resultStateStats.success(request.tweetIds.size) - case Throw(_) => resultStateStats.failed(request.tweetIds.size) - } - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetCountsObserver.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetCountsObserver.docx new file mode 100644 index 000000000..5c9d3f0e9 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetCountsObserver.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetCountsObserver.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetCountsObserver.scala deleted file mode 100644 index c97fdc2e7..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetCountsObserver.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.tweetypie.thriftscala.GetTweetCountsRequest -import com.twitter.tweetypie.thriftscala.GetTweetCountsResult - -private[service] object GetTweetCountsObserver { - type Type = ObserveExchange[GetTweetCountsRequest, Seq[GetTweetCountsResult]] - - def observeExchange(stats: StatsReceiver): Effect[Type] = { - val resultStateStats = ResultStateStats(stats) - - Effect { - case (request, response) => - response match { - case Return(_) | Throw(ClientError(_)) => - resultStateStats.success(request.tweetIds.size) - case Throw(_) => - resultStateStats.failed(request.tweetIds.size) - } - } - } - - def observeResults(stats: StatsReceiver): Effect[Seq[GetTweetCountsResult]] = { - val retweetCounter = stats.counter("retweets") - val replyCounter = stats.counter("replies") - val favoriteCounter = stats.counter("favorites") - - Effect { counts => - counts.foreach { c => - if (c.retweetCount.isDefined) retweetCounter.incr() - if (c.replyCount.isDefined) replyCounter.incr() - if (c.favoriteCount.isDefined) favoriteCounter.incr() - } - } - } - - def observeRequest(stats: StatsReceiver): Effect[GetTweetCountsRequest] = { - val requestSizesStat = stats.stat("request_size") - val optionsScope = stats.scope("options") - val includeRetweetCounter = optionsScope.counter("retweet_counts") - val includeReplyCounter = optionsScope.counter("reply_counts") - val includeFavoriteCounter = optionsScope.counter("favorite_counts") - val tweetAgeStat = stats.stat("tweet_age_seconds") - - Effect { request => - val size = request.tweetIds.size - requestSizesStat.add(size) - - // Measure Tweet.get_tweet_counts tweet age of requested Tweets. - // Tweet counts are stored in cache, falling back to TFlock on cache misses. - // Track client TweetId age to understand how that affects clients response latencies. - for { - id <- request.tweetIds - timestamp <- SnowflakeId.timeFromIdOpt(id) - age = Time.now.since(timestamp) - } tweetAgeStat.add(age.inSeconds) - - if (request.includeRetweetCount) includeRetweetCounter.incr(size) - if (request.includeReplyCount) includeReplyCounter.incr(size) - if (request.includeFavoriteCount) includeFavoriteCounter.incr(size) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetFieldsObserver.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetFieldsObserver.docx new file mode 100644 index 000000000..06ed8871c Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetFieldsObserver.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetFieldsObserver.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetFieldsObserver.scala deleted file mode 100644 index af6666b03..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetFieldsObserver.scala +++ /dev/null @@ -1,160 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.tweetypie.thriftscala._ - -private[service] object GetTweetFieldsObserver { - type Type = ObserveExchange[GetTweetFieldsRequest, Seq[GetTweetFieldsResult]] - - def observeExchange(statsReceiver: StatsReceiver): Effect[Type] = { - val resultStateStats = ResultStateStats(statsReceiver) - - val stats = statsReceiver.scope("results") - val tweetResultFailed = stats.counter("tweet_result_failed") - val quoteResultFailed = stats.counter("quote_result_failed") - val overCapacity = stats.counter("over_capacity") - - def observeFailedResult(r: GetTweetFieldsResult): Unit = { - r.tweetResult match { - case TweetFieldsResultState.Failed(failed) => - tweetResultFailed.incr() - - if (failed.overCapacity) overCapacity.incr() - case _ => - } - - if (r.quotedTweetResult.exists(_.isInstanceOf[TweetFieldsResultState.Failed])) - quoteResultFailed.incr() - } - - Effect { - case (request, response) => - response match { - case Return(xs) => - xs foreach { - case x if isFailedResult(x) => - observeFailedResult(x) - resultStateStats.failed() - case _ => - resultStateStats.success() - } - case Throw(ClientError(_)) => - resultStateStats.success(request.tweetIds.size) - case Throw(_) => - resultStateStats.failed(request.tweetIds.size) - } - } - } - - def observeRequest(stats: StatsReceiver, byClient: Boolean): Effect[GetTweetFieldsRequest] = { - val requestSizeStat = stats.stat("request_size") - val optionsScope = stats.scope("options") - val tweetFieldsScope = optionsScope.scope("tweet_field") - val countsFieldsScope = optionsScope.scope("counts_field") - val mediaFieldsScope = optionsScope.scope("media_field") - val includeRetweetedTweetCounter = optionsScope.counter("include_retweeted_tweet") - val includeQuotedTweetCounter = optionsScope.counter("include_quoted_tweet") - val forUserIdCounter = optionsScope.counter("for_user_id") - val cardsPlatformKeyCounter = optionsScope.counter("cards_platform_key") - val cardsPlatformKeyScope = optionsScope.scope("cards_platform_key") - val extensionsArgsCounter = optionsScope.counter("extensions_args") - val doNotCacheCounter = optionsScope.counter("do_not_cache") - val simpleQuotedTweetCounter = optionsScope.counter("simple_quoted_tweet") - val visibilityPolicyScope = optionsScope.scope("visibility_policy") - val userVisibleCounter = visibilityPolicyScope.counter("user_visible") - val noFilteringCounter = visibilityPolicyScope.counter("no_filtering") - val noSafetyLevelCounter = optionsScope.counter("no_safety_level") - val safetyLevelCounter = optionsScope.counter("safety_level") - val safetyLevelScope = optionsScope.scope("safety_level") - - Effect { - case GetTweetFieldsRequest(tweetIds, options) => - requestSizeStat.add(tweetIds.size) - options.tweetIncludes.foreach { - case TweetInclude.TweetFieldId(id) => tweetFieldsScope.counter(id.toString).incr() - case TweetInclude.CountsFieldId(id) => countsFieldsScope.counter(id.toString).incr() - case TweetInclude.MediaEntityFieldId(id) => mediaFieldsScope.counter(id.toString).incr() - case _ => - } - if (options.includeRetweetedTweet) includeRetweetedTweetCounter.incr() - if (options.includeQuotedTweet) includeQuotedTweetCounter.incr() - if (options.forUserId.nonEmpty) forUserIdCounter.incr() - if (options.cardsPlatformKey.nonEmpty) cardsPlatformKeyCounter.incr() - if (!byClient) { - options.cardsPlatformKey.foreach { cardsPlatformKey => - cardsPlatformKeyScope.counter(cardsPlatformKey).incr() - } - } - if (options.extensionsArgs.nonEmpty) extensionsArgsCounter.incr() - if (options.safetyLevel.nonEmpty) { - safetyLevelCounter.incr() - } else { - noSafetyLevelCounter.incr() - } - options.visibilityPolicy match { - case TweetVisibilityPolicy.UserVisible => userVisibleCounter.incr() - case TweetVisibilityPolicy.NoFiltering => noFilteringCounter.incr() - case _ => - } - options.safetyLevel.foreach { level => safetyLevelScope.counter(level.toString).incr() } - if (options.doNotCache) doNotCacheCounter.incr() - if (options.simpleQuotedTweet) simpleQuotedTweetCounter.incr() - } - } - - def observeResults(stats: StatsReceiver): Effect[Seq[GetTweetFieldsResult]] = { - val resultsCounter = stats.counter("results") - val resultsScope = stats.scope("results") - val observeState = GetTweetFieldsObserver.observeResultState(resultsScope) - - Effect { results => - resultsCounter.incr(results.size) - results.foreach { r => - observeState(r.tweetResult) - r.quotedTweetResult.foreach { qtResult => - resultsCounter.incr() - observeState(qtResult) - } - } - } - } - - /** - * Given a GetTweetFieldsResult result, do we observe the result as a failure or not. - */ - private def isFailedResult(result: GetTweetFieldsResult): Boolean = { - result.tweetResult.isInstanceOf[TweetFieldsResultState.Failed] || - result.quotedTweetResult.exists(_.isInstanceOf[TweetFieldsResultState.Failed]) - } - - private def observeResultState(stats: StatsReceiver): Effect[TweetFieldsResultState] = { - val foundCounter = stats.counter("found") - val notFoundCounter = stats.counter("not_found") - val failedCounter = stats.counter("failed") - val filteredCounter = stats.counter("filtered") - val filteredReasonScope = stats.scope("filtered_reason") - val otherCounter = stats.counter("other") - val observeTweet = Observer - .countTweetAttributes(stats.scope("found"), byClient = false) - - Effect { - case TweetFieldsResultState.Found(found) => - foundCounter.incr() - observeTweet(found.tweet) - found.retweetedTweet.foreach(observeTweet) - - case TweetFieldsResultState.NotFound(_) => notFoundCounter.incr() - case TweetFieldsResultState.Failed(_) => failedCounter.incr() - case TweetFieldsResultState.Filtered(f) => - filteredCounter.incr() - // Since reasons have parameters, eg. AuthorBlockViewer(true) and we don't - // need the "(true)" part, we do .getClass.getSimpleName to get rid of that - filteredReasonScope.counter(f.reason.getClass.getSimpleName).incr() - - case _ => otherCounter.incr() - } - } - -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetsObserver.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetsObserver.docx new file mode 100644 index 000000000..c00766f30 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetsObserver.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetsObserver.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetsObserver.scala deleted file mode 100644 index 77f1829a5..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/GetTweetsObserver.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.servo.exception.thriftscala.ClientError -import com.twitter.tweetypie.thriftscala.GetTweetOptions -import com.twitter.tweetypie.thriftscala.GetTweetResult -import com.twitter.tweetypie.thriftscala.GetTweetsRequest - -private[service] object GetTweetsObserver { - type Type = ObserveExchange[GetTweetsRequest, Seq[GetTweetResult]] - - def observeExchange(stats: StatsReceiver): Effect[Type] = { - val resultStateStats = ResultStateStats(stats) - - Effect { - case (request, response) => - response match { - case Return(xs) => - xs.foreach { - case result if Observer.successStatusStates(result.tweetState) => - resultStateStats.success() - case _ => - resultStateStats.failed() - } - case Throw(ClientError(_)) => - resultStateStats.success(request.tweetIds.size) - case Throw(_) => - resultStateStats.failed(request.tweetIds.size) - } - } - } - - def observeResults(stats: StatsReceiver, byClient: Boolean): Effect[Seq[GetTweetResult]] = - countStates(stats).also(countTweetReadAttributes(stats, byClient)) - - def observeRequest(stats: StatsReceiver, byClient: Boolean): Effect[GetTweetsRequest] = { - val requestSizeStat = stats.stat("request_size") - val optionsScope = stats.scope("options") - val languageScope = optionsScope.scope("language") - val includeSourceTweetCounter = optionsScope.counter("source_tweet") - val includeQuotedTweetCounter = optionsScope.counter("quoted_tweet") - val includePerspectiveCounter = optionsScope.counter("perspective") - val includeConversationMutedCounter = optionsScope.counter("conversation_muted") - val includePlacesCounter = optionsScope.counter("places") - val includeCardsCounter = optionsScope.counter("cards") - val includeRetweetCountsCounter = optionsScope.counter("retweet_counts") - val includeReplyCountsCounter = optionsScope.counter("reply_counts") - val includeFavoriteCountsCounter = optionsScope.counter("favorite_counts") - val includeQuoteCountsCounter = optionsScope.counter("quote_counts") - val bypassVisibilityFilteringCounter = optionsScope.counter("bypass_visibility_filtering") - val excludeReportedCounter = optionsScope.counter("exclude_reported") - val cardsPlatformKeyScope = optionsScope.scope("cards_platform_key") - val extensionsArgsCounter = optionsScope.counter("extensions_args") - val doNotCacheCounter = optionsScope.counter("do_not_cache") - val additionalFieldsScope = optionsScope.scope("additional_fields") - val safetyLevelScope = optionsScope.scope("safety_level") - val includeProfileGeoEnrichment = optionsScope.counter("profile_geo_enrichment") - val includeMediaAdditionalMetadata = optionsScope.counter("media_additional_metadata") - val simpleQuotedTweet = optionsScope.counter("simple_quoted_tweet") - val forUserIdCounter = optionsScope.counter("for_user_id") - - def includesPerspectivals(options: GetTweetOptions) = - options.includePerspectivals && options.forUserId.nonEmpty - - Effect { - case GetTweetsRequest(tweetIds, _, Some(options), _) => - requestSizeStat.add(tweetIds.size) - if (!byClient) languageScope.counter(options.languageTag).incr() - if (options.includeSourceTweet) includeSourceTweetCounter.incr() - if (options.includeQuotedTweet) includeQuotedTweetCounter.incr() - if (includesPerspectivals(options)) includePerspectiveCounter.incr() - if (options.includeConversationMuted) includeConversationMutedCounter.incr() - if (options.includePlaces) includePlacesCounter.incr() - if (options.includeCards) includeCardsCounter.incr() - if (options.includeRetweetCount) includeRetweetCountsCounter.incr() - if (options.includeReplyCount) includeReplyCountsCounter.incr() - if (options.includeFavoriteCount) includeFavoriteCountsCounter.incr() - if (options.includeQuoteCount) includeQuoteCountsCounter.incr() - if (options.bypassVisibilityFiltering) bypassVisibilityFilteringCounter.incr() - if (options.excludeReported) excludeReportedCounter.incr() - if (options.extensionsArgs.nonEmpty) extensionsArgsCounter.incr() - if (options.doNotCache) doNotCacheCounter.incr() - if (options.includeProfileGeoEnrichment) includeProfileGeoEnrichment.incr() - if (options.includeMediaAdditionalMetadata) includeMediaAdditionalMetadata.incr() - if (options.simpleQuotedTweet) simpleQuotedTweet.incr() - if (options.forUserId.nonEmpty) forUserIdCounter.incr() - if (!byClient) { - options.cardsPlatformKey.foreach { cardsPlatformKey => - cardsPlatformKeyScope.counter(cardsPlatformKey).incr() - } - } - options.additionalFieldIds.foreach { id => - additionalFieldsScope.counter(id.toString).incr() - } - options.safetyLevel.foreach { level => safetyLevelScope.counter(level.toString).incr() } - } - } - - /** - * We count the number of times each tweet state is returned as a - * general measure of the health of TweetyPie. partial and not_found - * tweet states should be close to zero. - */ - private def countStates(stats: StatsReceiver): Effect[Seq[GetTweetResult]] = { - val state = Observer.observeStatusStates(stats) - Effect { results => results.foreach { tweetResult => state(tweetResult.tweetState) } } - } - - private def countTweetReadAttributes( - stats: StatsReceiver, - byClient: Boolean - ): Effect[Seq[GetTweetResult]] = { - val tweetObserver = Observer.countTweetAttributes(stats, byClient) - Effect { results => - results.foreach { tweetResult => tweetResult.tweet.foreach(tweetObserver) } - } - } - -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/Observer.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/Observer.docx new file mode 100644 index 000000000..6698d9092 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/Observer.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/Observer.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/Observer.scala deleted file mode 100644 index c5a9782cb..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/Observer.scala +++ /dev/null @@ -1,365 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.tweetypie.additionalfields.AdditionalFields -import com.twitter.tweetypie.media.MediaKeyClassifier -import com.twitter.tweetypie.thriftscala._ -import com.twitter.tweetypie.tweettext.TweetText.codePointLength -import com.twitter.conversions.DurationOps._ - -/** - * Observer can be used for storing - * - one-off handler specific metrics with minor logic - * - reusable Tweetypie service metrics for multiple handlers - */ -private[service] object Observer { - - val successStatusStates: Set[StatusState] = Set( - StatusState.Found, - StatusState.NotFound, - StatusState.DeactivatedUser, - StatusState.SuspendedUser, - StatusState.ProtectedUser, - StatusState.ReportedTweet, - StatusState.UnsupportedClient, - StatusState.Drop, - StatusState.Suppress, - StatusState.Deleted, - StatusState.BounceDeleted - ) - - def observeStatusStates(statsReceiver: StatsReceiver): Effect[StatusState] = { - val stats = statsReceiver.scope("status_state") - val total = statsReceiver.counter("status_results") - - val foundCounter = stats.counter("found") - val notFoundCounter = stats.counter("not_found") - val partialCounter = stats.counter("partial") - val timedOutCounter = stats.counter("timed_out") - val failedCounter = stats.counter("failed") - val deactivatedCounter = stats.counter("deactivated") - val suspendedCounter = stats.counter("suspended") - val protectedCounter = stats.counter("protected") - val reportedCounter = stats.counter("reported") - val overCapacityCounter = stats.counter("over_capacity") - val unsupportedClientCounter = stats.counter("unsupported_client") - val dropCounter = stats.counter("drop") - val suppressCounter = stats.counter("suppress") - val deletedCounter = stats.counter("deleted") - val bounceDeletedCounter = stats.counter("bounce_deleted") - - Effect { st => - total.incr() - st match { - case StatusState.Found => foundCounter.incr() - case StatusState.NotFound => notFoundCounter.incr() - case StatusState.Partial => partialCounter.incr() - case StatusState.TimedOut => timedOutCounter.incr() - case StatusState.Failed => failedCounter.incr() - case StatusState.DeactivatedUser => deactivatedCounter.incr() - case StatusState.SuspendedUser => suspendedCounter.incr() - case StatusState.ProtectedUser => protectedCounter.incr() - case StatusState.ReportedTweet => reportedCounter.incr() - case StatusState.OverCapacity => overCapacityCounter.incr() - case StatusState.UnsupportedClient => unsupportedClientCounter.incr() - case StatusState.Drop => dropCounter.incr() - case StatusState.Suppress => suppressCounter.incr() - case StatusState.Deleted => deletedCounter.incr() - case StatusState.BounceDeleted => bounceDeletedCounter.incr() - case _ => - } - } - } - - def observeSetFieldsRequest(stats: StatsReceiver): Effect[SetAdditionalFieldsRequest] = - Effect { request => - val tweet = request.additionalFields - AdditionalFields.nonEmptyAdditionalFieldIds(tweet).foreach { id => - val fieldScope = "field_%d".format(id) - val fieldCounter = stats.counter(fieldScope) - val sizeStats = stats.stat(fieldScope) - - tweet.getFieldBlob(id).foreach { blob => - fieldCounter.incr() - sizeStats.add(blob.content.length) - } - } - } - - def observeSetRetweetVisibilityRequest( - stats: StatsReceiver - ): Effect[SetRetweetVisibilityRequest] = { - val setInvisibleCounter = stats.counter("set_invisible") - val setVisibleCounter = stats.counter("set_visible") - - Effect { request => - if (request.visible) setVisibleCounter.incr() else setInvisibleCounter.incr() - } - } - - def observeDeleteFieldsRequest(stats: StatsReceiver): Effect[DeleteAdditionalFieldsRequest] = { - val requestSizeStat = stats.stat("request_size") - - Effect { request => - requestSizeStat.add(request.tweetIds.size) - - request.fieldIds.foreach { id => - val fieldScope = "field_%d".format(id) - val fieldCounter = stats.counter(fieldScope) - fieldCounter.incr() - } - } - } - - def observeDeleteTweetsRequest(stats: StatsReceiver): Effect[DeleteTweetsRequest] = { - val requestSizeStat = stats.stat("request_size") - val userErasureTweetsStat = stats.counter("user_erasure_tweets") - val isBounceDeleteStat = stats.counter("is_bounce_delete_tweets") - - Effect { - case DeleteTweetsRequest(tweetIds, _, _, _, isUserErasure, _, isBounceDelete, _, _) => - requestSizeStat.add(tweetIds.size) - if (isUserErasure) { - userErasureTweetsStat.incr(tweetIds.size) - } - if (isBounceDelete) { - isBounceDeleteStat.incr(tweetIds.size) - } - } - } - - def observeRetweetRequest(stats: StatsReceiver): Effect[RetweetRequest] = { - val optionsScope = stats.scope("options") - val narrowcastCounter = optionsScope.counter("narrowcast") - val nullcastCounter = optionsScope.counter("nullcast") - val darkCounter = optionsScope.counter("dark") - val successOnDupCounter = optionsScope.counter("success_on_dup") - - Effect { request => - if (request.narrowcast.nonEmpty) narrowcastCounter.incr() - if (request.nullcast) nullcastCounter.incr() - if (request.dark) darkCounter.incr() - if (request.returnSuccessOnDuplicate) successOnDupCounter.incr() - } - } - - def observeScrubGeo(stats: StatsReceiver): Effect[GeoScrub] = { - val optionsScope = stats.scope("options") - val hosebirdEnqueueCounter = optionsScope.counter("hosebird_enqueue") - val requestSizeStat = stats.stat("request_size") - - Effect { request => - requestSizeStat.add(request.statusIds.size) - if (request.hosebirdEnqueue) hosebirdEnqueueCounter.incr() - } - } - - def observeEventOrRetry(stats: StatsReceiver, isRetry: Boolean): Unit = { - val statName = if (isRetry) "retry" else "event" - stats.counter(statName).incr() - } - - def observeAsyncInsertRequest(stats: StatsReceiver): Effect[AsyncInsertRequest] = { - val insertScope = stats.scope("insert") - val ageStat = insertScope.stat("age") - Effect { request => - observeEventOrRetry(insertScope, request.retryAction.isDefined) - ageStat.add(SnowflakeId.timeFromId(request.tweet.id).untilNow.inMillis) - } - } - - def observeAsyncSetAdditionalFieldsRequest( - stats: StatsReceiver - ): Effect[AsyncSetAdditionalFieldsRequest] = { - val setAdditionalFieldsScope = stats.scope("set_additional_fields") - Effect { request => - observeEventOrRetry(setAdditionalFieldsScope, request.retryAction.isDefined) - } - } - - def observeAsyncSetRetweetVisibilityRequest( - stats: StatsReceiver - ): Effect[AsyncSetRetweetVisibilityRequest] = { - val setRetweetVisibilityScope = stats.scope("set_retweet_visibility") - - Effect { request => - observeEventOrRetry(setRetweetVisibilityScope, request.retryAction.isDefined) - } - } - - def observeAsyncUndeleteTweetRequest(stats: StatsReceiver): Effect[AsyncUndeleteTweetRequest] = { - val undeleteTweetScope = stats.scope("undelete_tweet") - Effect { request => observeEventOrRetry(undeleteTweetScope, request.retryAction.isDefined) } - } - - def observeAsyncDeleteTweetRequest(stats: StatsReceiver): Effect[AsyncDeleteRequest] = { - val deleteTweetScope = stats.scope("delete_tweet") - Effect { request => observeEventOrRetry(deleteTweetScope, request.retryAction.isDefined) } - } - - def observeAsyncDeleteAdditionalFieldsRequest( - stats: StatsReceiver - ): Effect[AsyncDeleteAdditionalFieldsRequest] = { - val deleteAdditionalFieldsScope = stats.scope("delete_additional_fields") - Effect { request => - observeEventOrRetry( - deleteAdditionalFieldsScope, - request.retryAction.isDefined - ) - } - } - - def observeAsyncTakedownRequest(stats: StatsReceiver): Effect[AsyncTakedownRequest] = { - val takedownScope = stats.scope("takedown") - Effect { request => observeEventOrRetry(takedownScope, request.retryAction.isDefined) } - } - - def observeAsyncUpdatePossiblySensitiveTweetRequest( - stats: StatsReceiver - ): Effect[AsyncUpdatePossiblySensitiveTweetRequest] = { - val updatePossiblySensitiveTweetScope = stats.scope("update_possibly_sensitive_tweet") - Effect { request => - observeEventOrRetry(updatePossiblySensitiveTweetScope, request.action.isDefined) - } - } - - def observeReplicatedInsertTweetRequest(stats: StatsReceiver): Effect[Tweet] = { - val ageStat = stats.stat("age") // in milliseconds - Effect { request => ageStat.add(SnowflakeId.timeFromId(request.id).untilNow.inMillis) } - } - - def camelToUnderscore(str: String): String = { - val bldr = new StringBuilder - str.foldLeft(false) { (prevWasLowercase, c) => - if (prevWasLowercase && c.isUpper) { - bldr += '_' - } - bldr += c.toLower - c.isLower - } - bldr.result - } - - def observeAdditionalFields(stats: StatsReceiver): Effect[Tweet] = { - val additionalScope = stats.scope("additional_fields") - - Effect { tweet => - for (fieldId <- AdditionalFields.nonEmptyAdditionalFieldIds(tweet)) - additionalScope.counter(fieldId.toString).incr() - } - } - - /** - * We count how many tweets have each of these attributes so that we - * can observe general trends, as well as for tracking down the - * cause of behavior changes, like increased calls to certain - * services. - */ - def countTweetAttributes(stats: StatsReceiver, byClient: Boolean): Effect[Tweet] = { - val ageStat = stats.stat("age") - val tweetCounter = stats.counter("tweets") - val retweetCounter = stats.counter("retweets") - val repliesCounter = stats.counter("replies") - val inReplyToTweetCounter = stats.counter("in_reply_to_tweet") - val selfRepliesCounter = stats.counter("self_replies") - val directedAtCounter = stats.counter("directed_at") - val mentionsCounter = stats.counter("mentions") - val mentionsStat = stats.stat("mentions") - val urlsCounter = stats.counter("urls") - val urlsStat = stats.stat("urls") - val hashtagsCounter = stats.counter("hashtags") - val hashtagsStat = stats.stat("hashtags") - val mediaCounter = stats.counter("media") - val mediaStat = stats.stat("media") - val photosCounter = stats.counter("media", "photos") - val gifsCounter = stats.counter("media", "animated_gifs") - val videosCounter = stats.counter("media", "videos") - val cardsCounter = stats.counter("cards") - val card2Counter = stats.counter("card2") - val geoCoordsCounter = stats.counter("geo_coordinates") - val placeCounter = stats.counter("place") - val quotedTweetCounter = stats.counter("quoted_tweet") - val selfRetweetCounter = stats.counter("self_retweet") - val languageScope = stats.scope("language") - val textLengthStat = stats.stat("text_length") - val selfThreadCounter = stats.counter("self_thread") - val communitiesTweetCounter = stats.counter("communities") - - observeAdditionalFields(stats).also { - Effect[Tweet] { tweet => - def coreDataField[T](f: TweetCoreData => T): Option[T] = - tweet.coreData.map(f) - - def coreDataOptionField[T](f: TweetCoreData => Option[T]) = - coreDataField(f).flatten - - (SnowflakeId.isSnowflakeId(tweet.id) match { - case true => Some(SnowflakeId.timeFromId(tweet.id)) - case false => coreDataField(_.createdAtSecs.seconds.afterEpoch) - }).foreach { createdAt => ageStat.add(createdAt.untilNow.inSeconds) } - - if (!byClient) { - val mentions = getMentions(tweet) - val urls = getUrls(tweet) - val hashtags = getHashtags(tweet) - val media = getMedia(tweet) - val mediaKeys = media.flatMap(_.mediaKey) - val share = coreDataOptionField(_.share) - val selfThreadMetadata = getSelfThreadMetadata(tweet) - val communities = getCommunities(tweet) - - tweetCounter.incr() - if (share.isDefined) retweetCounter.incr() - if (coreDataOptionField(_.directedAtUser).isDefined) directedAtCounter.incr() - - coreDataOptionField(_.reply).foreach { reply => - repliesCounter.incr() - if (reply.inReplyToStatusId.nonEmpty) { - // repliesCounter counts all Tweets with a Reply struct, - // but that includes both directed-at Tweets and - // conversational replies. Only conversational replies - // have inReplyToStatusId present, so this counter lets - // us split apart those two cases. - inReplyToTweetCounter.incr() - } - - // Not all Tweet objects have CoreData yet isSelfReply() requires it. Thus, this - // invocation is guarded by the `coreDataOptionField(_.reply)` above. - if (isSelfReply(tweet)) selfRepliesCounter.incr() - } - - if (mentions.nonEmpty) mentionsCounter.incr() - if (urls.nonEmpty) urlsCounter.incr() - if (hashtags.nonEmpty) hashtagsCounter.incr() - if (media.nonEmpty) mediaCounter.incr() - if (selfThreadMetadata.nonEmpty) selfThreadCounter.incr() - if (communities.nonEmpty) communitiesTweetCounter.incr() - - mentionsStat.add(mentions.size) - urlsStat.add(urls.size) - hashtagsStat.add(hashtags.size) - mediaStat.add(media.size) - - if (mediaKeys.exists(MediaKeyClassifier.isImage(_))) photosCounter.incr() - if (mediaKeys.exists(MediaKeyClassifier.isGif(_))) gifsCounter.incr() - if (mediaKeys.exists(MediaKeyClassifier.isVideo(_))) videosCounter.incr() - - if (tweet.cards.exists(_.nonEmpty)) cardsCounter.incr() - if (tweet.card2.nonEmpty) card2Counter.incr() - if (coreDataOptionField(_.coordinates).nonEmpty) geoCoordsCounter.incr() - if (TweetLenses.place.get(tweet).nonEmpty) placeCounter.incr() - if (TweetLenses.quotedTweet.get(tweet).nonEmpty) quotedTweetCounter.incr() - if (share.exists(_.sourceUserId == getUserId(tweet))) selfRetweetCounter.incr() - - tweet.language - .map(_.language) - .foreach(lang => languageScope.counter(lang).incr()) - coreDataField(_.text).foreach(text => textLengthStat.add(codePointLength(text))) - } - } - } - } - -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/PostTweetObserver.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/PostTweetObserver.docx new file mode 100644 index 000000000..1f27cabfe Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/PostTweetObserver.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/PostTweetObserver.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/PostTweetObserver.scala deleted file mode 100644 index 6d20169d0..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/PostTweetObserver.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.escherbird.thriftscala.TweetEntityAnnotation -import com.twitter.tweetypie.thriftscala.BatchComposeMode -import com.twitter.tweetypie.thriftscala.PostTweetRequest -import com.twitter.tweetypie.thriftscala.PostTweetResult -import com.twitter.tweetypie.thriftscala.TweetCreateState -import com.twitter.util.Memoize - -private[service] object PostTweetObserver { - def observeResults(stats: StatsReceiver, byClient: Boolean): Effect[PostTweetResult] = { - val stateScope = stats.scope("state") - val tweetObserver = Observer.countTweetAttributes(stats, byClient) - - val stateCounters = - Memoize { st: TweetCreateState => stateScope.counter(Observer.camelToUnderscore(st.name)) } - - Effect { result => - stateCounters(result.state).incr() - if (result.state == TweetCreateState.Ok) result.tweet.foreach(tweetObserver) - } - } - - private def isCommunity(req: PostTweetRequest): Boolean = { - val CommunityGroupId = 8L - val CommunityDomainId = 31L - req.additionalFields - .flatMap(_.escherbirdEntityAnnotations).exists { e => - e.entityAnnotations.collect { - case TweetEntityAnnotation(CommunityGroupId, CommunityDomainId, _) => true - }.nonEmpty - } - } - - def observerRequest(stats: StatsReceiver): Effect[PostTweetRequest] = { - val optionsScope = stats.scope("options") - val narrowcastCounter = optionsScope.counter("narrowcast") - val nullcastCounter = optionsScope.counter("nullcast") - val inReplyToStatusIdCounter = optionsScope.counter("in_reply_to_status_id") - val placeIdCounter = optionsScope.counter("place_id") - val geoCoordinatesCounter = optionsScope.counter("geo_coordinates") - val placeMetadataCounter = optionsScope.counter("place_metadata") - val mediaUploadIdCounter = optionsScope.counter("media_upload_id") - val darkCounter = optionsScope.counter("dark") - val tweetToNarrowcastingCounter = optionsScope.counter("tweet_to_narrowcasting") - val autoPopulateReplyMetadataCounter = optionsScope.counter("auto_populate_reply_metadata") - val attachmentUrlCounter = optionsScope.counter("attachment_url") - val excludeReplyUserIdsCounter = optionsScope.counter("exclude_reply_user_ids") - val excludeReplyUserIdsStat = optionsScope.stat("exclude_reply_user_ids") - val uniquenessIdCounter = optionsScope.counter("uniqueness_id") - val batchModeScope = optionsScope.scope("batch_mode") - val batchModeFirstCounter = batchModeScope.counter("first") - val batchModeSubsequentCounter = batchModeScope.counter("subsequent") - val communitiesCounter = optionsScope.counter("communities") - - Effect { request => - if (request.narrowcast.nonEmpty) narrowcastCounter.incr() - if (request.nullcast) nullcastCounter.incr() - if (request.inReplyToTweetId.nonEmpty) inReplyToStatusIdCounter.incr() - if (request.geo.flatMap(_.placeId).nonEmpty) placeIdCounter.incr() - if (request.geo.flatMap(_.coordinates).nonEmpty) geoCoordinatesCounter.incr() - if (request.geo.flatMap(_.placeMetadata).nonEmpty) placeMetadataCounter.incr() - if (request.mediaUploadIds.nonEmpty) mediaUploadIdCounter.incr() - if (request.dark) darkCounter.incr() - if (request.enableTweetToNarrowcasting) tweetToNarrowcastingCounter.incr() - if (request.autoPopulateReplyMetadata) autoPopulateReplyMetadataCounter.incr() - if (request.attachmentUrl.nonEmpty) attachmentUrlCounter.incr() - if (request.excludeReplyUserIds.exists(_.nonEmpty)) excludeReplyUserIdsCounter.incr() - if (isCommunity(request)) communitiesCounter.incr() - if (request.uniquenessId.nonEmpty) uniquenessIdCounter.incr() - request.transientContext.flatMap(_.batchCompose).foreach { - case BatchComposeMode.BatchFirst => batchModeFirstCounter.incr() - case BatchComposeMode.BatchSubsequent => batchModeSubsequentCounter.incr() - case _ => - } - - excludeReplyUserIdsStat.add(request.excludeReplyUserIds.size) - } - } -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/ResultStateStats.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/ResultStateStats.docx new file mode 100644 index 000000000..8498f74a0 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/ResultStateStats.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/ResultStateStats.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/ResultStateStats.scala deleted file mode 100644 index b9cedf68e..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/ResultStateStats.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.finagle.stats.StatsReceiver - -/** - * "Result State" is, for every singular tweet read, we categorize the tweet - * result as a success or failure. - * These stats enable us to track true TPS success rates. - */ -private[service] case class ResultStateStats(private val underlying: StatsReceiver) { - private val stats = underlying.scope("result_state") - private val successCounter = stats.counter("success") - private val failedCounter = stats.counter("failed") - - def success(delta: Long = 1): Unit = successCounter.incr(delta) - def failed(delta: Long = 1): Unit = failedCounter.incr(delta) -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/StoredTweetsObserver.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/StoredTweetsObserver.docx new file mode 100644 index 000000000..424829705 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/StoredTweetsObserver.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/StoredTweetsObserver.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/StoredTweetsObserver.scala deleted file mode 100644 index 8a525c158..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/StoredTweetsObserver.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.tweetypie -package service -package observer - -import com.twitter.tweetypie.thriftscala.StoredTweetError -import com.twitter.tweetypie.thriftscala.StoredTweetInfo -import com.twitter.tweetypie.thriftscala.StoredTweetState.BounceDeleted -import com.twitter.tweetypie.thriftscala.StoredTweetState.ForceAdded -import com.twitter.tweetypie.thriftscala.StoredTweetState.HardDeleted -import com.twitter.tweetypie.thriftscala.StoredTweetState.NotFound -import com.twitter.tweetypie.thriftscala.StoredTweetState.SoftDeleted -import com.twitter.tweetypie.thriftscala.StoredTweetState.Undeleted -import com.twitter.tweetypie.thriftscala.StoredTweetState.UnknownUnionField - -private[service] trait StoredTweetsObserver { - - protected def observeStoredTweets( - storedTweets: Seq[StoredTweetInfo], - stats: StatsReceiver - ): Unit = { - val stateScope = stats.scope("state") - val errorScope = stats.scope("error") - - val sizeCounter = stats.counter("count") - sizeCounter.incr(storedTweets.size) - - val returnedStatesCount = storedTweets - .groupBy(_.storedTweetState match { - case None => "found" - case Some(_: HardDeleted) => "hard_deleted" - case Some(_: SoftDeleted) => "soft_deleted" - case Some(_: BounceDeleted) => "bounce_deleted" - case Some(_: Undeleted) => "undeleted" - case Some(_: ForceAdded) => "force_added" - case Some(_: NotFound) => "not_found" - case Some(_: UnknownUnionField) => "unknown" - }) - .mapValues(_.size) - - returnedStatesCount.foreach { - case (state, count) => stateScope.counter(state).incr(count) - } - - val returnedErrorsCount = storedTweets - .foldLeft(Seq[StoredTweetError]()) { (errors, storedTweetInfo) => - errors ++ storedTweetInfo.errors - } - .groupBy(_.name) - .mapValues(_.size) - - returnedErrorsCount.foreach { - case (error, count) => errorScope.counter(error).incr(count) - } - } - -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/package.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/package.docx new file mode 100644 index 000000000..7f2a8af97 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/package.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/package.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/package.scala deleted file mode 100644 index 4cfaea9f4..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/package.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.tweetypie -package service - -import com.twitter.util.Try - -package object observer { - - /** - * Generic Request/Result observer container for making observations on both requests/results. - */ - type ObserveExchange[Req, Res] = (Req, Try[Res]) - -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/package.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/package.docx new file mode 100644 index 000000000..7d5831fe4 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/package.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/package.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/package.scala deleted file mode 100644 index c6e0e861b..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/service/package.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.tweetypie - -import com.twitter.servo.request -import com.twitter.servo.request.ClientRequestAuthorizer - -package object service { - type ClientRequestAuthorizer = request.ClientRequestAuthorizer - - type UnauthorizedException = request.ClientRequestAuthorizer.UnauthorizedException - val UnauthorizedException: ClientRequestAuthorizer.UnauthorizedException.type = - request.ClientRequestAuthorizer.UnauthorizedException -} diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/store/AsyncEnqueueStore.docx b/tweetypie/server/src/main/scala/com/twitter/tweetypie/store/AsyncEnqueueStore.docx new file mode 100644 index 000000000..3dc5dcf61 Binary files /dev/null and b/tweetypie/server/src/main/scala/com/twitter/tweetypie/store/AsyncEnqueueStore.docx differ diff --git a/tweetypie/server/src/main/scala/com/twitter/tweetypie/store/AsyncEnqueueStore.scala b/tweetypie/server/src/main/scala/com/twitter/tweetypie/store/AsyncEnqueueStore.scala deleted file mode 100644 index 3ad816e40..000000000 --- a/tweetypie/server/src/main/scala/com/twitter/tweetypie/store/AsyncEnqueueStore.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.tweetypie -package store - -import com.twitter.tweetypie.thriftscala._ - -/** - * AsyncEnqueueStore converts certains TweetStoreEvent types into their async-counterpart - * events, and enqueues those to a deferredrpc-backed ThriftTweetService instance. - */ -trait AsyncEnqueueStore - extends TweetStoreBase[AsyncEnqueueStore] - with InsertTweet.Store - with DeleteTweet.Store - with UndeleteTweet.Store - with IncrFavCount.Store - with IncrBookmarkCount.Store - with SetAdditionalFields.Store - with SetRetweetVisibility.Store - with Takedown.Store - with DeleteAdditionalFields.Store - with UpdatePossiblySensitiveTweet.Store { - def wrap(w: TweetStore.Wrap): AsyncEnqueueStore = - new TweetStoreWrapper[AsyncEnqueueStore](w, this) - with AsyncEnqueueStore - with InsertTweet.StoreWrapper - with DeleteTweet.StoreWrapper - with UndeleteTweet.StoreWrapper - with IncrFavCount.StoreWrapper - with IncrBookmarkCount.StoreWrapper - with SetAdditionalFields.StoreWrapper - with SetRetweetVisibility.StoreWrapper - with Takedown.StoreWrapper - with DeleteAdditionalFields.StoreWrapper - with UpdatePossiblySensitiveTweet.StoreWrapper -} - -object AsyncEnqueueStore { - def apply( - tweetService: ThriftTweetService, - scrubUserInAsyncInserts: User => User, - scrubSourceTweetInAsyncInserts: Tweet => Tweet, - scrubSourceUserInAsyncInserts: User => User - ): AsyncEnqueueStore = - new AsyncEnqueueStore { - override val insertTweet: FutureEffect[InsertTweet.Event] = - FutureEffect[InsertTweet.Event] { e => - tweetService.asyncInsert( - e.toAsyncRequest( - scrubUserInAsyncInserts, - scrubSourceTweetInAsyncInserts, - scrubSourceUserInAsyncInserts - ) - ) - } - - override val deleteTweet: FutureEffect[DeleteTweet.Event] = - FutureEffect[DeleteTweet.Event] { e => tweetService.asyncDelete(e.toAsyncRequest) } - - override val undeleteTweet: FutureEffect[UndeleteTweet.Event] = - FutureEffect[UndeleteTweet.Event] { e => - tweetService.asyncUndeleteTweet(e.toAsyncUndeleteTweetRequest) - } - - override val incrFavCount: FutureEffect[IncrFavCount.Event] = - FutureEffect[IncrFavCount.Event] { e => tweetService.asyncIncrFavCount(e.toAsyncRequest) } - - override val incrBookmarkCount: FutureEffect[IncrBookmarkCount.Event] = - FutureEffect[IncrBookmarkCount.Event] { e => - tweetService.asyncIncrBookmarkCount(e.toAsyncRequest) - } - - override val setAdditionalFields: FutureEffect[SetAdditionalFields.Event] = - FutureEffect[SetAdditionalFields.Event] { e => - tweetService.asyncSetAdditionalFields(e.toAsyncRequest) - } - - override val setRetweetVisibility: FutureEffect[SetRetweetVisibility.Event] = - FutureEffect[SetRetweetVisibility.Event] { e => - tweetService.asyncSetRetweetVisibility(e.toAsyncRequest) - } - - override val deleteAdditionalFields: FutureEffect[DeleteAdditionalFields.Event] = - FutureEffect[DeleteAdditionalFields.Event] { e => - tweetService.asyncDeleteAdditionalFields(e.toAsyncRequest) - } - - override val updatePossiblySensitiveTweet: FutureEffect[UpdatePossiblySensitiveTweet.Event] = - FutureEffect[UpdatePossiblySensitiveTweet.Event] { e => - tweetService.asyncUpdatePossiblySensitiveTweet(e.toAsyncRequest) - } - - override val takedown: FutureEffect[Takedown.Event] = - FutureEffect[Takedown.Event] { e => tweetService.asyncTakedown(e.toAsyncRequest) } - } -}