Mirrors/the-algorithm
2
0
Fork 0
mirror of https://github.com/twitter/the-algorithm.git synced 2024-12-22 10:11:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 645342ba2f into 72eda9a24f

Browse Source
This commit is contained in:
ImperiumTakp 2023-07-17 21:42:19 -05:00 committed by GitHub
commit b572e6b1ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/scala/com/twitter/simclusters_v2/scalding/KnownForSources.scala
View File

@ -102,23 +102,23 @@ object KnownForSources {
TypedPipe TypedPipe
.from(TextLine(textFile)) .from(TextLine(textFile))
.flatMap { str => .flatMap { str =>
if (!str.startsWith("#")) { str match {
try { case s"#$_" => None
case _ => try {
val tokens = str.trim.split("\\s+") val tokens = str.trim.split("\\s+")
val res = Array.newBuilder[(Int, Float)]
val userId = tokens(0).toLong val userId = tokens(0).toLong
for (i <- 1 until tokens.length) { (1 until tokens.length).foldRight(Array.newBuilder[(Int, Float)])((i, r) => {
val Array(cIdStr, scoreStr) = tokens(i).split(":") val Array(cIdStr, scoreStr) = tokens(i).split(":")
val clusterId = cIdStr.toInt val clusterId = cIdStr.toInt
val score = scoreStr.toFloat val score = scoreStr.toFloat
val newEntry = (clusterId, score) val newEntry = (clusterId, score)
res += newEntry r += newEntry
}).result() match {
case (res) if res.nonEmpty => Some((userId, res.result()))
_ => None
} }
val result = res.result }
if (result.nonEmpty) { catch {
Some((userId, res.result()))
} else None
} catch {
case ex: Throwable => case ex: Throwable =>
log.warning( log.warning(
s"Error while loading knownFor from $textFile for line <$str>: " + s"Error while loading knownFor from $textFile for line <$str>: " +
@ -126,7 +126,7 @@ object KnownForSources {
) )
None None
} }
} else None }
} }
} }