Mirrors/the-algorithm
2
0
Fork 0
mirror of https://github.com/twitter/the-algorithm.git synced 2024-06-02 17:28:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

Change if refactored to match

Browse Source
This commit is contained in:
denon1 2023-04-01 02:04:07 +02:00
parent 573fab9526
commit 376f35f08f
1 changed files with 23 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/scala/com/twitter/simclusters_v2/scalding/KnownForSources.scala
View File

@ -102,28 +102,30 @@ object KnownForSources {
TypedPipe
.from(TextLine(textFile))
.flatMap { str =>
if (str.startsWith("#")) {none}
try {
val tokens = str.trim.split("\\s+")
val userId = tokens(0).toLong
(1 until tokens.length).foldRight(Array.newBuilder[(Int, Float)])((i, r) => {
val Array(cIdStr, scoreStr) = tokens(i).split(":")
val clusterId = cIdStr.toInt
val score = scoreStr.toFloat
val newEntry = (clusterId, score)
r += newEntry
}).result() match {
case (res) if res.nonEmpty => Some((userId, res.result()))
_ => none
str match {
case s"#$_" => none
case _ => try {
val tokens = str.trim.split("\\s+")
val userId = tokens(0).toLong
(1 until tokens.length).foldRight(Array.newBuilder[(Int, Float)])((i, r) => {
val Array(cIdStr, scoreStr) = tokens(i).split(":")
val clusterId = cIdStr.toInt
val score = scoreStr.toFloat
val newEntry = (clusterId, score)
r += newEntry
}).result() match {
case (res) if res.nonEmpty => Some((userId, res.result()))
_ => none
}
}
catch {
case ex: Throwable =>
log.warning(
s"Error while loading knownFor from $textFile for line <$str>: " +
ex.getMessage
)
None
}
}
catch {
case ex: Throwable =>
log.warning(
s"Error while loading knownFor from $textFile for line <$str>: " +
ex.getMessage
)
None
}
}
}