Mirrors/the-algorithm
2
0
Fork 0
mirror of https://github.com/twitter/the-algorithm.git synced 2024-12-22 10:11:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge e1691d5f25 into 72eda9a24f

Browse Source
This commit is contained in:
ImperiumTakp 2023-07-17 21:40:32 -05:00 committed by GitHub
commit 7382d72eaa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 42 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
src/scala/com/twitter/simclusters_v2/scalding/UpdateKnownFor.scala
View File

@ -120,32 +120,28 @@ object UpdateKnownFor {
clusterScoresToFinalScore: ClusterScoresForNode => Double,
minNeighborsInCluster: Int
): Option[(Int, Double)] = {
val clusterToScores = statsOfClustersInNeighborhood.toList.flatMap {
case (clusterId, statsInNeighborhood) =>
val clusterOverallStats = clusterOverallStatsMap(clusterId)
if (statsInNeighborhood.nodeCount >= minNeighborsInCluster) {
Some(
(
clusterId,
clusterScoresToFinalScore(
getScoresForCluster(
overallNeighborhoodStats,
statsInNeighborhood,
clusterOverallStats.nodeCount,
clusterOverallStats.sumOfMembershipWeights,
globalAvgEdgeWeight,
truePositiveWtFactor
)
statsOfClustersInNeighborhood
.toList
.flatMap {
case (clusterId, statsInNeighborhood) =>
val clusterOverallStats = clusterOverallStatsMap(clusterId)
statsInNeighborhood match {
case statsInNeighborhood if statsInNeighborhood.nodeCount < minNeighborsInCluster => None
case _ =>
val score = getScoresForCluster(
overallNeighborhoodStats,
statsInNeighborhood,
clusterOverallStats.nodeCount,
clusterOverallStats.sumOfMembershipWeights,
globalAvgEdgeWeight,
truePositiveWtFactor
)
)
)
} else {
None
}
Some((clusterId, clusterScoresToFinalScore(score)))
}
} match {
case scores if scores.isEmpty => None
case _ => Some(clusterToScores.maxBy(_._2))
}
if (clusterToScores.nonEmpty) {
Some(clusterToScores.maxBy(_._2))
} else None
}
def updateGeneric(
@ -170,35 +166,32 @@ object UpdateKnownFor {
collectInformationPerNode(graph, inputUserToClusters, avgMembershipScore)
.mapValues {
case NodeInformation(originalClusters, overallStats, statsOfClustersInNeighborhood) =>
val newClusterWithScoreOpt = if (overallStats.nodeCount < minNeighborsInCluster) {
nodesWithSmallDegree.inc()
None
} else {
pickBestCluster(
overallStats,
statsOfClustersInNeighborhood,
clusterOverallStatsMap,
globalAvgWeight,
truePositiveWtFactor,
clusterScoresToFinalScore,
minNeighborsInCluster
)
}
newClusterWithScoreOpt match {
overallStats match {
case stats if (overallStats.nodeCount < minNeighborsInCluster) =>
nodesWithSmallDegree.inc()
None
case _ =>
pickBestCluster(
overallStats,
statsOfClustersInNeighborhood,
clusterOverallStatsMap,
globalAvgWeight,
truePositiveWtFactor,
clusterScoresToFinalScore,
minNeighborsInCluster
)
} match {
case Some((newClusterId, score)) =>
if (originalClusters.isEmpty) {
emptyToSomething.inc()
} else if (originalClusters.contains(newClusterId)) {
sameCluster.inc()
} else {
diffCluster.inc()
originalClusters match {
case cluster if cluster.isEmpty => emptyToSomething.inc()
case cluster if cluster.contains(newClusterId) => sameCluster.inc()
case _ => diffCluster.inc()
}
Array((newClusterId, score.toFloat))
case None =>
if (originalClusters.isEmpty) {
emptyToEmpty.inc()
} else {
somethingToEmpty.inc()
originalClusters match {
case cluster if cluster.isEmpty => emptyToEmpty.inc()
case _ => somethingToEmpty.inc()
}
Array.empty[(Int, Float)]
}