Mirrors/the-algorithm
2
0
Fork 0
mirror of https://github.com/twitter/the-algorithm.git synced 2024-12-22 10:11:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge e1691d5f25 into 72eda9a24f

Browse Source
This commit is contained in:
ImperiumTakp 2023-07-17 21:40:32 -05:00 committed by GitHub
commit 7382d72eaa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 42 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
src/scala/com/twitter/simclusters_v2/scalding/UpdateKnownFor.scala
View File

@ -120,32 +120,28 @@ object UpdateKnownFor {
clusterScoresToFinalScore: ClusterScoresForNode => Double, clusterScoresToFinalScore: ClusterScoresForNode => Double,
minNeighborsInCluster: Int minNeighborsInCluster: Int
): Option[(Int, Double)] = { ): Option[(Int, Double)] = {
val clusterToScores = statsOfClustersInNeighborhood.toList.flatMap { statsOfClustersInNeighborhood
case (clusterId, statsInNeighborhood) => .toList
val clusterOverallStats = clusterOverallStatsMap(clusterId) .flatMap {
if (statsInNeighborhood.nodeCount >= minNeighborsInCluster) { case (clusterId, statsInNeighborhood) =>
Some( val clusterOverallStats = clusterOverallStatsMap(clusterId)
( statsInNeighborhood match {
clusterId, case statsInNeighborhood if statsInNeighborhood.nodeCount < minNeighborsInCluster => None
clusterScoresToFinalScore( case _ =>
getScoresForCluster( val score = getScoresForCluster(
overallNeighborhoodStats, overallNeighborhoodStats,
statsInNeighborhood, statsInNeighborhood,
clusterOverallStats.nodeCount, clusterOverallStats.nodeCount,
clusterOverallStats.sumOfMembershipWeights, clusterOverallStats.sumOfMembershipWeights,
globalAvgEdgeWeight, globalAvgEdgeWeight,
truePositiveWtFactor truePositiveWtFactor
)
) )
) Some((clusterId, clusterScoresToFinalScore(score)))
) }
} else { } match {
None case scores if scores.isEmpty => None
} case _ => Some(clusterToScores.maxBy(_._2))
} }
if (clusterToScores.nonEmpty) {
Some(clusterToScores.maxBy(_._2))
} else None
} }
def updateGeneric( def updateGeneric(
@ -170,35 +166,32 @@ object UpdateKnownFor {
collectInformationPerNode(graph, inputUserToClusters, avgMembershipScore) collectInformationPerNode(graph, inputUserToClusters, avgMembershipScore)
.mapValues { .mapValues {
case NodeInformation(originalClusters, overallStats, statsOfClustersInNeighborhood) => case NodeInformation(originalClusters, overallStats, statsOfClustersInNeighborhood) =>
val newClusterWithScoreOpt = if (overallStats.nodeCount < minNeighborsInCluster) { overallStats match {
nodesWithSmallDegree.inc() case stats if (overallStats.nodeCount < minNeighborsInCluster) =>
None nodesWithSmallDegree.inc()
} else { None
pickBestCluster( case _ =>
overallStats, pickBestCluster(
statsOfClustersInNeighborhood, overallStats,
clusterOverallStatsMap, statsOfClustersInNeighborhood,
globalAvgWeight, clusterOverallStatsMap,
truePositiveWtFactor, globalAvgWeight,
clusterScoresToFinalScore, truePositiveWtFactor,
minNeighborsInCluster clusterScoresToFinalScore,
) minNeighborsInCluster
} )
newClusterWithScoreOpt match { } match {
case Some((newClusterId, score)) => case Some((newClusterId, score)) =>
if (originalClusters.isEmpty) { originalClusters match {
emptyToSomething.inc() case cluster if cluster.isEmpty => emptyToSomething.inc()
} else if (originalClusters.contains(newClusterId)) { case cluster if cluster.contains(newClusterId) => sameCluster.inc()
sameCluster.inc() case _ => diffCluster.inc()
} else {
diffCluster.inc()
} }
Array((newClusterId, score.toFloat)) Array((newClusterId, score.toFloat))
case None => case None =>
if (originalClusters.isEmpty) { originalClusters match {
emptyToEmpty.inc() case cluster if cluster.isEmpty => emptyToEmpty.inc()
} else { case _ => somethingToEmpty.inc()
somethingToEmpty.inc()
} }
Array.empty[(Int, Float)] Array.empty[(Int, Float)]
} }