Github user jkbradley commented on a diff in the pull request:
https://github.com/apache/spark/pull/15018#discussion_r95094550
--- Diff:
mllib/src/main/scala/org/apache/spark/mllib/regression/IsotonicRegression.scala
---
@@ -328,74 +336,80 @@ class IsotonicRegression private (private var
isotonic: Boolean) extends Seriali
return Array.empty
}
- // Pools sub array within given bounds assigning weighted average
value to all elements.
- def pool(input: Array[(Double, Double, Double)], start: Int, end:
Int): Unit = {
- val poolSubArray = input.slice(start, end + 1)
- val weightedSum = poolSubArray.map(lp => lp._1 * lp._3).sum
- val weight = poolSubArray.map(_._3).sum
+ // Keeps track of the start and end indices of the blocks. if [i, j]
is a valid block from
+ // input(i) to input(j) (inclusive), then blockBounds(i) = j and
blockBounds(j) = i
+ val blockBounds = Array.range(0, input.length) // Initially, each data
point is its own block
- var i = start
- while (i <= end) {
- input(i) = (weightedSum / weight, input(i)._2, input(i)._3)
- i = i + 1
- }
+ // Keep track of the sum of weights and sum of weight * y for each
block. weights(start)
+ // gives the values for the block. Entries that are not at the start
of a block
+ // are meaningless.
+ val weights: Array[(Double, Double)] = input.map { case (y, _, weight)
=>
+ require(weight != 0.0)
+ (weight, weight * y)
}
- var i = 0
- val len = input.length
- while (i < len) {
- var j = i
+ // a few convenience functions to make the code more readable
+
+ // blockStart and blockEnd have identical implementations. We create
two different
+ // functions to make the code more expressive
+ def blockEnd(start: Int): Int = blockBounds(start)
+ def blockStart(end: Int): Int = blockBounds(end)
+
+ // the next block starts at the index after the end of this block
+ def nextBlock(start: Int): Int = blockEnd(start) + 1
+
+ // the previous block ends at the index before the start of this block
+ // we then use blockStart to find the start
+ def prevBlock(start: Int): Int = blockStart(start - 1)
+
+ // Merge two adjacent blocks, updating blockBounds and weights to
reflect the merge
+ // Return the start index of the merged block
+ def merge(block1: Int, block2: Int): Int = {
+ assert(blockEnd(block1) + 1 == block2, "attempting to merge
non-consecutive blocks")
--- End diff --
Please make the error message more informative:
* Make it clear that this indicates an internal bug within
IsotonicRegression
* State the invalid values
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
