Repository: spark Updated Branches: refs/heads/branch-2.2 5dd14f5d9 -> 66c7b4281
[SPARK-25576][BUILD][BRANCH-2.2] Fix lint failure ## What changes were proposed in this pull request? Line length fixes and ## How was this patch tested? Manually verified, but will ensure jenkins lint passes before merging Related Job: https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Compile/job/spark-branch-2.2-lint/913/console Closes #22596 from samdvr/SPARK-25576. Lead-authored-by: Sam Davarnia <[email protected]> Co-authored-by: Sam Davarnia <> Co-authored-by: Dongjoon Hyun <[email protected]> Co-authored-by: Sam Davarnia <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/66c7b428 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/66c7b428 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/66c7b428 Branch: refs/heads/branch-2.2 Commit: 66c7b42811f9a46726738da8e601d08eb2486b6e Parents: 5dd14f5 Author: Sam Davarnia <[email protected]> Authored: Tue Oct 2 10:13:54 2018 -0700 Committer: Dongjoon Hyun <[email protected]> Committed: Tue Oct 2 10:13:54 2018 -0700 ---------------------------------------------------------------------- .../spark/unsafe/types/UTF8StringSuite.java | 2 ++ .../unsafe/sort/UnsafeInMemorySorter.java | 9 ++++++--- .../unsafe/sort/UnsafeExternalSorterSuite.java | 21 ++++++++++++-------- .../unsafe/sort/UnsafeInMemorySorterSuite.java | 3 ++- .../sql/execution/UnsafeKVExternalSorter.java | 2 +- 5 files changed, 24 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/66c7b428/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java ---------------------------------------------------------------------- diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java index 33b9e11..ea79c28 100644 --- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java +++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java @@ -63,7 +63,9 @@ public class UTF8StringSuite { checkBasic("hello", 5); // 5 * 1 byte chars checkBasic("大 å ä¸ ç", 7); checkBasic("︽ï¹ï¼ ", 3); // 3 * 3 bytes chars + // checkstyle.off: AvoidEscapedUnicodeCharacters checkBasic("\uD83E\uDD19", 1); // 4 bytes char + // checkstyle.on: AvoidEscapedUnicodeCharacters } @Test http://git-wip-us.apache.org/repos/asf/spark/blob/66c7b428/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java index 839b41d..b025811 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java @@ -174,9 +174,12 @@ public final class UnsafeInMemorySorter { if (consumer != null) { consumer.freeArray(array); // the call to consumer.allocateArray may trigger a spill - // which in turn access this instance and eventually re-enter this method and try to free the array again. - // by setting the array to null and its length to 0 we effectively make the spill code-path a no-op. - // setting the array to null also indicates that it has already been de-allocated which prevents a double de-allocation in free(). + // which in turn access this instance and eventually re-enter this method + // and try to free the array again. + // By setting the array to null and its length to 0 + // we effectively make the spill code-path a no-op. + // Setting the array to null also indicates that it has already been + // de-allocated which prevents a double de-allocation in free(). array = null; usableCapacity = 0; pos = 0; http://git-wip-us.apache.org/repos/asf/spark/blob/66c7b428/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java index cce01a3..17c4d7e 100644 --- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java +++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java @@ -460,7 +460,7 @@ public class UnsafeExternalSorterSuite { @Test public void testOOMDuringSpill() throws Exception { final UnsafeExternalSorter sorter = newSorter(); - // we assume that given default configuration, + // We assume that given default configuration, // the size of the data we insert to the sorter (ints) // and assuming we shouldn't spill before pointers array is exhausted // (memory manager is not configured to throw at this point) @@ -469,23 +469,28 @@ public class UnsafeExternalSorterSuite { for (int i = 0; sorter.hasSpaceForAnotherRecord(); ++i) { insertNumber(sorter, i); } - // we expect the next insert to attempt growing the pointerssArray - // first allocation is expected to fail, then a spill is triggered which attempts another allocation + // We expect the next insert to attempt growing the pointerssArray + // first allocation is expected to fail, then a spill is + // triggered which attempts another allocation // which also fails and we expect to see this OOM here. // the original code messed with a released array within the spill code // and ended up with a failed assertion. - // we also expect the location of the OOM to be org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset + // We also expect the location of the OOM to be + // org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset memoryManager.markconsequentOOM(2); try { insertNumber(sorter, 1024); fail("expected OutOfMmoryError but it seems operation surprisingly succeeded"); } - // we expect an OutOfMemoryError here, anything else (i.e the original NPE is a failure) + // We expect an OutOfMemoryError here, anything else + // (i.e the original NPE is a failure) catch (OutOfMemoryError oom){ String oomStackTrace = Utils.exceptionString(oom); - assertThat("expected OutOfMemoryError in org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset", - oomStackTrace, - Matchers.containsString("org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset")); + assertThat("expected OutOfMemoryError in " + + "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset", + oomStackTrace, + Matchers.containsString( + "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset")); } } } http://git-wip-us.apache.org/repos/asf/spark/blob/66c7b428/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java index cfb0030..c145532 100644 --- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java +++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java @@ -183,7 +183,8 @@ public class UnsafeInMemorySorterSuite { } catch (OutOfMemoryError oom) { // as expected } - // [SPARK-21907] this failed on NPE at org.apache.spark.memory.MemoryConsumer.freeArray(MemoryConsumer.java:108) + // [SPARK-21907] this failed on NPE at + // org.apache.spark.memory.MemoryConsumer.freeArray(MemoryConsumer.java:108) sorter.free(); // simulate a 'back to back' free. sorter.free(); http://git-wip-us.apache.org/repos/asf/spark/blob/66c7b428/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java ---------------------------------------------------------------------- diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java index 7549dec..44bc681 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java @@ -259,7 +259,7 @@ public final class UnsafeKVExternalSorter { Object baseObj2, long baseOff2, int baseLen2) { - // Note that since ordering doesn't need the total length of the record, we just pass -1 + // Note that since ordering doesn't need the total length of the record, we just pass -1 // into the row. row1.pointTo(baseObj1, baseOff1 + 4, -1); row2.pointTo(baseObj2, baseOff2 + 4, -1); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
