PARQUET-685 - Deprecated ParquetInputSplit constructor passes parametâ¦
The problem was not discovered because the test was bugous. Updated both sides. Author: Gabor Szadovszky <gabor.szadovs...@cloudera.com> Closes #372 from gszadovszky/PARQUET-685 and squashes the following commits: 9cbeee2 [Gabor Szadovszky] PARQUET-685 - Deprecated ParquetInputSplit constructor passes parameters in the wrong order. Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/7fdeba2a Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/7fdeba2a Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/7fdeba2a Branch: refs/heads/parquet-1.8.x Commit: 7fdeba2aaa38cb5f065fc98f3477747be36997cc Parents: 21940ee Author: Gabor Szadovszky <gabor.szadovs...@cloudera.com> Authored: Wed Oct 5 13:21:40 2016 -0700 Committer: Ryan Blue <b...@apache.org> Committed: Mon Jan 9 16:54:54 2017 -0800 ---------------------------------------------------------------------- .../main/java/org/apache/parquet/hadoop/ParquetInputSplit.java | 3 +-- .../test/java/org/apache/parquet/hadoop/TestInputFormat.java | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/7fdeba2a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java ---------------------------------------------------------------------- diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java index 8a13a3c..b97daa5 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java @@ -26,7 +26,6 @@ import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.zip.GZIPInputStream; @@ -89,7 +88,7 @@ public class ParquetInputSplit extends FileSplit implements Writable { String fileSchema, Map<String, String> extraMetadata, Map<String, String> readSupportMetadata) { - this(path, start, length, end(blocks, requestedSchema), hosts, offsets(blocks)); + this(path, start, end(blocks, requestedSchema), length, hosts, offsets(blocks)); } private static long end(List<BlockMetaData> blocks, String requestedSchema) { http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/7fdeba2a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java ---------------------------------------------------------------------- diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java index 9fe3008..d8b3de3 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java @@ -481,11 +481,8 @@ public class TestInputFormat { .generateSplitInfo(blocks, hdfsBlocks, min, max); for (ClientSideMetadataSplitStrategy.SplitInfo splitInfo : splitInfos) { - BlockMetaData lastRowGroup = splitInfo.getRowGroups().get(splitInfo.getRowGroupCount() - 1); - long end = lastRowGroup.getStartingPos() + lastRowGroup.getTotalByteSize(); - ParquetInputSplit split = new ParquetInputSplit(fileStatus.getPath(), - splitInfo.hdfsBlock.getOffset(), end, splitInfo.hdfsBlock.getHosts(), + splitInfo.hdfsBlock.getOffset(), splitInfo.hdfsBlock.getLength(), splitInfo.hdfsBlock.getHosts(), splitInfo.rowGroups, schema.toString(), null, null, extramd); splits.add(split); }
