Repository: hive Updated Branches: refs/heads/master 96f591446 -> 0f934d29b
HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun Radhakrishnan, reviewed by Daniel Dai) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0f934d29 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0f934d29 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0f934d29 Branch: refs/heads/master Commit: 0f934d29bb6bfc992eca050337b8f48fb47aaed8 Parents: 96f5914 Author: Daniel Dai <da...@hortonworks.com> Authored: Wed Aug 5 13:50:19 2015 -0700 Committer: Daniel Dai <da...@hortonworks.com> Committed: Wed Aug 5 13:50:19 2015 -0700 ---------------------------------------------------------------------- .../hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java | 7 ++++--- .../mapreduce/DynamicPartitionFileRecordWriterContainer.java | 3 ++- .../hive/hcatalog/mapreduce/FileOutputFormatContainer.java | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java index 3a07b0c..f620b83 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java @@ -51,8 +51,9 @@ class DefaultOutputFormatContainer extends OutputFormatContainer { super(of); } - static synchronized String getOutputName(int partition) { - return "part-" + NUMBER_FORMAT.format(partition); + static synchronized String getOutputName(TaskAttemptContext context) { + return context.getConfiguration().get("mapreduce.output.basename", "part") + + "-" + NUMBER_FORMAT.format(context.getTaskAttemptID().getTaskID().getId()); } /** @@ -65,7 +66,7 @@ class DefaultOutputFormatContainer extends OutputFormatContainer { @Override public RecordWriter<WritableComparable<?>, HCatRecord> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { - String name = getOutputName(context.getTaskAttemptID().getTaskID().getId()); + String name = getOutputName(context); return new DefaultRecordWriterContainer(context, getBaseOutputFormat().getRecordWriter(null, new JobConf(context.getConfiguration()), name, InternalUtil.createReporter(context))); } http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java index 60f1b60..320ace4 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java @@ -205,7 +205,8 @@ class DynamicPartitionFileRecordWriterContainer extends FileRecordWriterContaine Path parentDir = new Path(currTaskContext.getConfiguration().get("mapred.work.output.dir")); Path childPath = - new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, "part", "")); + new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, + currTaskContext.getConfiguration().get("mapreduce.output.basename", "part"), "")); RecordWriter baseRecordWriter = baseOF.getRecordWriter(parentDir.getFileSystem(currTaskContext.getConfiguration()), http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java index 001b59b..95ee3b4 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java @@ -97,7 +97,8 @@ class FileOutputFormatContainer extends OutputFormatContainer { (org.apache.hadoop.mapred.RecordWriter)null, context); } else { Path parentDir = new Path(context.getConfiguration().get("mapred.work.output.dir")); - Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()), "part")); + Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()), + context.getConfiguration().get("mapreduce.output.basename", "part"))); rw = new StaticPartitionFileRecordWriterContainer( getBaseOutputFormat().getRecordWriter(