[48/53] [abbrv] hive git commit: HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun Radhakrishnan, reviewed by Daniel Dai)

2015-08-05 Thread sershe 
HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun 
Radhakrishnan, reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0f934d29
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0f934d29
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0f934d29

Branch: refs/heads/llap
Commit: 0f934d29bb6bfc992eca050337b8f48fb47aaed8
Parents: 96f5914
Author: Daniel Dai 
Authored: Wed Aug 5 13:50:19 2015 -0700
Committer: Daniel Dai 
Committed: Wed Aug 5 13:50:19 2015 -0700

--
 .../hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java | 7 ---
 .../mapreduce/DynamicPartitionFileRecordWriterContainer.java  | 3 ++-
 .../hive/hcatalog/mapreduce/FileOutputFormatContainer.java| 3 ++-
 3 files changed, 8 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
index 3a07b0c..f620b83 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
@@ -51,8 +51,9 @@ class DefaultOutputFormatContainer extends 
OutputFormatContainer {
 super(of);
   }
 
-  static synchronized String getOutputName(int partition) {
-return "part-" + NUMBER_FORMAT.format(partition);
+  static synchronized String getOutputName(TaskAttemptContext context) {
+return context.getConfiguration().get("mapreduce.output.basename", "part")
++ "-" + 
NUMBER_FORMAT.format(context.getTaskAttemptID().getTaskID().getId());
   }
 
   /**
@@ -65,7 +66,7 @@ class DefaultOutputFormatContainer extends 
OutputFormatContainer {
   @Override
   public RecordWriter, HCatRecord>
   getRecordWriter(TaskAttemptContext context) throws IOException, 
InterruptedException {
-String name = 
getOutputName(context.getTaskAttemptID().getTaskID().getId());
+String name = getOutputName(context);
 return new DefaultRecordWriterContainer(context,
   getBaseOutputFormat().getRecordWriter(null, new 
JobConf(context.getConfiguration()), name, 
InternalUtil.createReporter(context)));
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
index 60f1b60..320ace4 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
@@ -205,7 +205,8 @@ class DynamicPartitionFileRecordWriterContainer extends 
FileRecordWriterContaine
 
   Path parentDir = new 
Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
   Path childPath =
-  new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, 
"part", ""));
+  new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext,
+  
currTaskContext.getConfiguration().get("mapreduce.output.basename", "part"), 
""));
 
   RecordWriter baseRecordWriter =
   
baseOF.getRecordWriter(parentDir.getFileSystem(currTaskContext.getConfiguration()),

http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
index 001b59b..95ee3b4 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
@@ -97,7 +97,8 @@ class FileOutputFormatContainer extends OutputFormatContainer 
{
   (org.apache.hadoop.mapred.RecordWriter)null, context);
 } else {
   Path parentDir = new 
Path(context.getConfiguration().get("mapred.work.output.dir"));

hive git commit: HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun Radhakrishnan, reviewed by Daniel Dai)

2015-08-05 Thread daijy 
Repository: hive
Updated Branches:
  refs/heads/branch-1.2 4ccc0c37a -> c4dc7b5f7


HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun 
Radhakrishnan, reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c4dc7b5f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c4dc7b5f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c4dc7b5f

Branch: refs/heads/branch-1.2
Commit: c4dc7b5f7714a1ef9902ced964fa800e46eb6c0f
Parents: 4ccc0c3
Author: Daniel Dai 
Authored: Wed Aug 5 13:50:19 2015 -0700
Committer: Daniel Dai 
Committed: Wed Aug 5 13:51:29 2015 -0700

--
 .../hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java | 7 ---
 .../mapreduce/DynamicPartitionFileRecordWriterContainer.java  | 3 ++-
 .../hive/hcatalog/mapreduce/FileOutputFormatContainer.java| 3 ++-
 3 files changed, 8 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/c4dc7b5f/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
index 3a07b0c..f620b83 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
@@ -51,8 +51,9 @@ class DefaultOutputFormatContainer extends 
OutputFormatContainer {
 super(of);
   }
 
-  static synchronized String getOutputName(int partition) {
-return "part-" + NUMBER_FORMAT.format(partition);
+  static synchronized String getOutputName(TaskAttemptContext context) {
+return context.getConfiguration().get("mapreduce.output.basename", "part")
++ "-" + 
NUMBER_FORMAT.format(context.getTaskAttemptID().getTaskID().getId());
   }
 
   /**
@@ -65,7 +66,7 @@ class DefaultOutputFormatContainer extends 
OutputFormatContainer {
   @Override
   public RecordWriter, HCatRecord>
   getRecordWriter(TaskAttemptContext context) throws IOException, 
InterruptedException {
-String name = 
getOutputName(context.getTaskAttemptID().getTaskID().getId());
+String name = getOutputName(context);
 return new DefaultRecordWriterContainer(context,
   getBaseOutputFormat().getRecordWriter(null, new 
JobConf(context.getConfiguration()), name, 
InternalUtil.createReporter(context)));
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/c4dc7b5f/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
index 60f1b60..320ace4 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
@@ -205,7 +205,8 @@ class DynamicPartitionFileRecordWriterContainer extends 
FileRecordWriterContaine
 
   Path parentDir = new 
Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
   Path childPath =
-  new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, 
"part", ""));
+  new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext,
+  
currTaskContext.getConfiguration().get("mapreduce.output.basename", "part"), 
""));
 
   RecordWriter baseRecordWriter =
   
baseOF.getRecordWriter(parentDir.getFileSystem(currTaskContext.getConfiguration()),

http://git-wip-us.apache.org/repos/asf/hive/blob/c4dc7b5f/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
index 001b59b..95ee3b4 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
@@ -97,7 +97,8 @@ class FileOutputFormatContainer extends OutputFormatContainer 
{
   (org.apache.hadoop.mapred.RecordWriter)null, context);
 } else {

hive git commit: HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun Radhakrishnan, reviewed by Daniel Dai)

2015-08-05 Thread daijy 
Repository: hive
Updated Branches:
  refs/heads/master 96f591446 -> 0f934d29b


HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun 
Radhakrishnan, reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0f934d29
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0f934d29
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0f934d29

Branch: refs/heads/master
Commit: 0f934d29bb6bfc992eca050337b8f48fb47aaed8
Parents: 96f5914
Author: Daniel Dai 
Authored: Wed Aug 5 13:50:19 2015 -0700
Committer: Daniel Dai 
Committed: Wed Aug 5 13:50:19 2015 -0700

--
 .../hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java | 7 ---
 .../mapreduce/DynamicPartitionFileRecordWriterContainer.java  | 3 ++-
 .../hive/hcatalog/mapreduce/FileOutputFormatContainer.java| 3 ++-
 3 files changed, 8 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
index 3a07b0c..f620b83 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
@@ -51,8 +51,9 @@ class DefaultOutputFormatContainer extends 
OutputFormatContainer {
 super(of);
   }
 
-  static synchronized String getOutputName(int partition) {
-return "part-" + NUMBER_FORMAT.format(partition);
+  static synchronized String getOutputName(TaskAttemptContext context) {
+return context.getConfiguration().get("mapreduce.output.basename", "part")
++ "-" + 
NUMBER_FORMAT.format(context.getTaskAttemptID().getTaskID().getId());
   }
 
   /**
@@ -65,7 +66,7 @@ class DefaultOutputFormatContainer extends 
OutputFormatContainer {
   @Override
   public RecordWriter, HCatRecord>
   getRecordWriter(TaskAttemptContext context) throws IOException, 
InterruptedException {
-String name = 
getOutputName(context.getTaskAttemptID().getTaskID().getId());
+String name = getOutputName(context);
 return new DefaultRecordWriterContainer(context,
   getBaseOutputFormat().getRecordWriter(null, new 
JobConf(context.getConfiguration()), name, 
InternalUtil.createReporter(context)));
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
index 60f1b60..320ace4 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
@@ -205,7 +205,8 @@ class DynamicPartitionFileRecordWriterContainer extends 
FileRecordWriterContaine
 
   Path parentDir = new 
Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
   Path childPath =
-  new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, 
"part", ""));
+  new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext,
+  
currTaskContext.getConfiguration().get("mapreduce.output.basename", "part"), 
""));
 
   RecordWriter baseRecordWriter =
   
baseOF.getRecordWriter(parentDir.getFileSystem(currTaskContext.getConfiguration()),

http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
index 001b59b..95ee3b4 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
@@ -97,7 +97,8 @@ class FileOutputFormatContainer extends OutputFormatContainer 
{
   (org.apache.hadoop.mapred.RecordWriter)null, context);
 } else {
   Pat

hive git commit: HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun Radhakrishnan, reviewed by Daniel Dai)

2015-08-05 Thread daijy 
Repository: hive
Updated Branches:
  refs/heads/branch-1 fb736ab78 -> 7330d9113


HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun 
Radhakrishnan, reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7330d911
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7330d911
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7330d911

Branch: refs/heads/branch-1
Commit: 7330d9113f14b98a5aedf9eb36e6063fae1631e7
Parents: fb736ab
Author: Daniel Dai 
Authored: Wed Aug 5 13:50:19 2015 -0700
Committer: Daniel Dai 
Committed: Wed Aug 5 13:51:06 2015 -0700

--
 .../hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java | 7 ---
 .../mapreduce/DynamicPartitionFileRecordWriterContainer.java  | 3 ++-
 .../hive/hcatalog/mapreduce/FileOutputFormatContainer.java| 3 ++-
 3 files changed, 8 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/7330d911/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
index 3a07b0c..f620b83 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
@@ -51,8 +51,9 @@ class DefaultOutputFormatContainer extends 
OutputFormatContainer {
 super(of);
   }
 
-  static synchronized String getOutputName(int partition) {
-return "part-" + NUMBER_FORMAT.format(partition);
+  static synchronized String getOutputName(TaskAttemptContext context) {
+return context.getConfiguration().get("mapreduce.output.basename", "part")
++ "-" + 
NUMBER_FORMAT.format(context.getTaskAttemptID().getTaskID().getId());
   }
 
   /**
@@ -65,7 +66,7 @@ class DefaultOutputFormatContainer extends 
OutputFormatContainer {
   @Override
   public RecordWriter, HCatRecord>
   getRecordWriter(TaskAttemptContext context) throws IOException, 
InterruptedException {
-String name = 
getOutputName(context.getTaskAttemptID().getTaskID().getId());
+String name = getOutputName(context);
 return new DefaultRecordWriterContainer(context,
   getBaseOutputFormat().getRecordWriter(null, new 
JobConf(context.getConfiguration()), name, 
InternalUtil.createReporter(context)));
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/7330d911/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
index 60f1b60..320ace4 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
@@ -205,7 +205,8 @@ class DynamicPartitionFileRecordWriterContainer extends 
FileRecordWriterContaine
 
   Path parentDir = new 
Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
   Path childPath =
-  new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, 
"part", ""));
+  new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext,
+  
currTaskContext.getConfiguration().get("mapreduce.output.basename", "part"), 
""));
 
   RecordWriter baseRecordWriter =
   
baseOF.getRecordWriter(parentDir.getFileSystem(currTaskContext.getConfiguration()),

http://git-wip-us.apache.org/repos/asf/hive/blob/7330d911/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
--
diff --git 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
index 001b59b..95ee3b4 100644
--- 
a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
+++ 
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
@@ -97,7 +97,8 @@ class FileOutputFormatContainer extends OutputFormatContainer 
{
   (org.apache.hadoop.mapred.RecordWriter)null, context);
 } else {