[05/50] [abbrv] hive git commit: HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions (Wei Zheng, reviewed by Eugene Koifman)

2016-04-04 Thread jdere 
HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions (Wei 
Zheng, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f9d1b6ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f9d1b6ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f9d1b6ab

Branch: refs/heads/llap
Commit: f9d1b6ab77ab15b8337c17fbe38557c1f7b5ce58
Parents: d3a5f20
Author: Wei Zheng 
Authored: Thu Mar 24 17:29:59 2016 -0700
Committer: Wei Zheng 
Committed: Thu Mar 24 17:29:59 2016 -0700

--
 .../hive/hcatalog/streaming/HiveEndPoint.java   | 11 +
 .../hadoop/hive/ql/txn/compactor/Cleaner.java   |  5 +++
 .../hive/ql/txn/compactor/CompactorThread.java  |  5 +++
 .../hadoop/hive/ql/txn/compactor/Initiator.java |  9 +++-
 .../hadoop/hive/ql/txn/compactor/Worker.java|  8 +++-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 47 
 6 files changed, 82 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 4c77842..baeafad 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -18,6 +18,7 @@
 
 package org.apache.hive.hcatalog.streaming;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.cli.CliSessionState;
@@ -342,6 +343,11 @@ public class HiveEndPoint {
 return null;
   }
 } );
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+}
   } catch (IOException e) {
 LOG.error("Error closing connection to " + endPt, e);
   } catch (InterruptedException e) {
@@ -937,6 +943,11 @@ public class HiveEndPoint {
   }
 }
 );
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+}
   } catch (IOException e) {
 throw new ImpersonationFailed("Failed closing Txn Batch as user '" + 
username +
 "' on  endPoint :" + endPt, e);

http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index 9ffeaec..4c31a49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -272,6 +272,11 @@ public class Cleaner extends CompactorThread {
 return null;
   }
 });
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception + " for " +
+  ci.getFullPartitionName());}
   }
   txnHandler.markCleaned(ci);
 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
index 8495c66..4d6e24e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
@@ -174,6 +174,11 @@ abstract class CompactorThread extends Thread implements 
MetaStoreThread {
   return null;
 }
   });
+  try {
+FileSystem.closeAllForUGI(ugi);
+  } catch (IOException exception) {
+LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+  }
 
   if (wrapper.size() == 1) {
 LOG.debug("Running job as " + wrapper.get(0));

[1/2] hive git commit: HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions (Wei Zheng, reviewed by Eugene Koifman)

2016-03-30 Thread weiz 
Repository: hive
Updated Branches:
  refs/heads/branch-2.0 f4468ce68 -> eda730320


HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions (Wei 
Zheng, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1785ca00
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1785ca00
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1785ca00

Branch: refs/heads/branch-2.0
Commit: 1785ca000596177c28511dc151deb967c3ce1710
Parents: f4468ce
Author: Wei Zheng 
Authored: Thu Mar 24 17:29:59 2016 -0700
Committer: Wei Zheng 
Committed: Wed Mar 30 15:10:42 2016 -0700

--
 .../hive/hcatalog/streaming/HiveEndPoint.java   | 11 +
 .../hadoop/hive/ql/txn/compactor/Cleaner.java   |  5 +++
 .../hive/ql/txn/compactor/CompactorThread.java  |  5 +++
 .../hadoop/hive/ql/txn/compactor/Initiator.java |  9 +++-
 .../hadoop/hive/ql/txn/compactor/Worker.java|  6 +++
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 47 
 6 files changed, 82 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1785ca00/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 4c77842..baeafad 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -18,6 +18,7 @@
 
 package org.apache.hive.hcatalog.streaming;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.cli.CliSessionState;
@@ -342,6 +343,11 @@ public class HiveEndPoint {
 return null;
   }
 } );
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+}
   } catch (IOException e) {
 LOG.error("Error closing connection to " + endPt, e);
   } catch (InterruptedException e) {
@@ -937,6 +943,11 @@ public class HiveEndPoint {
   }
 }
 );
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+}
   } catch (IOException e) {
 throw new ImpersonationFailed("Failed closing Txn Batch as user '" + 
username +
 "' on  endPoint :" + endPt, e);

http://git-wip-us.apache.org/repos/asf/hive/blob/1785ca00/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index fbf5481..974184f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -224,6 +224,11 @@ public class Cleaner extends CompactorThread {
 return null;
   }
 });
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception + " for " +
+  ci.getFullPartitionName());}
   }
   txnHandler.markCleaned(ci);
 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/1785ca00/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
index 3f6b099..859caff 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
@@ -173,6 +173,11 @@ abstract class CompactorThread extends Thread implements 
MetaStoreThread {
   return null;
 }
   });
+  try {
+FileSystem.closeAllForUGI(ugi);
+  } catch (IOException exception) {
+LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+  }
 
   if (wrapper.size() == 1) {
 LOG.debug("Running job as " + wrapper.get(0));

hive git commit: HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions (Wei Zheng, reviewed by Eugene Koifman)

2016-03-24 Thread weiz 
Repository: hive
Updated Branches:
  refs/heads/branch-1 5bae0ad45 -> 82068205a


HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions (Wei 
Zheng, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/82068205
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/82068205
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/82068205

Branch: refs/heads/branch-1
Commit: 82068205a59ed4b6aeb2b353eb612ce0da73c5c2
Parents: 5bae0ad
Author: Wei Zheng 
Authored: Thu Mar 24 22:38:39 2016 -0700
Committer: Wei Zheng 
Committed: Thu Mar 24 22:38:39 2016 -0700

--
 .../hive/hcatalog/streaming/HiveEndPoint.java   | 11 +
 .../hadoop/hive/ql/txn/compactor/Cleaner.java   |  6 +++
 .../hive/ql/txn/compactor/CompactorThread.java  |  5 +++
 .../hadoop/hive/ql/txn/compactor/Initiator.java |  9 +++-
 .../hadoop/hive/ql/txn/compactor/Worker.java|  7 +++
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 46 
 6 files changed, 83 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/82068205/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index b0bbd66..2e81bf8 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -20,6 +20,7 @@ package org.apache.hive.hcatalog.streaming;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
@@ -342,6 +343,11 @@ public class HiveEndPoint {
 return null;
   }
 } );
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+}
   } catch (IOException e) {
 LOG.error("Error closing connection to " + endPt, e);
   } catch (InterruptedException e) {
@@ -937,6 +943,11 @@ public class HiveEndPoint {
   }
 }
 );
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+}
   } catch (IOException e) {
 throw new ImpersonationFailed("Failed closing Txn Batch as user '" + 
username +
 "' on  endPoint :" + endPt, e);

http://git-wip-us.apache.org/repos/asf/hive/blob/82068205/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index 1e6e8a1..d861bc2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -272,6 +272,12 @@ public class Cleaner extends CompactorThread {
 return null;
   }
 });
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi + 
" for " +
+  ci.getFullPartitionName(), exception);
+}
   }
   txnHandler.markCleaned(ci);
 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/82068205/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
index ae8865c..952b27a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
@@ -174,6 +174,11 @@ abstract class CompactorThread extends Thread implements 
MetaStoreThread {
   return null;
 }
   });
+  try {
+FileSystem.closeAllForUGI(ugi);
+  } catch (IOException exception) {
+LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+  }

hive git commit: HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions (Wei Zheng, reviewed by Eugene Koifman)

2016-03-24 Thread weiz 
Repository: hive
Updated Branches:
  refs/heads/master d3a5f20b4 -> f9d1b6ab7


HIVE-13151 : Clean up UGI objects in FileSystem cache for transactions (Wei 
Zheng, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f9d1b6ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f9d1b6ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f9d1b6ab

Branch: refs/heads/master
Commit: f9d1b6ab77ab15b8337c17fbe38557c1f7b5ce58
Parents: d3a5f20
Author: Wei Zheng 
Authored: Thu Mar 24 17:29:59 2016 -0700
Committer: Wei Zheng 
Committed: Thu Mar 24 17:29:59 2016 -0700

--
 .../hive/hcatalog/streaming/HiveEndPoint.java   | 11 +
 .../hadoop/hive/ql/txn/compactor/Cleaner.java   |  5 +++
 .../hive/ql/txn/compactor/CompactorThread.java  |  5 +++
 .../hadoop/hive/ql/txn/compactor/Initiator.java |  9 +++-
 .../hadoop/hive/ql/txn/compactor/Worker.java|  8 +++-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 47 
 6 files changed, 82 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
--
diff --git 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 4c77842..baeafad 100644
--- 
a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ 
b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -18,6 +18,7 @@
 
 package org.apache.hive.hcatalog.streaming;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.cli.CliSessionState;
@@ -342,6 +343,11 @@ public class HiveEndPoint {
 return null;
   }
 } );
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+}
   } catch (IOException e) {
 LOG.error("Error closing connection to " + endPt, e);
   } catch (InterruptedException e) {
@@ -937,6 +943,11 @@ public class HiveEndPoint {
   }
 }
 );
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+}
   } catch (IOException e) {
 throw new ImpersonationFailed("Failed closing Txn Batch as user '" + 
username +
 "' on  endPoint :" + endPt, e);

http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index 9ffeaec..4c31a49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -272,6 +272,11 @@ public class Cleaner extends CompactorThread {
 return null;
   }
 });
+try {
+  FileSystem.closeAllForUGI(ugi);
+} catch (IOException exception) {
+  LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception + " for " +
+  ci.getFullPartitionName());}
   }
   txnHandler.markCleaned(ci);
 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/f9d1b6ab/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java 
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
index 8495c66..4d6e24e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
@@ -174,6 +174,11 @@ abstract class CompactorThread extends Thread implements 
MetaStoreThread {
   return null;
 }
   });
+  try {
+FileSystem.closeAllForUGI(ugi);
+  } catch (IOException exception) {
+LOG.error("Could not clean up file-system handles for UGI: " + ugi, 
exception);
+  }
 
   if (wrapper.size() == 1) {
 LOG.debug("Running job as " + wrapper.get(0));