[hive] branch branch-3 updated: HIVE-27611: Backport of HIVE-22168: Remove very expensive logging from the llap cache hotpath (Slim B via Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 5a9aa562581 HIVE-27611: Backport of HIVE-22168: Remove very expensive
logging from the llap cache hotpath (Slim B via Jesus Camacho Rodriguez)
5a9aa562581 is described below
commit 5a9aa5625810296df84829be1261ec5d502ccc9e
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Mon Aug 28 20:46:20 2023 +0530
HIVE-27611: Backport of HIVE-22168: Remove very expensive logging from the
llap cache hotpath (Slim B via Jesus Camacho Rodriguez)
Signed-off-by: Sankar Hariappan
Closes (#4590)
---
.../java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java | 4 ++--
.../hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java | 12 ++--
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java
b/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java
index f68ebd7c6d6..ea354683d8c 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java
@@ -213,8 +213,8 @@ public class LlapCacheAwareFs extends FileSystem {
return new CacheChunk(buffer, startOffset, endOffset);
}
}, gotAllData);
- if (LOG.isInfoEnabled()) {
-LOG.info("Buffers after cache " +
RecordReaderUtils.stringifyDiskRanges(drl));
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Buffers after cache " +
RecordReaderUtils.stringifyDiskRanges(drl));
}
if (gotAllData.value) {
long sizeRead = 0;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index 348f9df773f..91173818f55 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -564,15 +564,15 @@ class EncodedReaderImpl implements EncodedReader {
long stripeOffset, boolean hasFileId, IdentityHashMap toRelease)
throws IOException {
DiskRangeList.MutateHelper toRead = new
DiskRangeList.MutateHelper(listToRead);
-if (LOG.isInfoEnabled()) {
- LOG.info("Resulting disk ranges to read (file " + fileKey + "): "
+if (LOG.isDebugEnabled()) {
+ LOG.debug("Resulting disk ranges to read (file " + fileKey + "): "
+ RecordReaderUtils.stringifyDiskRanges(toRead.next));
}
BooleanRef isAllInCache = new BooleanRef();
if (hasFileId) {
cacheWrapper.getFileData(fileKey, toRead.next, stripeOffset, CC_FACTORY,
isAllInCache);
- if (LOG.isInfoEnabled()) {
-LOG.info("Disk ranges after cache (found everything " +
isAllInCache.value + "; file "
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Disk ranges after cache (found everything " +
isAllInCache.value + "; file "
+ fileKey + ", base offset " + stripeOffset + "): "
+ RecordReaderUtils.stringifyDiskRanges(toRead.next));
}
@@ -2009,8 +2009,8 @@ class EncodedReaderImpl implements EncodedReader {
releaseBuffers(toRelease.keySet(), true);
toRelease.clear();
}
- if (LOG.isInfoEnabled()) {
-LOG.info("Disk ranges after pre-read (file " + fileKey + ", base
offset "
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Disk ranges after pre-read (file " + fileKey + ", base
offset "
+ stripeOffset + "): " +
RecordReaderUtils.stringifyDiskRanges(toRead.next));
}
iter = toRead.next; // Reset the iter to start.
[hive] branch branch-3 updated: HIVE-27612: Backport of HIVE-22169: Tez: SplitGenerator tries to look for plan files which won't exist for Tez (Gopal V via Vineet Garg)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 220ca4d901c HIVE-27612: Backport of HIVE-22169: Tez: SplitGenerator
tries to look for plan files which won't exist for Tez (Gopal V via Vineet Garg)
220ca4d901c is described below
commit 220ca4d901c76be62d71f478b20c1eadcbaf6052
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Mon Aug 28 14:15:16 2023 +0530
HIVE-27612: Backport of HIVE-22169: Tez: SplitGenerator tries to look for
plan files which won't exist for Tez (Gopal V via Vineet Garg)
Signed-off-by: Sankar Hariappan
Closes (#4591)
---
.../java/org/apache/hadoop/hive/ql/exec/Utilities.java | 16 +---
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index ee9150fe725..abac436e56a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -298,15 +298,17 @@ public final class Utilities {
return;
}
+
try {
- FileSystem fs = mapPath.getFileSystem(conf);
- if (fs.exists(mapPath)) {
-fs.delete(mapPath, true);
- }
- if (fs.exists(reducePath)) {
-fs.delete(reducePath, true);
+ if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) {
+FileSystem fs = mapPath.getFileSystem(conf);
+if (fs.exists(mapPath)) {
+ fs.delete(mapPath, true);
+}
+if (fs.exists(reducePath)) {
+ fs.delete(reducePath, true);
+}
}
-
} catch (Exception e) {
LOG.warn("Failed to clean-up tmp directories.", e);
} finally {
[hive] branch branch-3 updated: HIVE-27610: Backport of HIVE-22161: UDF: FunctionRegistry synchronizes on org.apache.hadoop.hive.ql.udf.UDFType class (Gopal V, reviewed by Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 8fa292d4892 HIVE-27610: Backport of HIVE-22161: UDF: FunctionRegistry
synchronizes on org.apache.hadoop.hive.ql.udf.UDFType class (Gopal V, reviewed
by Ashutosh Chauhan)
8fa292d4892 is described below
commit 8fa292d4892a3daf3403c1072d8a8ef5b73892eb
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Mon Aug 28 13:59:46 2023 +0530
HIVE-27610: Backport of HIVE-22161: UDF: FunctionRegistry synchronizes on
org.apache.hadoop.hive.ql.udf.UDFType class (Gopal V, reviewed by Ashutosh
Chauhan)
Signed-off-by: Sankar Hariappan
Closes (#4589)
---
.../java/org/apache/hive/common/util/AnnotationUtils.java | 14 ++
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/common/src/java/org/apache/hive/common/util/AnnotationUtils.java
b/common/src/java/org/apache/hive/common/util/AnnotationUtils.java
index a73faca4a25..bfbaea69fc7 100644
--- a/common/src/java/org/apache/hive/common/util/AnnotationUtils.java
+++ b/common/src/java/org/apache/hive/common/util/AnnotationUtils.java
@@ -23,17 +23,15 @@ import java.lang.reflect.Method;
public class AnnotationUtils {
- // to avoid https://bugs.openjdk.java.net/browse/JDK-7122142
+ // until JDK8, this had a lock around annotationClass to avoid
+ // https://bugs.openjdk.java.net/browse/JDK-7122142
public static T getAnnotation(Class clazz,
Class annotationClass) {
-synchronized (annotationClass) {
- return clazz.getAnnotation(annotationClass);
-}
+return clazz.getAnnotation(annotationClass);
}
- // to avoid https://bugs.openjdk.java.net/browse/JDK-7122142
+ // until JDK8, this had a lock around annotationClass to avoid
+ // https://bugs.openjdk.java.net/browse/JDK-7122142
public static T getAnnotation(Method method, Class
annotationClass) {
-synchronized (annotationClass) {
- return method.getAnnotation(annotationClass);
-}
+return method.getAnnotation(annotationClass);
}
}
[hive] branch master updated: HIVE-23680 : TestDbNotificationListener is unstable (Kirti Ruge, reviewed by Zsolt Miskolczi, Laszlo Vegh)
This is an automated email from the ASF dual-hosted git repository.
veghlaci05 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a02fe662ba1 HIVE-23680 : TestDbNotificationListener is unstable (Kirti
Ruge, reviewed by Zsolt Miskolczi, Laszlo Vegh)
a02fe662ba1 is described below
commit a02fe662ba1548dc5a80041d0649e131ca1be789
Author: rkirtir <111496669+rkir...@users.noreply.github.com>
AuthorDate: Mon Aug 28 13:22:24 2023 +0530
HIVE-23680 : TestDbNotificationListener is unstable (Kirti Ruge, reviewed
by Zsolt Miskolczi, Laszlo Vegh)
---
itests/hcatalog-unit/pom.xml | 1 +
.../listener/TestDbNotificationCleanup.java| 189 ++
.../listener/TestDbNotificationListener.java | 215 +++--
.../TestTransactionalDbNotificationListener.java | 169
4 files changed, 384 insertions(+), 190 deletions(-)
diff --git a/itests/hcatalog-unit/pom.xml b/itests/hcatalog-unit/pom.xml
index 5183a2ba802..4a049697b71 100644
--- a/itests/hcatalog-unit/pom.xml
+++ b/itests/hcatalog-unit/pom.xml
@@ -89,6 +89,7 @@
org.apache.hive
hive-standalone-metastore-server
+ tests
test
diff --git
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationCleanup.java
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationCleanup.java
new file mode 100644
index 000..e73a57ea58f
--- /dev/null
+++
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationCleanup.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.hcatalog.listener;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import java.util.concurrent.TimeUnit;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient.NotificationFilter;
+import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageEncoder;
+import org.apache.hadoop.hive.ql.DriverFactory;
+import org.apache.hadoop.hive.ql.IDriver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static
org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.EVENT_DB_LISTENER_CLEAN_STARTUP_WAIT_INTERVAL;
+
+
+
+public class TestDbNotificationCleanup {
+private static final Logger LOG =
LoggerFactory.getLogger(TestDbNotificationCleanup.class
+.getName());
+private static final int EVENTS_TTL = 30;
+private static final int CLEANUP_SLEEP_TIME = 10;
+private static Map emptyParameters = new HashMap();
+private static IMetaStoreClient msClient;
+private static IDriver driver;
+private static MessageDeserializer md;
+private static HiveConf conf;
+
+private long firstEventId;
+private final String testTempDir =
Paths.get(System.getProperty("java.io.tmpdir"), "testDbNotif").toString();
+@SuppressWarnings("rawtypes")
+@BeforeClass
+public static void connectToMetastore() throws Exception {
+conf = new HiveConf();
+
+
MetastoreConf.setVar(conf,MetastoreConf.ConfVars.TRANSACTIONAL_EVENT_LISTENERS,
+"org.apache.hive.hcatalog.listener.DbNotificationListener");
+conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+
