date:20230811

[hive] branch branch-3 updated: HIVE-27547: Backport HIVE-22219: Bringing a node manager down blocks restart of LLAP service (Jesus Camacho Rodriguez, reviewed by Slim Bouguerra)

2023-08-11 Thread sankarh

This is an automated email from the ASF dual-hosted git repository.

sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new ebcdbdde3d5 HIVE-27547: Backport HIVE-22219: Bringing a node manager 
down blocks restart of LLAP service (Jesus Camacho Rodriguez, reviewed by Slim 
Bouguerra)
ebcdbdde3d5 is described below

commit ebcdbdde3d5af126d5a4d9d5c08003ee9c19
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Fri Aug 11 12:43:17 2023 +0530

HIVE-27547: Backport HIVE-22219: Bringing a node manager down blocks 
restart of LLAP service (Jesus Camacho Rodriguez, reviewed by Slim Bouguerra)

Signed-off-by: Sankar Hariappan 
Closes (#4530)
---
 .../java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
 
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
index e0ada45ee40..c1bae653479 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
@@ -381,7 +381,7 @@ public class LlapStatusServiceDriver {
   cont.getId());
   appStatusBuilder.addNewRunningLlapInstance(llapInstance);
 }
-if (state == ServiceState.STABLE) {
+if (state == ServiceState.STARTED || state == ServiceState.STABLE || 
state == ServiceState.FLEX) {
   exitCode = ExitCode.SUCCESS;
 }
   } else {

[hive] branch branch-3 updated: HIVE-27545: Backport HIVE-22273: Access check is failed when a temporary directory is removed (Peter Vary reviewed by Marta Kuczora)

2023-08-11 Thread sankarh

This is an automated email from the ASF dual-hosted git repository.

sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new e0ffcbfc152 HIVE-27545: Backport HIVE-22273: Access check is failed 
when a temporary directory is removed (Peter Vary reviewed by Marta Kuczora)
e0ffcbfc152 is described below

commit e0ffcbfc152e816fa1309e3fcaf015e3c66168d5
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Fri Aug 11 12:40:39 2023 +0530

HIVE-27545: Backport HIVE-22273: Access check is failed when a temporary 
directory is removed (Peter Vary reviewed by Marta Kuczora)

Signed-off-by: Sankar Hariappan 
Closes (#4528)
---
 common/src/java/org/apache/hadoop/hive/common/FileUtils.java | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java 
b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index ec2f9f0ac89..f1f65b9b767 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -558,7 +558,13 @@ public final class FileUtils {
   return true;
 }
 // check all children
-FileStatus[] childStatuses = fs.listStatus(fileStatus.getPath());
+FileStatus[] childStatuses = null;
+try {
+  childStatuses = fs.listStatus(fileStatus.getPath());
+} catch (FileNotFoundException fe) {
+  LOG.debug("Skipping child access check since the directory is already 
removed");
+  return true;
+}
 for (FileStatus childStatus : childStatuses) {
   // check children recursively - recurse is true if we're here.
   if (!checkIsOwnerOfFileHierarchy(fs, childStatus, userName, true)) {

[hive] branch branch-3 updated: HIVE-27542: Backport HIVE-25659: Metastore direct sql queries with IN/(NOT IN) should be split based on max parameters allowed by SQL DB (#4525)

2023-08-11 Thread sankarh

This is an automated email from the ASF dual-hosted git repository.

sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new c90f2f7a805 HIVE-27542: Backport HIVE-25659: Metastore direct sql 
queries with IN/(NOT IN) should be split based on max parameters allowed by SQL 
DB (#4525)
c90f2f7a805 is described below

commit c90f2f7a805cf056978a5f3cc88525d10910dce0
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Fri Aug 11 12:37:35 2023 +0530

HIVE-27542: Backport HIVE-25659: Metastore direct sql queries with IN/(NOT 
IN) should be split based on max parameters allowed by SQL DB (#4525)

Signed-off-by: Sankar Hariappan 
Closes (#4525)
---
 .../hadoop/hive/metastore/conf/MetastoreConf.java  |  3 +++
 .../apache/hadoop/hive/metastore/txn/TxnUtils.java |  5 ++--
 .../hadoop/hive/metastore/txn/TestTxnUtils.java| 27 +++---
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 322edf10d93..aafd50ae466 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -457,6 +457,9 @@ public class MetastoreConf {
 
DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE("metastore.direct.sql.max.elements.values.clause",
 "hive.direct.sql.max.elements.values.clause",
 1000, "The maximum number of values in a VALUES clause for INSERT 
statement."),
+DIRECT_SQL_MAX_PARAMETERS("metastore.direct.sql.max.parameters",
+"hive.direct.sql.max.parameters", 1000, "The maximum query parameters 
\n" +
+"backend sql engine can support."),
 DIRECT_SQL_MAX_QUERY_LENGTH("metastore.direct.sql.max.query.length",
 "hive.direct.sql.max.query.length", 100, "The maximum\n" +
 " size of a query string (in KB)."),
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
index fa291d5f20a..61701625150 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
@@ -329,6 +329,7 @@ public class TxnUtils {
 // Get configuration parameters
 int maxQueryLength = MetastoreConf.getIntVar(conf, 
ConfVars.DIRECT_SQL_MAX_QUERY_LENGTH);
 int batchSize = MetastoreConf.getIntVar(conf, 
ConfVars.DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE);
+int maxParameters = MetastoreConf.getIntVar(conf, 
ConfVars.DIRECT_SQL_MAX_PARAMETERS);
 
 // Check parameter set validity as a public method.
 if (inList == null || inList.size() == 0 || maxQueryLength <= 0 || 
batchSize <= 0) {
@@ -380,7 +381,7 @@ public class TxnUtils {
   // Compute the size of a query when the 'nextValue' is added to the 
current query.
   int querySize = querySizeExpected(buf.length(), nextValue.length(), 
suffix.length(), addParens);
 
-  if (querySize > maxQueryLength * 1024) {
+  if ((querySize > maxQueryLength * 1024) || (currentCount >= 
maxParameters)) {
 // Check an edge case where the DIRECT_SQL_MAX_QUERY_LENGTH does not 
allow one 'IN' clause with single value.
 if (cursor4queryOfInClauses == 1 && cursor4InClauseElements == 0) {
   throw new IllegalArgumentException("The current " + 
ConfVars.DIRECT_SQL_MAX_QUERY_LENGTH.getVarname() + " is set too small to have 
one IN clause with single value!");
@@ -396,7 +397,7 @@ public class TxnUtils {
   buf.delete(buf.length()-newInclausePrefix.length(), buf.length());
 }
 
-buf.setCharAt(buf.length() - 1, ')'); // replace the "commar" to 
finish a 'IN' clause string.
+buf.setCharAt(buf.length() - 1, ')'); // replace the "comma" to finish 
a 'IN' clause string.
 
 if (addParens) {
   buf.append(")");
diff --git 
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
 
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
index 60be0f9c227..cd237b9caf2 100644
--- 
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
+++ 
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
@@ -61,6 +61,7 @@ public class TestTxnUtils {
 //  The first query happens to have 2 full batches.
 MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_QUERY_LENGTH, 1);
 MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 
10);
+

[hive] branch branch-3 updated: HIVE-27538: Backport HIVE-24201: WorkloadManager can support delayed move if destination pool does not have enough sessions (#4521)

2023-08-11 Thread sankarh

This is an automated email from the ASF dual-hosted git repository.

sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/branch-3 by this push:
 new 194420866bb HIVE-27538: Backport HIVE-24201: WorkloadManager can 
support delayed move if destination pool does not have enough sessions (#4521)
194420866bb is described below

commit 194420866bb631a84c116ac218201b55a4269100
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Fri Aug 11 12:33:55 2023 +0530

HIVE-27538: Backport HIVE-24201: WorkloadManager can support delayed move 
if destination pool does not have enough sessions (#4521)

Signed-off-by: Sankar Hariappan 
Closes (#4521)
---
 .../java/org/apache/hadoop/hive/conf/HiveConf.java |  12 ++
 .../ql/exec/tez/KillMoveTriggerActionHandler.java  |   6 +-
 .../hadoop/hive/ql/exec/tez/WmTezSession.java  |  12 ++
 .../hadoop/hive/ql/exec/tez/WorkloadManager.java   | 165 ++---
 .../hive/ql/exec/tez/TestWorkloadManager.java  | 159 
 5 files changed, 333 insertions(+), 21 deletions(-)

diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 96f44fae490..f9a47324473 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3138,6 +3138,18 @@ public class HiveConf extends Configuration {
 new TimeValidator(TimeUnit.SECONDS),
 "The timeout for AM registry registration, after which (on attempting 
to use the\n" +
 "session), we kill it and try to get another one."),
+HIVE_SERVER2_WM_DELAYED_MOVE("hive.server2.wm.delayed.move", false,
+"Determines behavior of the wm move trigger when destination pool is 
full.\n" +
+"If true, the query will run in source pool as long as possible if 
destination pool is full;\n" +
+"if false, the query will be killed if destination pool is full."),
+
HIVE_SERVER2_WM_DELAYED_MOVE_TIMEOUT("hive.server2.wm.delayed.move.timeout", 
"3600",
+new TimeValidator(TimeUnit.SECONDS),
+"The amount of time a delayed move is allowed to run in the source 
pool,\n" +
+"when a delayed move session times out, the session is moved to the 
destination pool.\n" +
+"A value of 0 indicates no timeout"),
+
HIVE_SERVER2_WM_DELAYED_MOVE_VALIDATOR_INTERVAL("hive.server2.wm.delayed.move.validator.interval",
 "60",
+new TimeValidator(TimeUnit.SECONDS),
+"Interval for checking for expired delayed moves."),
 HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "",
 "A list of comma separated values corresponding to YARN queues of the 
same name.\n" +
 "When HiveServer2 is launched in Tez mode, this configuration needs to 
be set\n" +
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillMoveTriggerActionHandler.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillMoveTriggerActionHandler.java
index b16f1c30a07..5eb1b69ede5 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillMoveTriggerActionHandler.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillMoveTriggerActionHandler.java
@@ -47,8 +47,10 @@ public class KillMoveTriggerActionHandler implements 
TriggerActionHandler moveFuture = wm.applyMoveSessionAsync(wmTezSession, 
destPoolName);
-  moveFutures.put(wmTezSession, moveFuture);
+  if (!wmTezSession.isDelayedMove()) {
+Future moveFuture = 
wm.applyMoveSessionAsync(wmTezSession, destPoolName);
+moveFutures.put(wmTezSession, moveFuture);
+  }
   break;
 default:
   throw new RuntimeException("Unsupported action: " + 
entry.getValue());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WmTezSession.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WmTezSession.java
index fa2b02e5913..6004d712c4c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WmTezSession.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WmTezSession.java
@@ -55,6 +55,7 @@ public class WmTezSession extends TezSessionPoolSession 
implements AmPluginNode
   @JsonProperty("queryId")
   private String queryId;
   private SettableFuture returnFuture = null;
+  private boolean isDelayedMove;
 
   private final WorkloadManager wmParent;
 
@@ -72,6 +73,7 @@ public class WmTezSession extends TezSessionPoolSession 
implements AmPluginNode
   SessionExpirationTracker expiration, HiveConf conf) {
 super(sessionId, parent, expiration, conf);
 wmParent = parent;
+isDelayedMove = false;
   }
 
   @VisibleForTesting
@@ -79,6 +81,7 @@ public class WmTezSession extends TezSessionPoolSession 
implements AmPluginNode
   SessionExpirationTracker expiration, HiveConf conf) {
 super(sessionId, testParent,

[hive] branch branch-3 updated: HIVE-27547: Backport HIVE-22219: Bringing a node manager down blocks restart of LLAP service (Jesus Camacho Rodriguez, reviewed by Slim Bouguerra)

[hive] branch branch-3 updated: HIVE-27545: Backport HIVE-22273: Access check is failed when a temporary directory is removed (Peter Vary reviewed by Marta Kuczora)

[hive] branch branch-3 updated: HIVE-27542: Backport HIVE-25659: Metastore direct sql queries with IN/(NOT IN) should be split based on max parameters allowed by SQL DB (#4525)

[hive] branch branch-3 updated: HIVE-27538: Backport HIVE-24201: WorkloadManager can support delayed move if destination pool does not have enough sessions (#4521)

4 matches

Site Navigation

Mail list logo

Footer information