[hive] branch master updated: disable flaky oracle test.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4903585a34a disable flaky oracle test.
4903585a34a is described below
commit 4903585a34ae44bb3fec4207b5acab63f6bfc8c1
Author: Ayush Saxena
AuthorDate: Fri Aug 25 04:00:30 2023 +0530
disable flaky oracle test.
---
Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 2c551a21568..9b7d9ba260d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -272,7 +272,7 @@ fi
}
def branches = [:]
- for (def d in ['derby','postgres',/*'mysql',*/'oracle']) {
+ for (def d in ['derby','postgres',/*'mysql','oracle'*/]) {
def dbType=d
def splitName = "init@$dbType"
branches[splitName] = {
[hive] branch master updated: HIVE-27595: replaced cartesian-product table filtering by sort + binary search (#4601)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new dc9d1ad0965 HIVE-27595: replaced cartesian-product table filtering by
sort + binary search (#4601)
dc9d1ad0965 is described below
commit dc9d1ad0965ece6a09abdb6481ca113dafce7c75
Author: Henrib
AuthorDate: Thu Aug 24 15:33:16 2023 +0200
HIVE-27595: replaced cartesian-product table filtering by sort + binary
search (#4601)
* HIVE-27595: replaced cartesian-product filtering by sort + binary search
lookup reducing complexity from m*n to (m+n)*logn; (Henri Biestro reviewed by
John Sherman, Saihemanth)
---
.../plugin/AuthorizationMetaStoreFilterHook.java | 23 +-
.../plugin/HivePrivilegeObjectUtils.java | 125 -
.../plugin/metastore/HiveMetaStoreAuthorizer.java | 49 +---
.../metastore/TestHiveMetaStoreAuthorizer.java | 2 +-
.../metastore/TestTablePermissionFilterAlgos.java | 282 +
5 files changed, 425 insertions(+), 56 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
index ad218af9a05..5723022f62b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
@@ -19,10 +19,6 @@ package
org.apache.hadoop.hive.ql.security.authorization.plugin;
import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private;
import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
@@ -31,6 +27,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.TableMeta;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
+import static
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObjectUtils.TablePrivilegeLookup;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.slf4j.Logger;
@@ -162,20 +159,18 @@ public class AuthorizationMetaStoreFilterHook extends
DefaultMetaStoreFilterHook
return objs;
}
- private ImmutablePair tableMetaKey(String dbName, String
tableName) {
-return new ImmutablePair(dbName, tableName);
- }
-
@Override
public List filterTableMetas(List tableMetas) throws
MetaException {
List listObjs = tableMetasToPrivilegeObjs(tableMetas);
List filteredList = getFilteredObjects(listObjs);
-Set> filteredNames = filteredList.stream()
-.map(e -> tableMetaKey(e.getDbname(), e.getObjectName()))
-.collect(Collectors.toSet());
-return tableMetas.stream()
-.filter(e -> filteredNames.contains(tableMetaKey(e.getDbName(),
e.getTableName(
-.collect(Collectors.toList());
+final List ret = new ArrayList<>();
+final TablePrivilegeLookup index = new TablePrivilegeLookup(filteredList);
+for(TableMeta table : tableMetas) {
+ if (index.lookup(table.getDbName(), table.getTableName()) != null) {
+ret.add(table);
+ }
+}
+return ret;
}
@Override
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java
index 4ab7870e7b3..479dce09ab7 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java
@@ -19,8 +19,6 @@ package
org.apache.hadoop.hive.ql.security.authorization.plugin;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -59,7 +57,130 @@ public class HivePrivilegeObjectUtils {
objs.add(new HivePrivilegeObject(HivePrivilegeObjectType.DATACONNECTOR,
null, dcname));
}
return objs;
+ }
+
+ /**
+ * A helper enabling efficient lookup of a table in a list by database name,
table name.
+ * When filtering a source list of tables by checking their presence in
another permission list
+ * using the database name, table name, we need to avoid performing a
cartesian product. This
+ * product stems from checking each table from
[hive] branch branch-3 updated: HIVE-27606: Backport of HIVE-21171: Skip creating scratch dirs for tez if RPC is on (Vineet Garg, reviewed by Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 2f5fc8db630 HIVE-27606: Backport of HIVE-21171: Skip creating scratch
dirs for tez if RPC is on (Vineet Garg, reviewed by Ashutosh Chauhan)
2f5fc8db630 is described below
commit 2f5fc8db630e9cc0384cf056d13dfd1e9e348f37
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Thu Aug 24 15:52:04 2023 +0530
HIVE-27606: Backport of HIVE-21171: Skip creating scratch dirs for tez if
RPC is on (Vineet Garg, reviewed by Ashutosh Chauhan)
Signed-off-by: Sankar Hariappan
Closes (#4585)
---
ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 7 +--
.../java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java | 13 -
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 26faf55fb8d..ee9150fe725 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -662,8 +662,11 @@ public final class Utilities {
// this is the unique conf ID, which is kept in JobConf as part of the
plan file name
String jobID = UUID.randomUUID().toString();
Path planPath = new Path(hiveScratchDir, jobID);
- FileSystem fs = planPath.getFileSystem(conf);
- fs.mkdirs(planPath);
+ if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) {
+FileSystem fs = planPath.getFileSystem(conf);
+// since we are doing RPC creating a directory is un-necessary
+fs.mkdirs(planPath);
+ }
HiveConf.setVar(conf, HiveConf.ConfVars.PLAN,
planPath.toUri().toString());
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 535994a9901..ff7f0b440be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -1509,11 +1509,14 @@ public class DagUtils {
scratchDir = new Path(scratchDir, userName);
Path tezDir = getTezDir(scratchDir);
-FileSystem fs = tezDir.getFileSystem(conf);
-LOG.debug("TezDir path set " + tezDir + " for user: " + userName);
-// since we are adding the user name to the scratch dir, we do not
-// need to give more permissions here
-fs.mkdirs(tezDir);
+if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) {
+ FileSystem fs = tezDir.getFileSystem(conf);
+ LOG.debug("TezDir path set " + tezDir + " for user: " + userName);
+ // since we are adding the user name to the scratch dir, we do not
+ // need to give more permissions here
+ // Since we are doing RPC creating a dir is not necessary
+ fs.mkdirs(tezDir);
+}
return tezDir;
[hive] branch branch-3 updated: HIVE-27607: Backport of HIVE-21182: Skip setting up hive scratch dir during planning (Vineet Garg, reviewed by Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new eb4bd1f714a HIVE-27607: Backport of HIVE-21182: Skip setting up hive
scratch dir during planning (Vineet Garg, reviewed by Ashutosh Chauhan)
eb4bd1f714a is described below
commit eb4bd1f714aac89cd5dfab566d4313c7b0cde897
Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com>
AuthorDate: Thu Aug 24 15:48:59 2023 +0530
HIVE-27607: Backport of HIVE-21182: Skip setting up hive scratch dir during
planning (Vineet Garg, reviewed by Ashutosh Chauhan)
Signed-off-by: Sankar Hariappan
Closes (#4586)
---
ql/src/java/org/apache/hadoop/hive/ql/Context.java | 21 -
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
2 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index b4d5806d4ed..4a47f4c5f4f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -531,14 +531,14 @@ public class Context {
/**
* Create a map-reduce scratch directory on demand and return it.
+ * @param mkDir flag to indicate if scratch dir is to be created or not
*
*/
- public Path getMRScratchDir() {
-
+ public Path getMRScratchDir(boolean mkDir) {
// if we are executing entirely on the client side - then
// just (re)use the local scratch directory
if(isLocalOnlyExecutionMode()) {
- return getLocalScratchDir(!isExplainSkipExecution());
+ return getLocalScratchDir(mkDir);
}
try {
@@ -546,16 +546,23 @@ public class Context {
URI uri = dir.toUri();
Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
- !isExplainSkipExecution(), uri.getPath());
+ mkDir, uri.getPath());
LOG.info("New scratch dir is " + newScratchDir);
return newScratchDir;
} catch (IOException e) {
throw new RuntimeException(e);
} catch (IllegalArgumentException e) {
throw new RuntimeException("Error while making MR scratch "
- + "directory - check filesystem config (" + e.getCause() + ")", e);
+ + "directory - check filesystem config ("
+ e.getCause() + ")", e);
}
}
+ /**
+ * Create a map-reduce scratch directory on demand and return it.
+ *
+ */
+ public Path getMRScratchDir() {
+return getMRScratchDir(!isExplainSkipExecution());
+ }
/**
* Create a temporary directory depending of the path specified.
@@ -674,6 +681,10 @@ public class Context {
return new Path(getStagingDir(new Path(uri), !isExplainSkipExecution()),
MR_PREFIX + nextPathId());
}
+ public Path getMRTmpPath(boolean mkDir) {
+return new Path(getMRScratchDir(mkDir), MR_PREFIX +
+nextPathId());
+ }
/**
* Get a path to store map-reduce intermediate data in.
*
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 0f1577353b9..b28fa1c0727 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -2585,7 +2585,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
stagingPath = ctx.getMRTmpPath(tablePath.toUri());
}
} else {
- stagingPath = ctx.getMRTmpPath();
+ stagingPath = ctx.getMRTmpPath(false);
}
return stagingPath;
[hive] branch master updated (12f7d5ea894 -> 1ebef40aba0)
This is an automated email from the ASF dual-hosted git repository. zabetak pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git from 12f7d5ea894 HIVE-27431: Clean invalid properties in test module (#4408). (zhangbutao, reviewed by Denys Kuzmenko, Ayush Saxena) add 1ebef40aba0 HIVE-27638: Preparing for 4.0.0-beta-2 development (Stamatis Zampetakis reviewed by Ayush Saxena, Attila Turoczy) No new revisions were added by this update. Summary of changes: accumulo-handler/pom.xml |2 +- beeline/pom.xml|2 +- classification/pom.xml |2 +- cli/pom.xml|2 +- common/pom.xml |2 +- contrib/pom.xml|2 +- druid-handler/pom.xml |2 +- hbase-handler/pom.xml |2 +- hcatalog/core/pom.xml |2 +- hcatalog/hcatalog-pig-adapter/pom.xml |4 +- hcatalog/pom.xml |4 +- hcatalog/server-extensions/pom.xml |2 +- hcatalog/webhcat/java-client/pom.xml |2 +- hcatalog/webhcat/svr/pom.xml |2 +- hplsql/pom.xml |2 +- iceberg/iceberg-catalog/pom.xml|2 +- iceberg/iceberg-handler/pom.xml|2 +- iceberg/iceberg-shading/pom.xml|2 +- iceberg/patched-iceberg-api/pom.xml|2 +- iceberg/patched-iceberg-core/pom.xml |2 +- iceberg/pom.xml|4 +- itests/custom-serde/pom.xml|2 +- itests/custom-udfs/pom.xml |2 +- itests/custom-udfs/udf-classloader-udf1/pom.xml|2 +- itests/custom-udfs/udf-classloader-udf2/pom.xml|2 +- itests/custom-udfs/udf-classloader-util/pom.xml|2 +- .../custom-udfs/udf-vectorized-badexample/pom.xml |2 +- itests/hcatalog-unit/pom.xml |2 +- itests/hive-blobstore/pom.xml |2 +- itests/hive-jmh/pom.xml|2 +- itests/hive-minikdc/pom.xml|2 +- itests/hive-unit-hadoop2/pom.xml |2 +- itests/hive-unit/pom.xml |2 +- itests/pom.xml |2 +- itests/qtest-accumulo/pom.xml |2 +- itests/qtest-druid/pom.xml |2 +- itests/qtest-iceberg/pom.xml |2 +- itests/qtest-kudu/pom.xml |2 +- itests/qtest/pom.xml |2 +- itests/test-serde/pom.xml |2 +- itests/util/pom.xml|2 +- jdbc-handler/pom.xml |2 +- jdbc/pom.xml |2 +- kafka-handler/pom.xml |2 +- kudu-handler/pom.xml |2 +- llap-client/pom.xml|2 +- llap-common/pom.xml|2 +- llap-ext-client/pom.xml|2 +- llap-server/pom.xml|2 +- llap-tez/pom.xml |2 +- metastore/pom.xml |2 +- .../upgrade/hive/hive-schema-4.0.0-beta-2.hive.sql | 2121 .../upgrade-4.0.0-beta-1-to-4.0.0-beta-2.hive.sql |3 + metastore/scripts/upgrade/hive/upgrade.order.hive |3 +- packaging/pom.xml |2 +- parser/pom.xml |2 +- pom.xml|6 +- ql/pom.xml |2 +- .../test/results/clientpositive/llap/sysdb.q.out |8 +- serde/pom.xml |2 +- service-rpc/pom.xml|2 +- service/pom.xml|2 +- shims/0.23/pom.xml |2 +- shims/aggregator/pom.xml |2 +- shims/common/pom.xml |2 +- shims/pom.xml |2 +- standalone-metastore/metastore-common/pom.xml |2 +- standalone-metastore/metastore-server/pom.xml |6 +- .../sql/derby/hive-schema-4.0.0-beta-2.derby.sql | 896 + .../upgrade-4.0.0-beta-1-to-4.0.0-beta-2.derby.sql |2 + .../src/main/sql/derby/upgrade.order.derby |1 + .../sql/mssql/hive-schema-4.0.0-beta-2.mssql.sql | 1476
