[hive] branch master updated: HIVE-26394 : Query based compaction fails for table with more than 6 columns (Mahesh Kumar Behera, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 69e6a5a4151 HIVE-26394 : Query based compaction fails for table with
more than 6 columns (Mahesh Kumar Behera, reviewed by Denys Kuzmenko)
69e6a5a4151 is described below
commit 69e6a5a4151100849d2b03b6b14b1605c3abc3f1
Author: mahesh kumar behera
AuthorDate: Mon Jul 18 17:52:29 2022 +0530
HIVE-26394 : Query based compaction fails for table with more than 6
columns (Mahesh Kumar Behera, reviewed by Denys Kuzmenko)
---
.../ql/txn/compactor/TestCrudCompactorOnTez.java | 4 +--
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 32 +++---
.../hadoop/hive/ql/io/orc/OrcNewInputFormat.java | 12 +---
3 files changed, 38 insertions(+), 10 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
index cffb58bc41c..eb9f4c4e2a8 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
@@ -1484,8 +1484,8 @@ public class TestCrudCompactorOnTez extends
CompactorOnTezTest {
String tableName = "testMinorCompaction";
executeStatementOnDriver("drop table if exists " + tableName, driver);
executeStatementOnDriver(
-"CREATE TABLE " + tableName + "(a INT, b STRING) " + " STORED AS ORC
TBLPROPERTIES ('transactional'='true')",
-driver);
+"CREATE TABLE " + tableName + "(a INT, b STRING, c int, d int, e
int, f int, j int, i int) " +
+" STORED AS ORC TBLPROPERTIES ('transactional'='true')",
driver);
CompactorTestUtil.runStreamingAPI(conf, dbName, tableName, Lists
.newArrayList(new CompactorTestUtil.StreamingConnectionOption(false,
false),
new CompactorTestUtil.StreamingConnectionOption(true, false),
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 0f1333b9a68..4b70ff5c5b7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -251,12 +251,29 @@ public class OrcInputFormat implements
InputFormat,
OrcRecordReader(Reader file, Configuration conf,
-FileSplit split) throws IOException {
+InputSplit inputSplit) throws IOException {
this.file = file;
numColumns = file.getSchema().getChildren().size();
+ FileSplit split = (FileSplit)inputSplit;
this.offset = split.getStart();
this.length = split.getLength();
- this.reader = createReaderFromFile(file, conf, offset, length);
+
+ // In case of query based compaction, the ACID table location is used as
the location of the external table.
+ // The assumption is that the table is treated as a external table. But
as per file, the table is ACID and thus
+ // the file schema can not be used to judge if the table is original or
not. It has to be as per the file split.
+
+ // CREATE temporary external table
delete_delta_default_tmp_compactor_testminorcompaction_1657797233724_result(
+ // `operation` int, `originalTransaction` bigint, `bucket` int, `rowId`
bigint, `currentTransaction` bigint,
+ // `row` struct<`a` :int, `b` :string, `c` :int, `d` :int, `e` :int, `f`
:int, `j` :int, `i` :int>)
+ // clustered by (`bucket`) sorted by (`originalTransaction`, `bucket`,
`rowId`) into 1 buckets stored as
+ // orc LOCATION
'file:/warehouse/testminorcompaction/delete_delta_001_006_v009'
+ // TBLPROPERTIES ('compactiontable'='true', 'bucketing_version'='2',
'transactional'='false')
+ if (inputSplit instanceof OrcSplit) {
+this.reader = createReaderFromFile(file, conf, offset, length,
((OrcSplit) inputSplit).isOriginal());
+ } else {
+this.reader = createReaderFromFile(file, conf, offset, length);
+ }
+
this.stats = new SerDeStats();
}
@@ -327,6 +344,14 @@ public class OrcInputFormat implements
InputFormat,
public static RecordReader createReaderFromFile(Reader file,
Configuration conf,
long offset, long length
+ ) throws IOException {
+return createReaderFromFile(file, conf, offset, length, isOriginal(file));
+
[hive] branch master updated: HIVE-26382 : Stats generation fails during CTAS for external partitioned table. (Mahesh Kumar Behera, reviewed by Ayush Saxena)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new eac61a4532e HIVE-26382 : Stats generation fails during CTAS for
external partitioned table. (Mahesh Kumar Behera, reviewed by Ayush Saxena)
eac61a4532e is described below
commit eac61a4532ee645441daf8b1c11e909e95df810f
Author: mahesh kumar behera
AuthorDate: Tue Jul 12 08:53:56 2022 +0530
HIVE-26382 : Stats generation fails during CTAS for external partitioned
table. (Mahesh Kumar Behera, reviewed by Ayush Saxena)
---
.../org/apache/hadoop/hive/ql/exec/MoveTask.java | 50 +
.../apache/hadoop/hive/ql/parse/TaskCompiler.java | 3 +-
ql/src/test/queries/clientpositive/ctas_blob.q | 6 ++-
.../results/clientpositive/llap/ctas_blob.q.out| 51 ++
4 files changed, 90 insertions(+), 20 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index d4514f4812f..250debfd9b7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -104,6 +104,29 @@ public class MoveTask extends Task implements
Serializable {
super();
}
+ private boolean moveFilesUsingManifestFile(FileSystem fs, Path sourcePath,
Path targetPath)
+ throws HiveException, IOException {
+if (work.isCTAS() && BlobStorageUtils.isBlobStorageFileSystem(conf, fs)) {
+ if (fs.exists(new Path(sourcePath, BLOB_MANIFEST_FILE))) {
+LOG.debug("Attempting to copy using the paths available in {}", new
Path(sourcePath, BLOB_MANIFEST_FILE));
+ArrayList filesKept;
+try (FSDataInputStream inStream = fs.open(new Path(sourcePath,
BLOB_MANIFEST_FILE))) {
+ String paths = IOUtils.toString(inStream, Charset.defaultCharset());
+ filesKept = new
ArrayList(Arrays.asList(paths.split(System.lineSeparator(;
+}
+// Remove the first entry from the list, it is the source path.
+Path srcPath = new Path(filesKept.remove(0));
+LOG.info("Copying files {} from {} to {}", filesKept, srcPath,
targetPath);
+// Do the move using the filesKept now directly to the target dir.
+Utilities.moveSpecifiedFilesInParallel(conf, fs, srcPath, targetPath,
new HashSet<>(filesKept));
+return true;
+ }
+ // Fallback case, in any case the _blob_files_kept isn't created, we can
do the normal logic. The file won't
+ // be created in case of empty source table as well
+}
+return false;
+ }
+
private void moveFile(Path sourcePath, Path targetPath, boolean isDfsDir)
throws HiveException {
try {
@@ -117,25 +140,12 @@ public class MoveTask extends Task implements
Serializable {
FileSystem fs = sourcePath.getFileSystem(conf);
- if (work.isCTAS() && BlobStorageUtils.isBlobStorageFileSystem(conf, fs))
{
-if (fs.exists(new Path(sourcePath, BLOB_MANIFEST_FILE))) {
- LOG.debug("Attempting to copy using the paths available in {}", new
Path(sourcePath, BLOB_MANIFEST_FILE));
- ArrayList filesKept;
- try (FSDataInputStream inStream = fs.open(new Path(sourcePath,
BLOB_MANIFEST_FILE))) {
-String paths = IOUtils.toString(inStream,
Charset.defaultCharset());
-filesKept = new
ArrayList(Arrays.asList(paths.split(System.lineSeparator(;
- }
- // Remove the first entry from the list, it is the source path.
- Path srcPath = new Path(filesKept.remove(0));
- LOG.info("Copying files {} from {} to {}", filesKept, srcPath,
targetPath);
- // Do the move using the filesKept now directly to the target dir.
- Utilities.moveSpecifiedFilesInParallel(conf, fs, srcPath,
targetPath, new HashSet<>(filesKept));
- perfLogger.perfLogEnd("MoveTask", PerfLogger.FILE_MOVES);
- return;
-}
-// Fallback case, in any case the _blob_files_kept isn't created, we
can do the normal logic. The file won't
-// be created in case of empty source table as well
+ // if _blob_files_kept is present, use it to move the files. Else fall
back to normal case.
+ if (moveFilesUsingManifestFile(fs, sourcePath, targetPath)) {
+perfLogger.perfLogEnd("MoveTask", PerfLogger.FILE_MOVES);
+return;
}
+
if (isDfsDir) {
moveFileInDfs (sourcePath, targetPath, conf);
} else {
@@ -470,6 +480,10 @@ public class MoveTask extends Task implements
Serializable {
// deal with dynamic partitions
DynamicPartitionCtx dpCtx = tbd.getDPCtx();
if (dpCtx != null && dpCtx.g
[hive] branch master updated: HIVE-26341: Replication logs & metrics shows last repl id as 0 when number of events is zero (Haymant Mangla, reviewed by Rajesh Balamohan and Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 248ce4aa5ee HIVE-26341: Replication logs & metrics shows last repl id
as 0 when number of events is zero (Haymant Mangla, reviewed by Rajesh
Balamohan and Mahesh Kumar Behera)
248ce4aa5ee is described below
commit 248ce4aa5ee079850b9d81e737493a1309485521
Author: Haymant Mangla <79496857+hmangl...@users.noreply.github.com>
AuthorDate: Tue Jun 21 10:07:09 2022 +0530
HIVE-26341: Replication logs & metrics shows last repl id as 0 when number
of events is zero (Haymant Mangla, reviewed by Rajesh Balamohan and Mahesh
Kumar Behera)
---
.../hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
index c9f0da41678..c5f6d6ed2f2 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
@@ -176,7 +176,8 @@ public class IncrementalLoadTasksBuilder {
taskChainTail = updateIncPendTask;
Map dbProps = new HashMap<>();
- dbProps.put(ReplicationSpec.KEY.CURR_STATE_ID_SOURCE.toString(),
String.valueOf(lastReplayedEvent));
+ dbProps.put(ReplicationSpec.KEY.CURR_STATE_ID_SOURCE.toString(),
+ String.valueOf((lastReplayedEvent == null) ? eventTo :
lastReplayedEvent));
ReplStateLogWork replStateLogWork = new ReplStateLogWork(replLogger,
dbProps, dumpDirectory,
metricCollector, shouldFailover);
Task barrierTask = TaskFactory.get(replStateLogWork, conf);
[hive] branch master updated: HIVE-26225: Delete operations in ObjectStore.cleanWriteNotificationEvents should be performed in different transactions (Haymant Mangla, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 54a0ab9bdb9 HIVE-26225: Delete operations in
ObjectStore.cleanWriteNotificationEvents should be performed in different
transactions (Haymant Mangla, reviewed by Mahesh Kumar Behera)
54a0ab9bdb9 is described below
commit 54a0ab9bdb9dc2da1a55c595ff0916d5c79d7cd4
Author: Haymant Mangla <79496857+hmangl...@users.noreply.github.com>
AuthorDate: Wed May 18 07:18:00 2022 +0530
HIVE-26225: Delete operations in ObjectStore.cleanWriteNotificationEvents
should be performed in different transactions (Haymant Mangla, reviewed by
Mahesh Kumar Behera)
---
.../apache/hadoop/hive/metastore/ObjectStore.java | 116 +++--
.../org/apache/hadoop/hive/metastore/RawStore.java | 2 +-
2 files changed, 40 insertions(+), 78 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 9b5ef82fdc2..83afdce22ea 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -24,7 +24,6 @@ import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCa
import static
org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
import java.io.IOException;
-import java.lang.reflect.Constructor;
import java.net.InetAddress;
import java.net.URI;
import java.nio.ByteBuffer;
@@ -37,7 +36,6 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
-import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -56,7 +54,6 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
import javax.jdo.JDODataStoreException;
@@ -11542,54 +11539,7 @@ public class ObjectStore implements RawStore,
Configurable {
@Override
public void cleanWriteNotificationEvents(int olderThan) {
-boolean commited = false;
-Query query = null;
-try {
- openTransaction();
- long tmp = System.currentTimeMillis() / 1000 - olderThan;
- int tooOld = (tmp > Integer.MAX_VALUE) ? 0 : (int) tmp;
- query = pm.newQuery(MTxnWriteNotificationLog.class, "eventTime <
tooOld");
- query.declareParameters("java.lang.Integer tooOld");
-
- int max_events = MetastoreConf.getIntVar(conf,
MetastoreConf.ConfVars.EVENT_CLEAN_MAX_EVENTS);
- max_events = max_events > 0 ? max_events : Integer.MAX_VALUE;
- query.setRange(0, max_events);
- query.setOrdering("txnId ascending");
-
- List toBeRemoved = (List)
query.execute(tooOld);
- int iteration = 0;
- int eventCount = 0;
- long minTxnId = 0;
- long minEventTime = 0;
- long maxTxnId = 0;
- long maxEventTime = 0;
- while (CollectionUtils.isNotEmpty(toBeRemoved)) {
-int listSize = toBeRemoved.size();
-if (iteration == 0) {
- MTxnWriteNotificationLog firstNotification = toBeRemoved.get(0);
- minTxnId = firstNotification.getTxnId();
- minEventTime = firstNotification.getEventTime();
-}
-MTxnWriteNotificationLog lastNotification = toBeRemoved.get(listSize -
1);
-maxTxnId = lastNotification.getTxnId();
-maxEventTime = lastNotification.getEventTime();
-
-pm.deletePersistentAll(toBeRemoved);
-eventCount += listSize;
-iteration++;
-toBeRemoved = (List) query.execute(tooOld);
- }
- if (iteration == 0) {
-LOG.info("No WriteNotification events found to be cleaned with
eventTime < {}.", tooOld);
- } else {
-LOG.info("WriteNotification Cleaned {} events with eventTime < {} in
{} iteration, " +
-"minimum txnId {} (with eventTime {}) and maximum txnId {} (with
eventTime {})",
-eventCount, tooOld, iteration, minTxnId, minEventTime, maxTxnId,
maxEventTime);
- }
- commited = commitTransaction();
-} finally {
- rollbackAndCleanup(commited, query);
-}
+cleanOlderEvents(olderThan, MTxnWriteNotificationLog.class,
"TxnWriteNotificationLog");
}
@Override
@@ -11756,6 +11706,10 @@ public class ObjectStore implements RawStore,
Configurable {
@Override
public void cleanNotificationEvents(int olderThan) {
+cle
[hive] branch master updated: HIVE-26105 : Show columns shows extra values if column comments contains specific Chinese character (Mahesh Kumar Behera, Ramesh Kumar Thangarajan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c589314240 HIVE-26105 : Show columns shows extra values if column
comments contains specific Chinese character (Mahesh Kumar Behera, Ramesh Kumar
Thangarajan)
c589314240 is described below
commit c5893142408470391f83b284b31fbe281ef4351f
Author: mahesh kumar behera
AuthorDate: Tue Apr 5 13:47:37 2022 +0530
HIVE-26105 : Show columns shows extra values if column comments contains
specific Chinese character (Mahesh Kumar Behera, Ramesh Kumar Thangarajan)
---
.../ql/ddl/table/column/show/ShowColumnsOperation.java | 3 ++-
ql/src/test/queries/clientpositive/show_columns.q | 4
.../test/results/clientpositive/llap/show_columns.q.out | 17 +
3 files changed, 23 insertions(+), 1 deletion(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java
index d621dcbf27..289479b7ee 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java
@@ -108,7 +108,8 @@ public class ShowColumnsOperation extends
DDLOperation {
private void writeColumns(DataOutputStream outStream, List
columns) throws IOException {
TextMetaDataTable tmd = new TextMetaDataTable();
for (FieldSchema fieldSchema : columns) {
- tmd.addRow(ShowUtils.extractColumnValues(fieldSchema, false, null));
+ // For show Columns operation, we just need the column name.
+ tmd.addRow(fieldSchema.getName());
}
// In case the query is served by HiveServer2, don't pad it with spaces,
diff --git a/ql/src/test/queries/clientpositive/show_columns.q
b/ql/src/test/queries/clientpositive/show_columns.q
index a3677db362..fe2d64f211 100644
--- a/ql/src/test/queries/clientpositive/show_columns.q
+++ b/ql/src/test/queries/clientpositive/show_columns.q
@@ -50,3 +50,7 @@ SHOW SORTED COLUMNS in foo_n7 from test_db "col+";
SHOW SORTED COLUMNS in foo_n7 from test_db "c";
SHOW SORTED COLUMNS from foo_n7 from test_db "c*";
SHOW SORTED COLUMNS from foo_n7 from test_db like 'c*';
+
+-- show column for table with chinese comments. 名 UTF code is 0x540D. D means
CR ( '\r'). It should not cause issue.
+create table tbl_test (fld string COMMENT '期末日期', fld1 string COMMENT '班次名称',
fld2 string COMMENT '排班人数');
+show columns from tbl_test;
\ No newline at end of file
diff --git a/ql/src/test/results/clientpositive/llap/show_columns.q.out
b/ql/src/test/results/clientpositive/llap/show_columns.q.out
index e3391669e2..4b2cb904db 100644
--- a/ql/src/test/results/clientpositive/llap/show_columns.q.out
+++ b/ql/src/test/results/clientpositive/llap/show_columns.q.out
@@ -451,3 +451,20 @@ col3
cola
colb
colc
+PREHOOK: query: create table tbl_test (fld string COMMENT '期末日期', fld1 string
COMMENT '班次名称', fld2 string COMMENT '排班人数')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl_test
+POSTHOOK: query: create table tbl_test (fld string COMMENT '期末日期', fld1 string
COMMENT '班次名称', fld2 string COMMENT '排班人数')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tbl_test
+PREHOOK: query: show columns from tbl_test
+PREHOOK: type: SHOWCOLUMNS
+PREHOOK: Input: default@tbl_test
+POSTHOOK: query: show columns from tbl_test
+POSTHOOK: type: SHOWCOLUMNS
+POSTHOOK: Input: default@tbl_test
+fld
+fld1
+fld2
[hive] branch master updated: HIVE-26098 : Duplicate path/Jar in hive.aux.jars.path or hive.reloadable.aux.jars.path causing IllegalArgumentException (Mahesh Kumar Behera, reviewed by Ayush Saxena)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ef36922 HIVE-26098 : Duplicate path/Jar in hive.aux.jars.path or
hive.reloadable.aux.jars.path causing IllegalArgumentException (Mahesh Kumar
Behera, reviewed by Ayush Saxena)
ef36922 is described below
commit ef369227ea2eb9bf8b0b54147e7231364d79810a
Author: mahesh kumar behera
AuthorDate: Fri Apr 1 12:48:34 2022 +0530
HIVE-26098 : Duplicate path/Jar in hive.aux.jars.path or
hive.reloadable.aux.jars.path causing IllegalArgumentException (Mahesh Kumar
Behera, reviewed by Ayush Saxena)
---
.../apache/hadoop/hive/ql/exec/tez/DagUtils.java | 22 +-
.../test/queries/clientpositive/whroot_external1.q | 16 +
.../clientpositive/llap/whroot_external1.q.out | 27 ++
3 files changed, 54 insertions(+), 11 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 73ba430..7622bcc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -1406,9 +1406,14 @@ public class DagUtils {
public LocalResource localizeResource(
Path src, Path dest, LocalResourceType type, Configuration conf) throws
IOException {
FileSystem destFS = dest.getFileSystem(conf);
-// We call copyFromLocal below, so we basically assume src is a local file.
-FileSystem srcFs = FileSystem.getLocal(conf);
-if (src != null && !checkPreExisting(srcFs, src, dest, conf)) {
+FileSystem srcFs;
+if (src.toUri().getScheme() != null) {
+ srcFs = src.getFileSystem(conf);
+} else {
+ srcFs = FileSystem.getLocal(conf);
+}
+
+if (!checkPreExisting(srcFs, src, dest, conf)) {
// copy the src to the destination and create local resource.
// do not overwrite.
String srcStr = src.toString();
@@ -1424,12 +1429,8 @@ public class DagUtils {
return createLocalResource(destFS, dest, type,
LocalResourceVisibility.PRIVATE);
}
try {
-if (src.toUri().getScheme()!=null) {
- FileUtil.copy(src.getFileSystem(conf), src, destFS, dest, false,
false, conf);
-}
-else {
- destFS.copyFromLocalFile(false, false, src, dest);
-}
+// FileUtil.copy takes care of copy from local filesystem internally.
+FileUtil.copy(srcFs, src, destFS, dest, false, false, conf);
synchronized (notifier) {
notifier.notifyAll(); // Notify if we have successfully copied the
file.
}
@@ -1458,8 +1459,7 @@ public class DagUtils {
}
}
}
-return createLocalResource(destFS, dest, type,
-LocalResourceVisibility.PRIVATE);
+return createLocalResource(destFS, dest, type,
LocalResourceVisibility.PRIVATE);
}
public boolean checkOrWaitForTheFile(FileSystem srcFs, Path src, Path dest,
Configuration conf,
diff --git a/ql/src/test/queries/clientpositive/whroot_external1.q
b/ql/src/test/queries/clientpositive/whroot_external1.q
index c3724a4..88c253c 100644
--- a/ql/src/test/queries/clientpositive/whroot_external1.q
+++ b/ql/src/test/queries/clientpositive/whroot_external1.q
@@ -112,3 +112,19 @@ dfs -copyFromLocal ../../data/files/test_dec_space.csv
hdfs:///tmp/test_dec_spac
create external table test_dec_space (id int, value decimal) ROW FORMAT
DELIMITED
FIELDS TERMINATED BY ',' location 'hdfs:///tmp/test_dec_space';
select * from test_dec_space;
+
+create table tbl (fld int);
+dfs -mkdir -p hdfs:///tmp/test_load_aux_jar;
+dfs -copyFromLocal ${system:hive.root}/data/files/identity_udf.jar
hdfs:///tmp/test_load_aux_jar/;
+
+-- both hive.aux.jars.path and hive.reloadable.aux.jars.path pointing to the
same jar.
+SET hive.aux.jars.path=hdfs:///tmp/test_load_aux_jar/identity_udf.jar;
+SET hive.reloadable.aux.jars.path=hdfs:///tmp/test_load_aux_jar/;
+
+-- reload will load the identity_udf.jar from tmp/test_load_aux_jar
+RELOAD;
+
+insert into tbl values(1);
+select * from tbl;
+
+dfs -rmr -f hdfs:///tmp/test_load_aux_jar/
\ No newline at end of file
diff --git a/ql/src/test/results/clientpositive/llap/whroot_external1.q.out
b/ql/src/test/results/clientpositive/llap/whroot_external1.q.out
index 94fd2ae..f9716e6 100644
--- a/ql/src/test/results/clientpositive/llap/whroot_external1.q.out
+++ b/ql/src/test/results/clientpositive/llap/whroot_external1.q.out
@@ -614,3 +614,30 @@ POSTHOOK: Input: default@test_dec_space
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 0
2 1
+PREHOOK: query: create table tbl (fld int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl
+POSTHOOK: query: create table tbl (fld int)
[hive] branch master updated (f0a98c7 -> e1ca33f)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from f0a98c7 HIVE-25989: HBase CTLT failure rollback should not drop the underlying HBase table (#3076) (Marton Bod, reviewed by Peter Vary) add e1ca33f HIVE-26017 : Insert with partition value containing colon and space is creating partition having wrong value (Mahesh Kumar Behera, reviewed by Ramesh Kumar Thangarajan) No new revisions were added by this update. Summary of changes: .../ql/plan/ConditionalResolverMergeFiles.java | 10 +- .../queries/clientpositive/test_merge_tez_files.q | 62 .../clientpositive/llap/test_merge_tez_files.q.out | 412 + 3 files changed, 483 insertions(+), 1 deletion(-) create mode 100644 ql/src/test/queries/clientpositive/test_merge_tez_files.q create mode 100644 ql/src/test/results/clientpositive/llap/test_merge_tez_files.q.out
[hive] branch master updated: HIVE-25864 : Hive query optimisation creates wrong plan for predicate pushdown with windowing function (Mahesh Kumar Behera, reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 003eaeb HIVE-25864 : Hive query optimisation creates wrong plan for
predicate pushdown with windowing function (Mahesh Kumar Behera, reviewed by
Zoltan Haindrich)
003eaeb is described below
commit 003eaeb294203c45b0626978765d3ef4265604af
Author: mahesh kumar behera
AuthorDate: Thu Jan 20 17:23:33 2022 +0530
HIVE-25864 : Hive query optimisation creates wrong plan for predicate
pushdown with windowing function (Mahesh Kumar Behera, reviewed by Zoltan
Haindrich)
---
.../rules/HiveFilterProjectTransposeRule.java | 4 +-
.../test/queries/clientpositive/ppd_windowing3.q | 73
.../clientpositive/llap/ppd_windowing3.q.out | 454 +
3 files changed, 529 insertions(+), 2 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
index 02b4a53..47aae3f 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
@@ -169,7 +169,7 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
RexNode newCondition = RelOptUtil.pushPastProject(ce, origproject);
if
(HiveCalciteUtil.isDeterministicFuncWithSingleInputRef(newCondition,
commonPartitionKeys)) {
- newPartKeyFilConds.add(newCondition);
+ newPartKeyFilConds.add(ce);
} else {
unpushedFilConds.add(ce);
}
@@ -200,7 +200,7 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
private static RelNode getNewProject(RexNode filterCondToPushBelowProj,
RexNode unPushedFilCondAboveProj, Project oldProj,
RelDataTypeFactory typeFactory, RelBuilder relBuilder) {
-// convert the filter to one that references the child of the project
+// convert the filter to one that references the child of the project.
RexNode newPushedCondition =
RelOptUtil.pushPastProject(filterCondToPushBelowProj, oldProj);
// Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
diff --git a/ql/src/test/queries/clientpositive/ppd_windowing3.q
b/ql/src/test/queries/clientpositive/ppd_windowing3.q
new file mode 100644
index 000..aa94a38
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/ppd_windowing3.q
@@ -0,0 +1,73 @@
+
+CREATE TABLE package_order (
+order_num string,
+vin_id string,
+package_start_dt string);
+
+CREATE TABLE package_order_gsp (order_num string,
+cust_acct_sk decimal(38,0),
+ to_vin string,
+ cancellation_dt string,
+ confirmation_num string);
+
+insert into package_order values ('1', 'DEADBEAF', '2022-01-22');
+insert into package_order values ('1', 'DEADBEAF', '2022-01-22');
+insert into package_order values ('1', 'DEADBEAF1', '2022-01-23');
+insert into package_order values ('1', 'DEADBEAF1', '2022-01-23');
+insert into package_order_gsp values ('1', 1.1, '1', null, '1');
+
+set hive.cbo.enable = false;
+set hive.explain.user=false;
+
+explain select *
+from (
+select t1.vin_id, row_number()over(partition by t1.vin_id order by
package_start_dt desc) rn
+from package_order_gsp su
+inner join package_order t1
+on su.confirmation_num=t1.order_num
+where su.cancellation_dt is null
+) tt
+where tt.vin_id='DEADBEAF';
+
+select *
+from (
+select t1.vin_id, row_number()over(partition by t1.vin_id order by
package_start_dt desc) rn
+from package_order_gsp su
+inner join package_order t1
+on su.confirmation_num=t1.order_num
+where su.cancellation_dt is null
+) tt
+where tt.vin_id='DEADBEAF';
+
+set hive.cbo.enable = true;
+
+explain select *
+from (
+select t1.vin_id, row_number()over(partition by t1.vin_id order by
package_start_dt desc) rn
+from package_order_gsp su
+inner join package_order t1
+on su.confirmation_num=t1.order_num
+where su.cancellation_dt is null
+) tt
+where tt.vin_id='DEADBEAF';
+
+select *
+from (
+select t1.vin_id, row_number()over(partition by t1.vin_id order by
package_start_dt desc) rn
+from package_order_gsp su
+inner join package_order t1
+on su.confirmation_num=t1.order_num
+where su.cancellation_dt is null
+) tt
+where tt.vin_id='DEADBEAF';
+
+select *
+from (
+select t1.vin_id, row_number()over(partition by t1.vin_id order by
package_start_d
[hive] branch master updated: HIVE-25638 : Select returns deleted records in Hive ACID table (Mahesh Kumar Behera, reviewed by Adam Szita)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2c80394 HIVE-25638 : Select returns deleted records in Hive ACID
table (Mahesh Kumar Behera, reviewed by Adam Szita)
2c80394 is described below
commit 2c803945c0a75a2e0898f39b0dd424d5a59f279b
Author: mahesh kumar behera
AuthorDate: Fri Oct 29 20:39:13 2021 +0530
HIVE-25638 : Select returns deleted records in Hive ACID table (Mahesh
Kumar Behera, reviewed by Adam Szita)
---
.../ql/io/orc/VectorizedOrcAcidRowBatchReader.java | 2 +-
.../orc/TestVectorizedOrcAcidRowBatchReader.java | 22 +++---
2 files changed, 16 insertions(+), 8 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
index 2b70378..385db48 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
@@ -586,7 +586,7 @@ public class VectorizedOrcAcidRowBatchReader
maxKey = keyIndex[lastStripeIndex];
} else {
if(columnStatsPresent) {
-maxKey =
getKeyInterval(stats.get(firstStripeIndex).getColumnStatistics()).getMaxKey();
+maxKey =
getKeyInterval(stats.get(lastStripeIndex).getColumnStatistics()).getMaxKey();
}
}
OrcRawRecordMerger.KeyInterval keyInterval =
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
index 0f78799..b7fd6ac 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java
@@ -244,12 +244,13 @@ public class TestVectorizedOrcAcidRowBatchReader {
//create 3 insert deltas so that we have 3 splits
RecordUpdater updater = new OrcRecordUpdater(root, options);
-updater.insert(options.getMinimumWriteId(),
-new DummyRow(1, 0, options.getMinimumWriteId(), bucket));
-updater.insert(options.getMinimumWriteId(),
-new DummyRow(2, 1, options.getMinimumWriteId(), bucket));
-updater.insert(options.getMinimumWriteId(),
-new DummyRow(3, 2, options.getMinimumWriteId(), bucket));
+
+//In the first delta add 2000 recs to simulate recs in multiple stripes.
+int numRows = 2000;
+for (int i = 1; i <= numRows; i++) {
+ updater.insert(options.getMinimumWriteId(),
+ new DummyRow(i, i-1, options.getMinimumWriteId(), bucket));
+}
updater.close(false);
options.minimumWriteId(2)
@@ -328,7 +329,7 @@ public class TestVectorizedOrcAcidRowBatchReader {
if(filterOn) {
assertEquals(new OrcRawRecordMerger.KeyInterval(
new RecordIdentifier(1, bucketProperty, 0),
- new RecordIdentifier(1, bucketProperty, 2)),
+ new RecordIdentifier(1, bucketProperty, numRows - 1)),
keyInterval);
}
else {
@@ -385,6 +386,13 @@ public class TestVectorizedOrcAcidRowBatchReader {
HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVETESTMODEACIDKEYIDXSKIP,
true);
testDeleteEventFiltering2();
}
+ @Test
+ public void testDeleteEventFilteringOnWithoutIdx3() throws Exception {
+HiveConf.setBoolVar(conf, HiveConf.ConfVars.FILTER_DELETE_EVENTS, true);
+HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVETESTMODEACIDKEYIDXSKIP,
true);
+conf.set("orc.stripe.size", "1000");
+testDeleteEventFiltering();
+ }
private void testDeleteEventFiltering2() throws Exception {
boolean filterOn =
[hive] branch master updated (381d21d -> 0c3fea9)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 381d21d HIVE-25529: Test reading/writing V2 tables with delete files (#2644) (Marton Bod, reviewed by Peter Vary) add 0c3fea9 HIVE-25527 : LLAP Scheduler task exits with fatal error if the executor node is down. (Mahesh Kumar Behera, reviewed by Panos Garefalakis) No new revisions were added by this update. Summary of changes: .../llap/tezplugins/LlapTaskSchedulerService.java | 38 ++ 1 file changed, 25 insertions(+), 13 deletions(-)
[hive] branch master updated: HIVE-25475: TestStatsReplicationScenarios.testForParallelBootstrapLoad is unstable (Haymant Mangla, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5c53f92 HIVE-25475:
TestStatsReplicationScenarios.testForParallelBootstrapLoad is unstable (Haymant
Mangla, reviewed by Mahesh Kumar Behera)
5c53f92 is described below
commit 5c53f92bdf3834988db0e0c92d7eff9891bdf639
Author: Haymant Mangla <79496857+hmangl...@users.noreply.github.com>
AuthorDate: Thu Aug 26 20:35:31 2021 +0530
HIVE-25475: TestStatsReplicationScenarios.testForParallelBootstrapLoad is
unstable (Haymant Mangla, reviewed by Mahesh Kumar Behera)
---
.../org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java | 1 -
ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java| 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java
index 62792be..cca82a8 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java
@@ -684,7 +684,6 @@ public class TestStatsReplicationScenarios {
}
@Test
- @Ignore("HIVE-25475")
public void testForParallelBootstrapLoad() throws Throwable {
LOG.info("Testing " + testName.getClass().getName() + "." +
testName.getMethodName());
testStatsReplicationCommon(true, false, false);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 4493928..3de2e8d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -5008,7 +5008,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
List newFiles = null;
if (newFilesStatus != null) {
- newFiles = new ArrayList<>();
+ newFiles = Collections.synchronizedList(new ArrayList());
}
// If we're moving files around for an ACID write then the rules and paths
are all different.
[hive] branch master updated (baf8247 -> ed1882e)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from baf8247 HIVE-25438 : Update partition column stats fails with invalid syntax error for MySql (Mahesh Kumar Behera, Ramesh Kumar Thangarajan) add ed1882e HIVE-25413 : Optimise ObjectStore::alterPartitions to reduce DB calls for getPartitions (Mahesh Kumar Behera, reviewed by Rajesh Balamohan) No new revisions were added by this update. Summary of changes: .../apache/hadoop/hive/metastore/ObjectStore.java | 115 ++--- 1 file changed, 77 insertions(+), 38 deletions(-)
[hive] branch master updated: HIVE-25438 : Update partition column stats fails with invalid syntax error for MySql (Mahesh Kumar Behera, Ramesh Kumar Thangarajan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new baf8247 HIVE-25438 : Update partition column stats fails with invalid
syntax error for MySql (Mahesh Kumar Behera, Ramesh Kumar Thangarajan)
baf8247 is described below
commit baf8247af78f35a68fab0756f9d571c73cdf5f96
Author: mahesh kumar behera
AuthorDate: Tue Aug 17 10:33:54 2021 +0530
HIVE-25438 : Update partition column stats fails with invalid syntax error
for MySql (Mahesh Kumar Behera, Ramesh Kumar Thangarajan)
---
.../apache/hadoop/hive/metastore/DirectSqlUpdateStat.java| 12
1 file changed, 12 insertions(+)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdateStat.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdateStat.java
index 6f1231d..2fb9d20 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdateStat.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdateStat.java
@@ -565,6 +565,14 @@ class DirectSqlUpdateStat {
return partitionInfoMap;
}
+ private void setAnsiQuotes(Connection dbConn) throws SQLException {
+if (sqlGenerator.getDbProduct().isMYSQL()) {
+ try (Statement stmt = dbConn.createStatement()) {
+stmt.execute("SET @@session.sql_mode=ANSI_QUOTES");
+ }
+}
+ }
+
/**
* Update the statistics for the given partitions. Add the notification logs
also.
* @return map of partition key to column stats if successful, null
otherwise.
@@ -582,6 +590,8 @@ class DirectSqlUpdateStat {
jdoConn = pm.getDataStoreConnection();
dbConn = (Connection) (jdoConn.getNativeConnection());
+ setAnsiQuotes(dbConn);
+
Map partitionInfoMap =
getPartitionInfo(dbConn, tbl.getId(), partColStatsMap);
Map> result =
@@ -647,6 +657,8 @@ class DirectSqlUpdateStat {
jdoConn = pm.getDataStoreConnection();
dbConn = (Connection) (jdoConn.getNativeConnection());
+ setAnsiQuotes(dbConn);
+
// This loop will be iterated at max twice. If there is no records, it
will first insert and then do a select.
// We are not using any upsert operations as select for update and then
update is required to make sure that
// the caller gets a reserved range for CSId not used by any other
thread.
[hive] branch master updated: HIVE-25414 : Optimise Hive::addWriteNotificationLog: Reduce FS call per notification. (Mahesh Kumar Behera, reviewed by Rajesh Balamohan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a0f0fc8 HIVE-25414 : Optimise Hive::addWriteNotificationLog: Reduce
FS call per notification. (Mahesh Kumar Behera, reviewed by Rajesh Balamohan)
a0f0fc8 is described below
commit a0f0fc82d538d117b222b0f696f33e40d8cc023f
Author: mahesh kumar behera
AuthorDate: Mon Aug 16 07:51:37 2021 +0530
HIVE-25414 : Optimise Hive::addWriteNotificationLog: Reduce FS call per
notification. (Mahesh Kumar Behera, reviewed by Rajesh Balamohan)
---
.../ql/ddl/table/create/CreateTableOperation.java | 5 +-
.../org/apache/hadoop/hive/ql/exec/MoveTask.java | 3 +-
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 11 ++--
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 66 --
.../hive/streaming/HiveStreamingConnection.java| 3 +-
5 files changed, 49 insertions(+), 39 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java
index ead495b..ef327f4 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.ddl.table.create;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.repl.ReplConst;
@@ -153,9 +154,9 @@ public class CreateTableOperation extends
DDLOperation {
if (!createdTable.isPartitioned() &&
AcidUtils.isTransactionalTable(createdTable)) {
org.apache.hadoop.hive.metastore.api.Table tTable =
createdTable.getTTable();
Path tabLocation = new Path(tTable.getSd().getLocation());
-List newFilesList;
+List newFilesList;
try {
- newFilesList =
HdfsUtils.listPath(tabLocation.getFileSystem(context.getConf()), tabLocation,
null, true);
+ newFilesList =
HdfsUtils.listLocatedFileStatus(tabLocation.getFileSystem(context.getConf()),
tabLocation, null, true);
} catch (IOException e) {
LOG.error("Error listing files", e);
throw new HiveException(e);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 4580f1d..19fcdd3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -344,8 +344,7 @@ public class MoveTask extends Task implements
Serializable {
FileSystem srcFs = sourcePath.getFileSystem(conf);
FileStatus[] srcs = srcFs.globStatus(sourcePath);
if(srcs != null) {
- List newFiles = new ArrayList<>();
- Hive.moveAcidFiles(srcFs, srcs, targetPath, newFiles);
+ Hive.moveAcidFiles(srcFs, srcs, targetPath, null);
} else {
LOG.debug("No files found to move from " + sourcePath + " to " +
targetPath);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 9790123..7060ecc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -2855,7 +2855,7 @@ public final class Utilities {
public static final class PartitionDetails {
public Map fullSpec;
public Partition partition;
-public List newFiles;
+public List newFiles;
public boolean hasOldPartition = false;
public AcidUtils.TableSnapshot tableSnapshot;
}
@@ -2897,7 +2897,7 @@ public final class Utilities {
Collections.synchronizedMap(new LinkedHashMap<>());
// calculate full path spec for each valid partition path
- allPartition.entrySet().forEach(partEntry -> {
+ for (Map.Entry>> partEntry :
allPartition.entrySet()) {
Path partPath = partEntry.getKey();
Map fullPartSpec = Maps.newLinkedHashMap(partSpec);
String staticParts = Warehouse.makeDynamicPartName(partSpec);
@@ -2911,11 +2911,14 @@ public final class Utilities {
PartitionDetails details = new PartitionDetails();
details.fullSpec = fullPartSpec;
if (partEntry.getValue().isPresent()) {
-details.newFiles = partEntry.getValue().get();
+details.newFiles = new ArrayList<>();
+for (Path filePath : partEntry.getValue().get()) {
+ details.newFiles.add(fs.getFileStatus(filePath));
+
[hive] branch master updated (8785dd2 -> 0bc7d1b)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 8785dd2 HIVE-25350. Replication fails for external tables on setting owner/groups. (#2498)(Ayush Saxena, reviewed by Arko Sharma) add 0bc7d1b HIVE-25417 : Null bit vector is not handled while getting the stats for Postgres backend (Mahesh Kumar Behera, reviewed by Zoltan Haindrich,Ramesh Kumar Thangarajan) No new revisions were added by this update. Summary of changes: .../queries/clientpositive/nullBitVectorStats.q| 210 .../clientpositive/llap/nullBitVectorStats.q.out | 579 + .../apache/hadoop/hive/metastore/ObjectStore.java | 10 - .../hadoop/hive/metastore/StatObjectConverter.java | 22 +- .../model/MPartitionColumnStatistics.java | 25 +- .../metastore/model/MTableColumnStatistics.java| 25 +- 6 files changed, 839 insertions(+), 32 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/nullBitVectorStats.q create mode 100644 ql/src/test/results/clientpositive/llap/nullBitVectorStats.q.out
[hive] branch master updated (2431fd1 -> e240417)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 2431fd1 HIVE-24705: Authorization support for Storage-handler based tables (Saihemanth via Naveen Gangam) add e240417 HIVE-25205 : Reduce overhead of adding write notification log during batch loading of partition. (Mahesh Kumar Behera, reviewed by Pravin Kumar Sinha) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hive/conf/HiveConf.java |4 + .../hcatalog/listener/DbNotificationListener.java | 255 +- .../listener/TestDbNotificationListener.java |5 + .../metastore/SynchronizedMetaStoreClient.java |5 + .../ql/ddl/table/create/CreateTableOperation.java |3 +- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 66 +- .../src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2873 +++- .../src/gen/thrift/gen-cpp/ThriftHiveMetastore.h | 126 + .../ThriftHiveMetastore_server.skeleton.cpp|5 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp| 3993 .../src/gen/thrift/gen-cpp/hive_metastore_types.h | 86 + .../hive/metastore/api/AlterPartitionsRequest.java | 36 +- .../hive/metastore/api/AlterTableRequest.java | 32 +- .../metastore/api/ClearFileMetadataRequest.java| 32 +- .../hive/metastore/api/ClientCapabilities.java | 36 +- .../hive/metastore/api/CreateDatabaseRequest.java | 44 +- .../hive/metastore/api/CreateTableRequest.java | 248 +- .../hive/metastore/api/ExtendedTableInfo.java | 64 +- .../hive/metastore/api/FindSchemasByColsResp.java | 36 +- .../metastore/api/GetAllFunctionsResponse.java | 36 +- .../hive/metastore/api/GetDatabaseRequest.java | 32 +- .../hive/metastore/api/GetFieldsResponse.java | 36 +- .../api/GetFileMetadataByExprRequest.java | 32 +- .../metastore/api/GetFileMetadataByExprResult.java | 48 +- .../hive/metastore/api/GetFileMetadataRequest.java | 32 +- .../hive/metastore/api/GetFileMetadataResult.java | 44 +- .../hive/metastore/api/GetOpenTxnsRequest.java | 36 +- .../metastore/api/GetPartitionNamesPsRequest.java | 32 +- .../metastore/api/GetPartitionNamesPsResponse.java | 32 +- .../hive/metastore/api/GetPartitionRequest.java| 32 +- .../metastore/api/GetPartitionsFilterSpec.java | 32 +- .../api/GetPartitionsPsWithAuthRequest.java| 64 +- .../api/GetPartitionsPsWithAuthResponse.java | 36 +- .../hive/metastore/api/GetPartitionsRequest.java | 64 +- .../hive/metastore/api/GetPartitionsResponse.java | 36 +- .../hive/metastore/api/GetProjectionsSpec.java | 32 +- .../hive/metastore/api/GetSchemaResponse.java | 36 +- .../hadoop/hive/metastore/api/GetTableRequest.java | 32 +- .../hive/metastore/api/GetTablesExtRequest.java| 32 +- .../hive/metastore/api/GetTablesRequest.java | 64 +- .../hadoop/hive/metastore/api/GetTablesResult.java | 36 +- .../hive/metastore/api/PartitionsResponse.java | 36 +- .../hive/metastore/api/PutFileMetadataRequest.java | 64 +- .../hive/metastore/api/RenamePartitionRequest.java | 32 +- .../hive/metastore/api/ReplicationMetricList.java | 36 +- .../hadoop/hive/metastore/api/SchemaVersion.java | 36 +- .../hive/metastore/api/ThriftHiveMetastore.java| 4807 .../hive/metastore/api/WMFullResourcePlan.java | 144 +- .../api/WMGetAllResourcePlanResponse.java | 36 +- .../api/WMGetTriggersForResourePlanResponse.java | 36 +- .../api/WMValidateResourcePlanResponse.java| 64 +- .../api/WriteNotificationLogBatchRequest.java | 724 +++ ...java => WriteNotificationLogBatchResponse.java} | 52 +- .../gen-php/metastore/AlterPartitionsRequest.php | 20 +- .../thrift/gen-php/metastore/AlterTableRequest.php | 18 +- .../gen-php/metastore/ClearFileMetadataRequest.php | 18 +- .../gen-php/metastore/ClientCapabilities.php | 18 +- .../gen-php/metastore/CreateDatabaseRequest.php| 26 +- .../gen-php/metastore/CreateTableRequest.php | 138 +- .../thrift/gen-php/metastore/ExtendedTableInfo.php | 36 +- .../gen-php/metastore/FindSchemasByColsResp.php| 20 +- .../gen-php/metastore/GetAllFunctionsResponse.php | 20 +- .../gen-php/metastore/GetDatabaseRequest.php | 18 +- .../thrift/gen-php/metastore/GetFieldsResponse.php | 20 +- .../metastore/GetFileMetadataByExprRequest.php | 18 +- .../metastore/GetFileMetadataByExprResult.php | 28 +- .../gen-php/metastore/GetFileMetadataRequest.php | 18 +- .../gen-php/metastore/GetFileMetadataResult.php| 26 +- .../gen-php/metastore/GetOpenTxnsRequest.php | 18 +- .../metastore/GetPartitionNamesPsRequest.php | 18 +- .../metastore/GetPartitionNamesPsResponse.php | 18 +- .../gen-php/metast
[hive] branch master updated (96b39cd -> 75dea70)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 96b39cd HIVE-25115: Compaction queue entries may accumulate in "ready for cleaning" state (Denys Kuzmenko, reviewed by Karen Coppage) add 75dea70 HIVE-25373 : Modify buildColumnStatsDesc to send configured number of stats for updation (Mahesh Kumar Behera, reviewed by Rajesh Balamohan) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 + .../hadoop/hive/ql/stats/ColStatsProcessor.java| 62 +++-- .../clientpositive/partitioned_table_stats.q | 17 ++ .../llap/partitioned_table_stats.q.out | 292 + 4 files changed, 355 insertions(+), 19 deletions(-)
[hive] branch master updated (773b929 -> e6d98b8)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 773b929 HIVE-25380: Remove the Hive Privilege Object for Database in the ReadTableEvent and CreateTableEvent (#2525) (Sai Hemanth Gantasala, reviewed by Yongzhi Chen) add e6d98b8 HIVE-25014: Optimise ObjectStore::updateTableColumnStatistics (Rajesh Balamohan, reviewed by Mahesh Kumar Behera) No new revisions were added by this update. Summary of changes: .../main/java/org/apache/hadoop/hive/metastore/ObjectStore.java | 8 1 file changed, 4 insertions(+), 4 deletions(-)
[hive] branch master updated (8514c87 -> 3ad8ad3)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 8514c87 HIVE-25384: Bump ORC to 1.6.9 (#2530) add 3ad8ad3 HIVE-25342 : Optimize set_aggr_stats_for for mergeColStats path. (Mahesh Kumar Behera, , reviewed by Rajesh Balamohan) No new revisions were added by this update. Summary of changes: .../queries/clientpositive/autoColumnStats_2.q | 3 ++ .../clientpositive/llap/autoColumnStats_2.q.out| 38 ++ .../hadoop/hive/metastore/DirectSqlUpdateStat.java | 4 ++- .../apache/hadoop/hive/metastore/HMSHandler.java | 19 +-- 4 files changed, 61 insertions(+), 3 deletions(-)
[hive] branch master updated (825be82 -> 7779ccf)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 825be82 HIVE-25348: Skip metrics collection about writes to tables with tblproperty no_auto_compaction=true if CTAS (Karen Coppage, reviewed by Laszlo Pinter) add 7779ccf HIVE-25338: AIOBE in conv UDF if input is empty (Naresh P R, reviewed by Mahesh Kumar Behera) No new revisions were added by this update. Summary of changes: .../org/apache/hadoop/hive/ql/udf/UDFConv.java | 3 ++ ql/src/test/queries/clientpositive/udf_conv.q | 5 +++ .../results/clientpositive/llap/udf_conv.q.out | 36 ++ 3 files changed, 44 insertions(+)
[hive] branch master updated (3fe1d58 -> c401347)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 3fe1d58 HIVE-25242. Query performs extremely slow with vectorized.adaptor = chosen (#2390) (Attila Magyar reviewed by Panagiotis Garefalakis) add c401347 HIVE-25243: Llap external client - Handle nested values when the parent struct is null (Shubham Chaurasia, reviewed by Mahesh Kumar Behera) No new revisions were added by this update. Summary of changes: .../hive/jdbc/TestJdbcWithMiniLlapVectorArrow.java | 1 + .../TestMiniLlapVectorArrowWithLlapIODisabled.java | 199 + .../apache/hadoop/hive/ql/io/arrow/Serializer.java | 15 ++ 3 files changed, 215 insertions(+) create mode 100644 itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMiniLlapVectorArrowWithLlapIODisabled.java
[hive] branch master updated (570f0a1 -> d7a6175)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 570f0a1 HIVE-25217: Move isEligibleForCompaction evaluation under the Initiator thread pool (Denys Kuzmenko, reviewed by Karen Coppage) add d7a6175 HIVE-25274: TestLimitOperator fails if default engine is Tez (#2417) (Laszlo Bodor reviewed by Mahesh Kumar Behera) No new revisions were added by this update. Summary of changes: .../org/apache/hadoop/hive/ql/exec/vector/TestVectorLimitOperator.java | 2 ++ 1 file changed, 2 insertions(+)
[hive] branch master updated (a398000 -> b39e9c9)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from a398000 HIVE-25271: Fix TestRpc.testServerPort and TestRpc.testServerAddress - disabled tests, should be fixed in HIVE-25271 add b39e9c9 HIVE-25225 : Update column stat throws NPE if direct sql is disabled (Mahesh Kumar Behera, reviewed by Peter Vary) No new revisions were added by this update. Summary of changes: .../apache/hadoop/hive/metastore/HMSHandler.java | 44 +++- .../hive/metastore/client/TestPartitionStat.java | 237 + 2 files changed, 275 insertions(+), 6 deletions(-) create mode 100644 standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestPartitionStat.java
[hive] branch master updated (17e18c0 -> d0bbe76)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from 17e18c0 HIVE-25055: Improve the exception handling in HMSHandler
(Zhihua Deng reviewed by Vihang Karajgaonkar)
add d0bbe76 HIVE-25204 : Reduce overhead of adding notification log for
update partition column statistics. (Mahesh Kumar Behera, reviewed by Aasha
Medhi)
No new revisions were added by this update.
Summary of changes:
.../hcatalog/listener/DbNotificationListener.java | 195 ++---
.../hadoop/hive/metastore/DirectSqlUpdateStat.java | 21 ++-
.../hive/metastore/MetaStoreEventListener.java | 8 +-
.../hive/metastore/MetaStoreListenerNotifier.java | 5 +-
...va => UpdatePartitionColumnStatEventBatch.java} | 53 +++---
.../hive/metastore/messaging/EventMessage.java | 1 +
.../hive/metastore/messaging/MessageBuilder.java | 1 +
7 files changed, 136 insertions(+), 148 deletions(-)
copy
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/{OpenTxnEvent.java
=> UpdatePartitionColumnStatEventBatch.java} (53%)
[hive] branch master updated (32730b0 -> 0f3af0f)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 32730b0 HIVE-23987: Upgrade arrow version to 0.11.0 (Jesus Camacho Rodriguez, reviewed by Panos Garefalakis) add 0f3af0f HIVE-25150 : Tab characters are not removed before decimal conversion similar to space character which is fixed as part of HIVE-24378 (Taraka Rama Rao Lethavadla, reviewed by Mahesh Kumar Behera) No new revisions were added by this update. Summary of changes: data/files/test_sp_chars_decimal.csv | 8 ql/src/test/queries/clientpositive/test_decimal.q | 6 +++ .../results/clientpositive/llap/test_decimal.q.out | 40 + .../hive/common/type/FastHiveDecimalImpl.java | 52 +- 4 files changed, 95 insertions(+), 11 deletions(-) create mode 100644 data/files/test_sp_chars_decimal.csv create mode 100644 ql/src/test/queries/clientpositive/test_decimal.q create mode 100644 ql/src/test/results/clientpositive/llap/test_decimal.q.out
[hive] branch master updated (eedcd82 -> bef2e83)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from eedcd82 HIVE-25179: Support all partition transforms for Iceberg in create table (#2333) (Laszlo Pinter, reviewed by Adam Szita, Marton Bod and Peter Vary) add bef2e83 HIVE-24073 : Execution exception in sort-merge semijoin (Mahesh Kumar Behera, reviewed by Jesus Camacho Rodriguez) No new revisions were added by this update. Summary of changes: .../hive/ql/exec/CommonMergeJoinOperator.java | 18 +- .../hadoop/hive/ql/exec/tez/MapRecordSource.java | 10 + .../hadoop/hive/ql/exec/tez/RecordSource.java | 1 + .../hive/ql/exec/tez/ReduceRecordSource.java | 1 + .../hive/ql/optimizer/ConvertJoinMapJoin.java | 13 + .../clientpositive/auto_sortmerge_join_10.q| 31 +- .../llap/auto_sortmerge_join_10.q.out | 419 ++--- .../clientpositive/llap/groupby_sort_1_23.q.out| 1 + .../llap/groupby_sort_skew_1_23.q.out | 1 + .../clientpositive/llap/subquery_in_having.q.out | 6 +- 10 files changed, 444 insertions(+), 57 deletions(-)
[hive] branch master updated (8c512b2 -> 323d12f)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 8c512b2 HIVE-25117: Vector PTF ClassCastException with Decimal64 (#2286) add 323d12f HIVE-24663 : Reduce overhead of partition column stat updation during batch loading of partitions. (Mahesh Kumar Behera, reviewed by Denys Kuzmenko) No new revisions were added by this update. Summary of changes: .../hcatalog/listener/DbNotificationListener.java | 25 + .../hcatalog/listener/DummyRawStoreFailEvent.java | 9 + .../ql/parse/TestStatsReplicationScenarios.java| 17 +- .../ql/ddl/table/info/desc/DescTableOperation.java | 8 +- .../columnstats_partlvl_dp_lowercase.q | 10 + .../clientpositive/columnstats_partlvl_dp.q| 6 + .../clientpositive/dynpart_sort_optimization2.q| 2 +- .../extrapolate_part_stats_partial_ndv.q | 52 ++ ql/src/test/queries/clientpositive/orc_merge6.q| 36 ++ .../clientpositive/parquet_vectorization_part.q| 30 + .../test/queries/clientpositive/quotedid_alter.q | 11 +- .../columnstats_partlvl_dp_lowercase.q.out | 56 ++ .../llap/columnstats_partlvl_dp.q.out | 92 +++ .../llap/extrapolate_part_stats_partial_ndv.q.out | 476 ++ .../results/clientpositive/llap/orc_merge6.q.out | 630 ++ .../llap/parquet_vectorization_part.q.out | 270 .../clientpositive/llap/quotedid_alter.q.out | 165 + .../hadoop/hive/metastore/DirectSqlUpdateStat.java | 705 + .../apache/hadoop/hive/metastore/HMSHandler.java | 105 ++- .../hadoop/hive/metastore/MetaStoreDirectSql.java | 17 + .../hive/metastore/MetaStoreEventListener.java | 10 + .../hive/metastore/MetaStoreListenerNotifier.java | 4 + .../hive/metastore/MetastoreDirectSqlUtils.java| 2 +- .../apache/hadoop/hive/metastore/ObjectStore.java | 11 + .../org/apache/hadoop/hive/metastore/RawStore.java | 6 + .../hadoop/hive/metastore/StatObjectConverter.java | 94 +++ .../hadoop/hive/metastore/cache/CachedStore.java | 64 +- .../metastore/DummyRawStoreControlledCommit.java | 9 + .../metastore/DummyRawStoreForJdoConnection.java | 9 + .../metastore/InjectableBehaviourObjectStore.java | 25 + 30 files changed, 2896 insertions(+), 60 deletions(-) create mode 100644 ql/src/test/queries/clientnegative/columnstats_partlvl_dp_lowercase.q create mode 100644 ql/src/test/results/clientnegative/columnstats_partlvl_dp_lowercase.q.out create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdateStat.java
[hive] branch master updated (a4e273b -> 37f13b0)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from a4e273b HIVE-24730: Shims classes override values from hive-site.xml and tez-site.xml silently (#1941) (Laszlo Bodor reviewed by Peter Vary and Mustafa Iman) add 37f13b0 HIVE-24883 : Support ARRAY/STRUCT types in equality SMB and Common merge join (Mahesh Kumar Behera, reviewed by Stamatis Zampetakis) No new revisions were added by this update. Summary of changes: .../test/resources/testconfiguration.properties|1 + .../hive/ql/exec/CommonMergeJoinOperator.java | 29 +- .../hadoop/hive/ql/exec/HiveStructComparator.java | 64 + .../hive/ql/exec/HiveWritableComparator.java | 62 + .../hadoop/hive/ql/exec/SMBMapJoinOperator.java| 12 +- .../hive/ql/exec/WritableComparatorFactory.java| 41 + .../hive/ql/optimizer/AbstractSMBJoinProc.java |6 + .../DynamicPartitionPruningOptimization.java |5 + .../hive/ql/optimizer/physical/Vectorizer.java |4 +- .../hadoop/hive/ql/plan/CommonMergeJoinDesc.java |9 + .../apache/hadoop/hive/ql/plan/MapJoinDesc.java| 11 + .../clientnegative/test_merge_join_map_type.q |5 + .../clientnegative/test_merge_join_union_type.q|5 + .../clientpositive/smb_mapjoin_complex_type.q | 43 + .../clientpositive/test_join_complex_type.q| 81 ++ .../clientnegative/test_merge_join_map_type.q.out |9 + .../test_merge_join_union_type.q.out |9 + .../llap/test_join_complex_type.q.out | 1346 .../clientpositive/smb_mapjoin_complex_type.q.out | 232 19 files changed, 1941 insertions(+), 33 deletions(-) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/HiveStructComparator.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/HiveWritableComparator.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/exec/WritableComparatorFactory.java create mode 100644 ql/src/test/queries/clientnegative/test_merge_join_map_type.q create mode 100644 ql/src/test/queries/clientnegative/test_merge_join_union_type.q create mode 100644 ql/src/test/queries/clientpositive/smb_mapjoin_complex_type.q create mode 100644 ql/src/test/queries/clientpositive/test_join_complex_type.q create mode 100644 ql/src/test/results/clientnegative/test_merge_join_map_type.q.out create mode 100644 ql/src/test/results/clientnegative/test_merge_join_union_type.q.out create mode 100644 ql/src/test/results/clientpositive/llap/test_join_complex_type.q.out create mode 100644 ql/src/test/results/clientpositive/smb_mapjoin_complex_type.q.out
[hive] branch master updated (b2321c3 -> fa68362)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from b2321c3 HIVE-24627: Add Debug Logging to Hive JDBC Connection (David Mollitor reviewed by Miklos Gergely) add fa68362 HIVE-24589 : Drop catalog failing with deadlock error for Oracle backend dbms. (Mahesh Kumar Behera, reviewed by Miklos Gergely) No new revisions were added by this update. Summary of changes: .../metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql | 2 ++ .../src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql | 3 +++ 2 files changed, 5 insertions(+)
[hive] branch master updated (8f56425 -> fb046c7)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 8f56425 HIVE-24276: HiveServer2 loggerconf jsp Cross-Site Scripting (XSS) Vulnerability (Rajkumar Singh, reviewed by Chao Sun, Vihang Karajgaonkar) add fb046c7 HIVE-24503 : Optimize vector row serde by avoiding type check at run time. (Mahesh Kumar Behera, reviewed by Rajesh Balamohan, Panos Garefalakis) No new revisions were added by this update. Summary of changes: .../hive/ql/exec/vector/VectorDeserializeRow.java | 1373 +++- .../hive/ql/exec/vector/VectorSerializeRow.java| 480 --- 2 files changed, 1065 insertions(+), 788 deletions(-)
[hive] branch master updated (70f19ae -> c1f4f6a)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 70f19ae HIVE-24410: Query-based compaction hangs because of doAs (Karen Coppage, reviewed by Peter Varga and Peter Vary) add c1f4f6a HIVE-24409 : Use LazyBinarySerDe2 in PlanUtils::getReduceValueTableDesc (Mahesh Kumar Behera, reviewed by Rajesh Balamohan) No new revisions were added by this update. Summary of changes: .../apache/hadoop/hive/ql/exec/AppMasterEventOperator.java | 7 +++ .../hadoop/hive/ql/optimizer/physical/Vectorizer.java | 8 ++-- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java | 3 ++- .../hadoop/hive/serde2/lazybinary/LazyBinarySerDe2.java | 13 + 4 files changed, 28 insertions(+), 3 deletions(-)
[hive] branch master updated (0ea9d61 -> ea52e61)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 0ea9d61 HIVE-24341: Sweep phase for proactive cache eviction (Adam Szita reviewed by Antal Sinkovits, Peter Vary) add ea52e61 HIVE-24378 : Leading and trailing spaces are not removed before decimal conversion (Mahesh Kumar Behera, reviewed by Panagiotis Garefalakis) No new revisions were added by this update. Summary of changes: data/files/test_dec_space.csv | 2 ++ .../test/queries/clientpositive/whroot_external1.q | 7 +++ .../clientpositive/llap/whroot_external1.q.out | 22 ++ .../hadoop/hive/serde2/lazy/LazyHiveDecimal.java | 2 +- .../hive/serde2/lazydio/LazyDioHiveDecimal.java| 2 +- 5 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 data/files/test_dec_space.csv
[hive] branch master updated: HIVE-24373 : Wrong predicate is pushed down for view with constant value projection. (Mahesh Kumar Behera, reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 243155b HIVE-24373 : Wrong predicate is pushed down for view with
constant value projection. (Mahesh Kumar Behera, reviewed by Jesus Camacho
Rodriguez)
243155b is described below
commit 243155b0db3a17ce12ec90697c8195b09fb2879c
Author: mahesh kumar behera
AuthorDate: Fri Nov 13 16:52:19 2020 +0530
HIVE-24373 : Wrong predicate is pushed down for view with constant value
projection. (Mahesh Kumar Behera, reviewed by Jesus Camacho Rodriguez)
---
.../apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java | 4 +-
ql/src/test/queries/clientpositive/ppd2.q | 38 +
ql/src/test/results/clientpositive/llap/ppd2.q.out | 154 +
3 files changed, 195 insertions(+), 1 deletion(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java
b/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java
index b4241c1..7926a03 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java
@@ -250,7 +250,9 @@ public class ExprWalkerInfo implements NodeProcessorCtx {
for (Map.Entry> entry :
nonFinalPreds.entrySet()) {
List converted = new ArrayList();
for (ExprNodeDesc newExpr : entry.getValue()) {
-converted.add(newToOldExprMap.get(newExpr));
+// We should clone it to avoid getting overwritten if two or more
operator uses
+// this same expression.
+converted.add(newToOldExprMap.get(newExpr).clone());
}
oldExprs.put(entry.getKey(), converted);
}
diff --git a/ql/src/test/queries/clientpositive/ppd2.q
b/ql/src/test/queries/clientpositive/ppd2.q
index 39e3170..2262331 100644
--- a/ql/src/test/queries/clientpositive/ppd2.q
+++ b/ql/src/test/queries/clientpositive/ppd2.q
@@ -77,3 +77,41 @@ from (
distribute by a.key
sort by a.key,a.cc desc) b
where b.cc>1;
+
+
+set hive.explain.user=false;
+set hive.cbo.enable=false;
+set hive.optimize.ppd=true;
+set hive.ppd.remove.duplicatefilters=true;
+
+DROP TABLE arc;
+CREATE table arc(`dt_from` string, `dt_to` string);
+
+DROP TABLE loc1;
+CREATE table loc1(`dt_from` string, `dt_to` string);
+
+-- INSERT INTO arc VALUES('2020', '2020');
+-- INSERT INTO loc1 VALUES('2020', '2020');
+
+DROP VIEW view;
+CREATE
+ VIEW view AS
+ SELECT
+'' as DT_FROM,
+uuid() as DT_TO
+ FROM
+ loc1
+ UNION ALL
+ SELECT
+dt_from as DT_FROM,
+uuid() as DT_TO
+ FROM
+ arc;
+
+EXPLAIN
+SELECT
+ dt_from, dt_to
+FROM
+ view
+WHERE
+ '2020' between dt_from and dt_to;
diff --git a/ql/src/test/results/clientpositive/llap/ppd2.q.out
b/ql/src/test/results/clientpositive/llap/ppd2.q.out
index 184c580..b067ae6 100644
--- a/ql/src/test/results/clientpositive/llap/ppd2.q.out
+++ b/ql/src/test/results/clientpositive/llap/ppd2.q.out
@@ -691,3 +691,157 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
95 2
97 2
98 2
+PREHOOK: query: DROP TABLE arc
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE arc
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE table arc(`dt_from` string, `dt_to` string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@arc
+POSTHOOK: query: CREATE table arc(`dt_from` string, `dt_to` string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@arc
+PREHOOK: query: DROP TABLE loc1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE loc1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE table loc1(`dt_from` string, `dt_to` string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@loc1
+POSTHOOK: query: CREATE table loc1(`dt_from` string, `dt_to` string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@loc1
+PREHOOK: query: DROP VIEW view
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW view
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: CREATE
+ VIEW view AS
+ SELECT
+'' as DT_FROM,
+uuid() as DT_TO
+ FROM
+ loc1
+ UNION ALL
+ SELECT
+dt_from as DT_FROM,
+uuid() as DT_TO
+ FROM
+ arc
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@arc
+PREHOOK: Input: default@loc1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@view
+POSTHOOK: query: CREATE
+ VIEW view AS
+ SELECT
+'' as DT_FROM,
+uuid() as DT_TO
+ FROM
+ loc1
+ UNION ALL
+ SELECT
+dt_from as DT_FROM,
+uuid() as DT_TO
+ FROM
+ arc
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@arc
+POSTHO
[hive] branch master updated (2e2fb6c -> 214134e)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 2e2fb6c disable TestLlapExtClientWithCloudDeploymentConfigs add 214134e HIVE-24362 : AST tree processing is suboptimal for tree with large number of nodes (Mahesh Kumar Behera, reviewed by Panagiotis Garefalakis) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[hive] branch master updated (2e2fb6c -> 214134e)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 2e2fb6c disable TestLlapExtClientWithCloudDeploymentConfigs add 214134e HIVE-24362 : AST tree processing is suboptimal for tree with large number of nodes (Mahesh Kumar Behera, reviewed by Panagiotis Garefalakis) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[hive] branch master updated (077952f -> 1053b31)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 077952f HIVE-21052: Make sure transactions get cleaned if they are aborted before addPartitions is called (Denys Kuzmenko, reviewed by Vipin Vishvkarma, Peter Varga, Karen Coppage) add 1053b31 HIVE-24284 : NPE when parsing druid logs using Hive (#1586) No new revisions were added by this update. Summary of changes: .../apache/hadoop/hive/serde2/TestSysLogSerDe.java | 120 + .../hadoop/hive/ql/log/syslog/SyslogParser.java| 8 +- 2 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 itests/hive-unit/src/test/java/org/apache/hadoop/hive/serde2/TestSysLogSerDe.java
[hive] branch master updated: HIVE-24198 : Map side SMB join is producing wrong result (#1524)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new eba8349 HIVE-24198 : Map side SMB join is producing wrong result
(#1524)
eba8349 is described below
commit eba83493f170f4c38868481a4df38b56b66cef7d
Author: mahesh kumar behera
AuthorDate: Sat Sep 26 16:51:40 2020 +0530
HIVE-24198 : Map side SMB join is producing wrong result (#1524)
Reviewed by Ashutosh Chauhan
---
.../apache/hadoop/hive/ql/exec/tez/DagUtils.java | 19 +++-
.../hadoop/hive/ql/exec/tez/SplitGrouper.java | 12 ++-
.../apache/hadoop/hive/ql/parse/TezCompiler.java | 27 +++--
.../clientpositive/auto_sortmerge_join_10.q| 17 +++
.../llap/auto_sortmerge_join_10.q.out | 119 +
5 files changed, 181 insertions(+), 13 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 43ab9e6..877a897 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -758,7 +758,14 @@ public class DagUtils {
Vertex mergeVx = createVertexFromMapWork(
conf, mapWork, mrScratchDir, vertexType);
- conf.setClass("mapred.input.format.class", HiveInputFormat.class,
InputFormat.class);
+ Class inputFormatClass = conf.getClass("mapred.input.format.class",
+ HiveInputFormat.class);
+ if (inputFormatClass != BucketizedHiveInputFormat.class &&
+ inputFormatClass != HiveInputFormat.class) {
+// As of now only these two formats are supported.
+inputFormatClass = HiveInputFormat.class;
+ }
+ conf.setClass("mapred.input.format.class", inputFormatClass,
InputFormat.class);
// mapreduce.tez.input.initializer.serialize.event.payload should be set
// to false when using this plug-in to avoid getting a serialized event
at run-time.
conf.setBoolean("mapreduce.tez.input.initializer.serialize.event.payload",
false);
@@ -769,7 +776,7 @@ public class DagUtils {
conf.set(Utilities.INPUT_NAME, mapWork.getName());
LOG.info("Going through each work and adding MultiMRInput");
mergeVx.addDataSource(mapWork.getName(),
-MultiMRInput.createConfigBuilder(conf,
HiveInputFormat.class).build());
+MultiMRInput.createConfigBuilder(conf, inputFormatClass).build());
}
// To be populated for SMB joins only for all the small tables
@@ -835,8 +842,12 @@ public class DagUtils {
groupSplitsInInputInitializer = false;
// grouping happens in execution phase. The input payload should not
enable grouping here,
// it will be enabled in the CustomVertex.
- inputFormatClass = HiveInputFormat.class;
- conf.setClass("mapred.input.format.class", HiveInputFormat.class,
InputFormat.class);
+ if (inputFormatClass != BucketizedHiveInputFormat.class &&
+ inputFormatClass != HiveInputFormat.class) {
+// As of now only these two formats are supported.
+inputFormatClass = HiveInputFormat.class;
+ }
+ conf.setClass("mapred.input.format.class", inputFormatClass,
InputFormat.class);
// mapreduce.tez.input.initializer.serialize.event.payload should be set
to false when using
// this plug-in to avoid getting a serialized event at run-time.
conf.setBoolean("mapreduce.tez.input.initializer.serialize.event.payload",
false);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java
index 2295edc..6b8aec1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java
@@ -32,6 +32,8 @@ import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -94,11 +96,17 @@ public class SplitGrouper {
// use the tez grouper to combine splits once per bucket
for (int bucketId : bucketSplitMultimap.keySet()) {
Collection inputSplitCollection =
bucketSplitMultimap.get(bucketId);
-
+ Class inputFormatClass = conf.getClass("mapred.input.format.class",
+ HiveInputFormat.class);
+ if (inputFormatClass != BucketizedHiveInputFormat.class &&
+ inputFormatClass != HiveInputFormat.class) {
+// As of now only these tw
[hive] branch master updated: HIVE-23953 : Use task counter information to compute keycount during hashtable loading (#1340)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ee186af HIVE-23953 : Use task counter information to compute keycount
during hashtable loading (#1340)
ee186af is described below
commit ee186af06e552bdb9790a783c3b0ff65953171e6
Author: mahesh kumar behera
AuthorDate: Tue Aug 4 16:59:39 2020 +0530
HIVE-23953 : Use task counter information to compute keycount during
hashtable loading (#1340)
HIVE-23953 : Use task counter information to compute keycount during
hashtable loading (Mahesh Kumar Behera, reviewed by Rajesh Balamohan)
---
.../hadoop/hive/ql/exec/tez/HashTableLoader.java | 29 --
.../fast/VectorMapJoinFastHashTableLoader.java | 28 +++--
2 files changed, 43 insertions(+), 14 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
index 9cb..fcb2cc5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.tez.runtime.api.Input;
import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.apache.tez.runtime.api.AbstractLogicalInput;
/**
* HashTableLoader for Tez constructs the hashtable from records read from
@@ -219,7 +220,19 @@ public class HashTableLoader implements
org.apache.hadoop.hive.ql.exec.HashTable
}
isFirstKey = false;
Long keyCountObj = parentKeyCounts.get(pos);
-long keyCount = (keyCountObj == null) ? -1 : keyCountObj.longValue();
+long estKeyCount = (keyCountObj == null) ? -1 : keyCountObj;
+
+long inputRecords = -1;
+try {
+ //TODO : Need to use class instead of string.
+ // https://issues.apache.org/jira/browse/HIVE-23981
+ inputRecords = ((AbstractLogicalInput)
input).getContext().getCounters().
+ findCounter("org.apache.tez.common.counters.TaskCounter",
+ "APPROXIMATE_INPUT_RECORDS").getValue();
+} catch (Exception e) {
+ LOG.debug("Failed to get value for counter
APPROXIMATE_INPUT_RECORDS", e);
+}
+long keyCount = Math.max(estKeyCount, inputRecords);
long memory = 0;
if (useHybridGraceHashJoin) {
@@ -242,8 +255,9 @@ public class HashTableLoader implements
org.apache.hadoop.hive.ql.exec.HashTable
tableContainer = new HashMapWrapper(hconf, keyCount);
}
-LOG.info("Loading hash table for input: {} cacheKey: {}
tableContainer: {} smallTablePos: {}", inputName,
- cacheKey, tableContainer.getClass().getSimpleName(), pos);
+LOG.info("Loading hash table for input: {} cacheKey: {}
tableContainer: {} smallTablePos: {} " +
+"estKeyCount : {} keyCount : {}", inputName, cacheKey,
+tableContainer.getClass().getSimpleName(), pos, estKeyCount,
keyCount);
tableContainer.setSerde(keyCtx, valCtx);
long startTime = System.currentTimeMillis();
@@ -272,11 +286,12 @@ public class HashTableLoader implements
org.apache.hadoop.hive.ql.exec.HashTable
tableContainer.seal();
mapJoinTables[pos] = tableContainer;
if (doMemCheck) {
- LOG.info("Finished loading hash table for input: {} cacheKey: {}
numEntries: {} estimatedMemoryUsage: {}",
-inputName, cacheKey, numEntries,
tableContainer.getEstimatedMemorySize());
+ LOG.info("Finished loading hash table for input: {} cacheKey: {}
numEntries: {} " +
+ "estimatedMemoryUsage: {} Load Time : {} ",
+inputName, cacheKey, numEntries,
tableContainer.getEstimatedMemorySize(), delta);
} else {
- LOG.info("Finished loading hash table for input: {} cacheKey: {}
numEntries: {}", inputName, cacheKey,
-numEntries);
+ LOG.info("Finished loading hash table for input: {} cacheKey: {}
numEntries: {} Load Time : {} ",
+ inputName, cacheKey, numEntries, delta);
}
} catch (Exception e) {
throw new HiveException(e);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
index a011027..0eba50c 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
+++
b/ql/src/java/org
[hive] branch master updated (abb5d7e -> 3e5e99e)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from abb5d7e HIVE-23892 HIVE-23750: addendum
add 3e5e99e HIVE-23897 : Create a common Retry Interface for replication
(#1329)
No new revisions were added by this update.
Summary of changes:
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 16 +
.../hadoop/hive/ql/exec/repl/AtlasDumpTask.java| 2 +-
.../hadoop/hive/ql/exec/repl/RangerDumpTask.java | 4 +-
.../hadoop/hive/ql/exec/repl/RangerLoadTask.java | 4 +-
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 19 +-
.../ql/exec/repl/atlas/AtlasRestClientBuilder.java | 3 +-
.../ql/exec/repl/atlas/AtlasRestClientImpl.java| 11 +-
...ingClient.java => RetryingClientTimeBased.java} | 52 ++-
.../ql/exec/repl/ranger/NoOpRangerRestClient.java | 8 +-
.../hive/ql/exec/repl/ranger/RangerRestClient.java | 8 +-
.../ql/exec/repl/ranger/RangerRestClientImpl.java | 84 ++---
.../apache/hadoop/hive/ql/exec/util/Retryable.java | 208
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 56 ++--
.../hive/ql/parse/repl/dump/io/FileOperations.java | 61 ++--
.../hive/ql/parse/repl/metric/MetricSink.java | 23 +-
.../hive/ql/exec/repl/TestRangerDumpTask.java | 11 +-
.../hive/ql/exec/repl/TestRangerLoadTask.java | 8 +-
.../ql/exec/repl/ranger/TestRangerRestClient.java | 65 ++--
.../hadoop/hive/ql/exec/util/TestRetryable.java| 352 +
19 files changed, 784 insertions(+), 211 deletions(-)
rename
ql/src/java/org/apache/hadoop/hive/ql/exec/repl/atlas/{RetryingClient.java =>
RetryingClientTimeBased.java} (68%)
create mode 100644
ql/src/java/org/apache/hadoop/hive/ql/exec/util/Retryable.java
create mode 100644
ql/src/test/org/apache/hadoop/hive/ql/exec/util/TestRetryable.java
[hive] branch master updated: HIVE-23894 - SubmitDag should not be retried incase of query cancel (#1293)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ee27dd3 HIVE-23894 - SubmitDag should not be retried incase of query
cancel (#1293)
ee27dd3 is described below
commit ee27dd3f61db407c249716b4d6fd29a5c0c6539e
Author: Naresh P R
AuthorDate: Thu Jul 30 08:26:16 2020 -0700
HIVE-23894 - SubmitDag should not be retried incase of query cancel (#1293)
---
.../apache/hadoop/hive/ql/exec/tez/TezTask.java| 22 --
1 file changed, 16 insertions(+), 6 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 3d27632..b086fc0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -578,6 +578,11 @@ public class TezTask extends Task {
dagClient = sessionState.getSession().submitDAG(dag);
}
} catch (Exception e) {
+ if (this.isShutdown) {
+// Incase of taskShutdown, no need to retry
+sessionDestroyOrReturnToPool(sessionStateRef, sessionState);
+throw e;
+ }
// In case of any other exception, retry. If this also fails, report
original error and exit.
try {
console.printInfo("Dag submit failed due to " + e.getMessage() + "
stack trace: "
@@ -588,12 +593,7 @@ public class TezTask extends Task {
// we failed to submit after retrying.
// If this is a non-pool session, destroy it.
// Otherwise move it to sessionPool, reopen will retry.
-sessionStateRef.value = null;
-if (sessionState.isDefault() && sessionState instanceof
TezSessionPoolSession) {
- sessionState.returnToSessionManager();
-} else {
- sessionState.destroy();
-}
+sessionDestroyOrReturnToPool(sessionStateRef, sessionState);
throw retryException;
}
}
@@ -602,6 +602,16 @@ public class TezTask extends Task {
return new SyncDagClient(dagClient);
}
+ private void sessionDestroyOrReturnToPool(Ref
sessionStateRef,
+ TezSessionState sessionState) throws Exception{
+sessionStateRef.value = null;
+if (sessionState.isDefault() && sessionState instanceof
TezSessionPoolSession) {
+ sessionState.returnToSessionManager();
+} else {
+ sessionState.destroy();
+}
+ }
+
/*
* close will move the temp files into the right place for the fetch
* task. If the job has failed it will clean up the files.
[hive] branch master updated: HIVE-23361 : Optimising privilege synchroniser. (Simhadri G, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0cde39a HIVE-23361 : Optimising privilege synchroniser. (Simhadri G,
reviewed by Mahesh Kumar Behera)
0cde39a is described below
commit 0cde39a7ddb4f8049e3b43724d929da25b045112
Author: Simhadri G
AuthorDate: Thu May 28 12:22:12 2020 +0530
HIVE-23361 : Optimising privilege synchroniser. (Simhadri G, reviewed by
Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../InformationSchemaWithPrivilegeTestBase.java| 2 +-
.../upgrade/hive/hive-schema-4.0.0.hive.sql| 16 +-
.../upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql | 195 +
.../hadoop/hive/ql/exec/FunctionRegistry.java | 1 +
.../authorization/PrivilegeSynchronizer.java | 29 ++-
.../udf/generic/GenericUDFStringToPrivilege.java | 101 +++
.../hive/ql/udf/generic/TestUDFSplitMapPrivs.java | 132 ++
.../test/queries/clientpositive/split_map_privs.q | 17 ++
.../clientpositive/llap/show_functions.q.out | 2 +
.../clientpositive/llap/split_map_privs.q.out | 66 +++
10 files changed, 553 insertions(+), 8 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java
b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java
index 7302e09..cebfdff 100644
---
a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java
+++
b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java
@@ -286,7 +286,7 @@ public abstract class
InformationSchemaWithPrivilegeTestBase {
List args = new ArrayList(baseArgs);
args.add("-f");
-
args.add("../../metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql");
+
args.add("../../metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql");
BeeLine beeLine = new BeeLine();
int result = beeLine.begin(args.toArray(new String[] {}), null);
beeLine.close();
diff --git a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql
b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql
index d857410..cc9aeef 100644
--- a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql
+++ b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql
@@ -1677,7 +1677,8 @@ WHERE
AND C.`COLUMN_NAME` = P.`COLUMN_NAME`
AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER'
OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR
P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))
- AND P.`TBL_COL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer();
+ AND array_contains(split_map_privs(P.`TBL_COL_PRIV`),"SELECT") AND
P.`AUTHORIZER`=current_authorizer();
+
CREATE OR REPLACE VIEW `COLUMN_PRIVILEGES`
(
@@ -1700,7 +1701,18 @@ SELECT DISTINCT
P.`TBL_COL_PRIV`,
IF (P.`GRANT_OPTION` == 0, 'NO', 'YES')
FROM
- `sys`.`TBL_COL_PRIVS` P JOIN `sys`.`TBLS` T ON (P.`TBL_ID` = T.`TBL_ID`)
+ (SELECT
+Q.`GRANTOR`,
+Q.`GRANT_OPTION`,
+Q.`PRINCIPAL_NAME`,
+Q.`PRINCIPAL_TYPE`,
+Q.`AUTHORIZER`,
+Q.`COLUMN_NAME`,
+`TBL_COL_PRIV_TMP`.`TBL_COL_PRIV`,
+Q.`TBL_ID`
+ FROM `sys`.`TBL_COL_PRIVS` AS Q
+ LATERAL VIEW explode(split_map_privs(Q.`TBL_COL_PRIV`))
`TBL_COL_PRIV_TMP` AS `TBL_COL_PRIV`) P
+ JOIN `sys`.`TBLS` T ON (P.`TBL_ID` = T.`TBL_ID`)
JOIN `sys`.`DBS` D ON (T.`DB_ID` = D.`DB_ID`)
JOIN `sys`.`SDS` S ON (S.`SD_ID` = T.`SD_ID`)
LEFT JOIN `sys`.`TBL_PRIVS` P2 ON (P.`TBL_ID` =
P2.`TBL_ID`)
diff --git a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql
b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql
index 0523e25..09c95d7 100644
--- a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql
+++ b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql
@@ -497,6 +497,201 @@ CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`,
'4.0.0' AS `SCHEMA_VER
USE INFORMATION_SCHEMA;
+
+CREATE OR REPLACE VIEW `COLUMNS`
+(
+ `TABLE_CATALOG`,
+ `TABLE_SCHEMA`,
+ `TABLE_NAME`,
+ `COLUMN_NAME`,
+ `ORDINAL_POSITION`,
+ `COLUMN_DEFAULT`,
+ `IS_NULLABLE`,
+ `DATA_TYPE`,
+ `CHARACTER_MAXIMUM_LENGTH`,
+ `CHARACTER_OCTET_LENGTH`,
+ `NUMERIC_PRECISION`,
+ `NUMERIC_PRECISION_RADIX`,
+ `NUMERIC_SCALE`,
+ `DATETIME_PRECISION`,
+ `INTERVAL_TYPE`,
+ `INTERVAL_PRECISION`,
+ `CHARACTER_SET_CATALOG`,
+ `CHARACTER_SET_SCHEMA`,
+ `CHARACTER_SET_NAME`,
+ `C
[hive] branch master updated: HIVE-23173 : User login success/failed attempts should be logged. ( Naresh P R reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 53f6530 HIVE-23173 : User login success/failed attempts should be
logged. ( Naresh P R reviewed by Mahesh Kumar Behera)
53f6530 is described below
commit 53f65309520dba9057fa835edaa207b8055dbd49
Author: Naresh P R
AuthorDate: Tue Apr 14 09:20:45 2020 +0530
HIVE-23173 : User login success/failed attempts should be logged. ( Naresh
P R reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../parse/TestScheduledReplicationScenarios.java | 58 +++---
.../java/org/apache/hive/jdbc/HiveConnection.java | 6 ++-
.../hive/jdbc/HttpRequestInterceptorBase.java | 7 ++-
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 10 ++--
.../apache/hive/service/auth/PlainSaslHelper.java | 7 ++-
.../hive/service/cli/thrift/ThriftCLIService.java | 9 ++--
.../hive/service/cli/thrift/ThriftHttpServlet.java | 16 ++
7 files changed, 73 insertions(+), 40 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java
index 692d40d..4a64927 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java
@@ -104,12 +104,12 @@ public class TestScheduledReplicationScenarios extends
BaseReplicationScenariosA
ScheduledQueryExecutionService.startScheduledQueryExecutorService(primary.hiveConf))
{
int next = 0;
ReplDumpWork.injectNextDumpDirForTest(String.valueOf(next));
- primary.run("create scheduled query s1 every 10 minutes as repl dump " +
primaryDbName);
- primary.run("alter scheduled query s1 execute");
+ primary.run("create scheduled query s1_t1 every 10 minutes as repl dump
" + primaryDbName);
+ primary.run("alter scheduled query s1_t1 execute");
Thread.sleep(6000);
- replica.run("create scheduled query s2 every 10 minutes as repl load " +
primaryDbName + " INTO "
+ replica.run("create scheduled query s2_t1 every 10 minutes as repl load
" + primaryDbName + " INTO "
+ replicatedDbName);
- replica.run("alter scheduled query s2 execute");
+ replica.run("alter scheduled query s2_t1 execute");
Thread.sleep(2);
replica.run("use " + replicatedDbName)
.run("show tables like 't1'")
@@ -123,9 +123,9 @@ public class TestScheduledReplicationScenarios extends
BaseReplicationScenariosA
.run("insert into t1 values(4)");
next++;
ReplDumpWork.injectNextDumpDirForTest(String.valueOf(next));
- primary.run("alter scheduled query s1 execute");
+ primary.run("alter scheduled query s1_t1 execute");
Thread.sleep(2);
- replica.run("alter scheduled query s2 execute");
+ replica.run("alter scheduled query s2_t1 execute");
Thread.sleep(2);
replica.run("use " + replicatedDbName)
.run("show tables like 't1'")
@@ -139,9 +139,9 @@ public class TestScheduledReplicationScenarios extends
BaseReplicationScenariosA
.run("insert into t1 values(6)");
next++;
ReplDumpWork.injectNextDumpDirForTest(String.valueOf(next));
- primary.run("alter scheduled query s1 execute");
+ primary.run("alter scheduled query s1_t1 execute");
Thread.sleep(3);
- replica.run("alter scheduled query s2 execute");
+ replica.run("alter scheduled query s2_t1 execute");
Thread.sleep(3);
replica.run("use " + replicatedDbName)
.run("show tables like 't1'")
@@ -152,8 +152,8 @@ public class TestScheduledReplicationScenarios extends
BaseReplicationScenariosA
} finally {
- primary.run("drop scheduled query s1");
- replica.run("drop scheduled query s2");
+ primary.run("drop scheduled query s1_t1");
+ replica.run("drop scheduled query s2_t1");
}
}
@@ -163,46 +163,46 @@ public class TestScheduledReplicationScenarios extends
BaseReplicationScenariosA
String withClause = " WITH('" +
HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname
+ "'='/replica_external_base')";
primary.
[hive] branch master updated: HIVE-23034 : Arrow serializer should not keep the reference of arrow offset and validity buffers. (Shubham Chaurasia, reviewed by Thejas Nair)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6f9ae63 HIVE-23034 : Arrow serializer should not keep the reference
of arrow offset and validity buffers. (Shubham Chaurasia, reviewed by Thejas
Nair)
6f9ae63 is described below
commit 6f9ae63e78eb0d529f3cd96f51f93247783265bc
Author: Shubham Chaurasia
AuthorDate: Thu Mar 19 08:08:53 2020 +0530
HIVE-23034 : Arrow serializer should not keep the reference of arrow offset
and validity buffers. (Shubham Chaurasia, reviewed by Thejas Nair)
Signed-off-by: Mahesh Kumar Behera
---
.../org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java | 47 ++
...iniLlap.java => BaseJdbcWithMiniLlap.java.orig} | 0
.../apache/hadoop/hive/ql/io/arrow/Serializer.java | 17
3 files changed, 54 insertions(+), 10 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
index 4c46db9..3973ec9 100644
---
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
+++
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
@@ -360,6 +360,53 @@ public abstract class BaseJdbcWithMiniLlap {
assertEquals(0, rowCount);
}
+ @Test(timeout = 30)
+ public void testInvalidReferenceCountScenario() throws Exception {
+final String tableName = "testInvalidReferenceCountScenario";
+try (Statement stmt = hs2Conn.createStatement()) {
+ String createQuery =
+ "create table " + tableName +
+ "(arr1
array>>>,
"
+ + "c2 int) STORED AS ORC";
+
+ // create table
+ stmt.execute("DROP TABLE IF EXISTS " + tableName);
+ stmt.execute(createQuery);
+ // load data
+ stmt.execute("INSERT INTO " + tableName + " VALUES "
+ // value 1
+ + "(ARRAY(NAMED_STRUCT('f1','a1', "
+ + "'f2','a2', "
+ + "'arr2',"
+ + " ARRAY("
+ + "NAMED_STRUCT('f3', cast(null as string), 'f4', cast(null as
string), 'f5', cast(null as string, "
+ + "NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', "
+ + "ARRAY(NAMED_STRUCT('f3', 'fielddd333', 'f4', 'field4',
'f5', 'field5'))), "
+ + "NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2',
ARRAY(NAMED_STRUCT('f3', cast(null as string), "
+ + "'f4', cast(null as string), 'f5', cast(null as string, "
+ + "NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2',
ARRAY(NAMED_STRUCT('f3', 'fielddd333', "
+ + "'f4', 'field4', 'f5', 'field5'))), NAMED_STRUCT('f1','a1',
'f2','a2', 'arr2', "
+ + "ARRAY(NAMED_STRUCT('f3', cast(null as string), 'f4', cast(null as
string), 'f5', cast(null as string,"
+ + " NAMED_STRUCT('f1','a1', 'f2','a2', 'arr2', "
+ + "ARRAY(NAMED_STRUCT('f3', 'fielddd333', 'f4', 'field4',
'f5', 'field5', 1)");
+
+ // generate 16384 rows from above records
+ for (int i = 0; i < 14; i++) {
+stmt.execute(String.format("insert into %s select * from %s",
tableName, tableName));
+ }
+ // validate test table
+ ResultSet res = stmt.executeQuery("SELECT count(*) FROM " + tableName);
+ assertTrue(res.next());
+ assertEquals(16384, res.getInt(1));
+ res.close();
+}
+// should not throw - IllegalReferenceCountException: refCnt: 0
+RowCollector rowCollector = new RowCollector();
+String query = "select * from " + tableName;
+int rowCount = processQuery(query, 1, rowCollector);
+assertEquals(16384, rowCount);
+
+ }
@Test(timeout = 6)
public void testNonAsciiStrings() throws Exception {
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java.orig
similarity index 100%
copy from
itests/hive-unit/src/test/java/org/apache/
[hive] branch master updated: HIVE-22990 : Build acknowledgement mechanism for repl dump and load. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 7256978 HIVE-22990 : Build acknowledgement mechanism for repl dump
and load. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
7256978 is described below
commit 7256978b85fa873249e0f292c868fdf46e01dab1
Author: Aasha Medhi
AuthorDate: Wed Mar 18 12:51:18 2020 +0530
HIVE-22990 : Build acknowledgement mechanism for repl dump and load. (Aasha
Medhi, reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../hive/ql/parse/TestReplicationScenarios.java| 136 ++---
...estReplicationScenariosAcidTablesBootstrap.java | 45 ++-
.../TestReplicationScenariosAcrossInstances.java | 91 --
.../TestReplicationScenariosExternalTables.java| 64 +-
.../parse/TestReplicationWithTableMigrationEx.java | 6 +-
.../apache/hadoop/hive/ql/plan/api/StageType.java | 3 +-
.../apache/hadoop/hive/ql/exec/TaskFactory.java| 7 +-
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 88 +
.../hive/ql/exec/repl/ReplLoadCompleteAckTask.java | 69 +++
.../hive/ql/exec/repl/ReplLoadCompleteAckWork.java | 44 +++
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 104 +++-
.../hadoop/hive/ql/exec/repl/ReplLoadWork.java | 7 --
.../hadoop/hive/ql/exec/repl/util/ReplUtils.java | 9 +-
.../hive/ql/parse/ReplicationSemanticAnalyzer.java | 46 ++-
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 18 +++
15 files changed, 401 insertions(+), 336 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index c46103a..49027a3 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -312,7 +312,12 @@ public class TestReplicationScenarios {
verifySetup("SELECT * from " + dbName + ".unptned_empty", empty, driver);
String replicatedDbName = dbName + "_dupe";
-bootstrapLoadAndVerify(dbName, replicatedDbName);
+Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replicatedDbName);
+
+FileSystem fs = new Path(bootstrapDump.dumpLocation).getFileSystem(hconf);
+Path dumpPath = new Path(bootstrapDump.dumpLocation,
ReplUtils.REPL_HIVE_BASE_DIR);
+assertTrue(fs.exists(new Path(dumpPath, ReplUtils.DUMP_ACKNOWLEDGEMENT)));
+assertTrue(fs.exists(new Path(dumpPath, ReplUtils.LOAD_ACKNOWLEDGEMENT)));
verifyRun("SELECT * from " + replicatedDbName + ".unptned", unptn_data,
driverMirror);
verifyRun("SELECT a from " + replicatedDbName + ".ptned WHERE b=1",
ptn_data_1, driverMirror);
@@ -395,6 +400,9 @@ public class TestReplicationScenarios {
assertEquals(false, hasMoveTask(task));
assertEquals(true, hasPartitionTask(task));
+Path loadPath = new Path(dump.dumpLocation, ReplUtils.REPL_HIVE_BASE_DIR);
+//delete load ack to reload the same dump
+loadPath.getFileSystem(hconf).delete(new Path(loadPath,
ReplUtils.LOAD_ACKNOWLEDGEMENT), true);
loadAndVerify(dbNameReplica, dbName, dump.lastReplId);
run("insert into table " + dbName + ".t2 partition(country='india') values
('delhi')", driver);
@@ -406,6 +414,9 @@ public class TestReplicationScenarios {
assertEquals(true, hasMoveTask(task));
assertEquals(true, hasPartitionTask(task));
+loadPath = new Path(dump.dumpLocation, ReplUtils.REPL_HIVE_BASE_DIR);
+//delete load ack to reload the same dump
+loadPath.getFileSystem(hconf).delete(new Path(loadPath,
ReplUtils.LOAD_ACKNOWLEDGEMENT), true);
loadAndVerify(dbNameReplica, dbName, dump.lastReplId);
run("insert into table " + dbName + ".t2 partition(country='us') values
('sf')", driver);
@@ -488,7 +499,7 @@ public class TestReplicationScenarios {
// Load to an empty database
Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, dbName + "_empty");
-String replDumpLocn = bootstrapDump.dumpLocation;
+
verifyRun("SELECT * from " + dbName + "_empty.unptned", unptn_data,
driverMirror);
String[] nullReplId = new String[]{ "NULL" };
@@ -496,8 +507,8 @@ public class TestReplicationScenarios {
// Create a database with a table
createDB(dbName + "_withtable", driverMirror);
run("CREATE TABLE " + dbName + "_withtable.unptned(a string) STORED AS
TEXTFILE", driverMirror);
-// Load using same dump t
[hive] branch master updated: HIVE-23022 : Arrow deserializer should ensure size of hive vector equal to arrow vector. (Shubham Chaurasia, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4b08c21 HIVE-23022 : Arrow deserializer should ensure size of hive
vector equal to arrow vector. (Shubham Chaurasia, reviewed by Mahesh Kumar
Behera)
4b08c21 is described below
commit 4b08c21aee1aa37dff5cd9b51f471f40c4a910ea
Author: Shubham Chaurasia
AuthorDate: Wed Mar 18 11:46:46 2020 +0530
HIVE-23022 : Arrow deserializer should ensure size of hive vector equal to
arrow vector. (Shubham Chaurasia, reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java | 110 +
.../hadoop/hive/ql/io/arrow/Deserializer.java | 7 +-
2 files changed, 115 insertions(+), 2 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
index 3c0532c..4c46db9 100644
---
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
+++
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java
@@ -224,6 +224,116 @@ public abstract class BaseJdbcWithMiniLlap {
}
@Test(timeout = 30)
+ public void testMultipleBatchesOfComplexTypes() throws Exception {
+final String tableName = "testMultipleBatchesOfComplexTypes";
+try (Statement stmt = hs2Conn.createStatement()) {
+ String createQuery =
+ "create table " + tableName + "(c1
array>, "
+ + "c2 int, "
+ + "c3 array>, "
+ + "c4 array>>) STORED AS ORC";
+
+ // create table
+ stmt.execute("DROP TABLE IF EXISTS " + tableName);
+ stmt.execute(createQuery);
+ // load data
+ stmt.execute("INSERT INTO " + tableName + " VALUES "
+ // value 1
+ + "(ARRAY(NAMED_STRUCT('f1','a1', 'f2','a2'),
NAMED_STRUCT('f1','a3', 'f2','a4')), "
+ + "1, ARRAY(ARRAY(1)), ARRAY(NAMED_STRUCT('f1',ARRAY('aa1', "
+ // value 2
+ + "(ARRAY(NAMED_STRUCT('f1','b1', 'f2','b2'),
NAMED_STRUCT('f1','b3', 'f2','b4')), 2, "
+ + "ARRAY(ARRAY(2,2), ARRAY(2,2)), "
+ + "ARRAY(NAMED_STRUCT('f1',ARRAY('aa2','aa2')),
NAMED_STRUCT('f1',ARRAY('aa2','aa2', "
+ // value 3
+ + "(ARRAY(NAMED_STRUCT('f1','c1', 'f2','c2'),
NAMED_STRUCT('f1','c3', 'f2','c4'), "
+ + "NAMED_STRUCT('f1','c5', 'f2','c6')), 3, " + "ARRAY(ARRAY(3,3,3),
ARRAY(3,3,3), ARRAY(3,3,3)), "
+ + "ARRAY(NAMED_STRUCT('f1',ARRAY('aa3','aa3','aa3')), "
+ + "NAMED_STRUCT('f1',ARRAY('aa3','aa3', 'aa3')),
NAMED_STRUCT('f1',ARRAY('aa3','aa3', 'aa3', "
+ // value 4
+ + "(ARRAY(NAMED_STRUCT('f1','d1', 'f2','d2'),
NAMED_STRUCT('f1','d3', 'f2','d4'),"
+ + " NAMED_STRUCT('f1','d5', 'f2','d6'), NAMED_STRUCT('f1','d7',
'f2','d8')), 4, "
+ +
"ARRAY(ARRAY(4,4,4,4),ARRAY(4,4,4,4),ARRAY(4,4,4,4),ARRAY(4,4,4,4)), "
+ + "ARRAY(NAMED_STRUCT('f1',ARRAY('aa4','aa4','aa4', 'aa4')), "
+ + "NAMED_STRUCT('f1',ARRAY('aa4','aa4','aa4', 'aa4')),
NAMED_STRUCT('f1',ARRAY('aa4','aa4','aa4', 'aa4')),"
+ + " NAMED_STRUCT('f1',ARRAY('aa4','aa4','aa4', 'aa4'");
+
+ // generate 4096 rows from above records
+ for (int i = 0; i < 10; i++) {
+stmt.execute(String.format("insert into %s select * from %s",
tableName, tableName));
+ }
+ // validate test table
+ ResultSet res = stmt.executeQuery("SELECT count(*) FROM " + tableName);
+ assertTrue(res.next());
+ assertEquals(4096, res.getInt(1));
+ res.close();
+}
+
+RowCo
[hive] branch master updated: HIVE-22998 : Dump partition info if hive.repl.dump.metadata.only.for.external.table conf is enabled. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 109ecb9 HIVE-22998 : Dump partition info if
hive.repl.dump.metadata.only.for.external.table conf is enabled. (Aasha Medhi,
reviewed by Mahesh Kumar Behera)
109ecb9 is described below
commit 109ecb99e50ac63add05a8a3b513b51e3e78a1cb
Author: Aasha Medhi
AuthorDate: Fri Mar 13 10:09:47 2020 +0530
HIVE-22998 : Dump partition info if
hive.repl.dump.metadata.only.for.external.table conf is enabled. (Aasha Medhi,
reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../TestReplicationScenariosExternalTables.java| 7 ++-
...icationScenariosExternalTablesMetaDataOnly.java | 11 +--
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 2 +-
.../hadoop/hive/ql/parse/ReplicationSpec.java | 12
.../hive/ql/parse/repl/dump/HiveWrapper.java | 3 ++-
.../hive/ql/parse/repl/dump/TableExport.java | 22 --
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 9 ++---
.../repl/dump/events/AbstractEventHandler.java | 2 +-
.../parse/repl/dump/events/AlterTableHandler.java | 3 ++-
.../parse/repl/dump/events/CreateTableHandler.java | 3 ++-
.../repl/dump/events/UpdatePartColStatHandler.java | 4 ++--
.../dump/events/UpdateTableColStatHandler.java | 3 ++-
12 files changed, 61 insertions(+), 20 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
index 81feaf5..1ba8003 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
@@ -167,7 +167,8 @@ public class TestReplicationScenariosExternalTables extends
BaseReplicationAcros
.run("select country from t2 where country = 'us'")
.verifyResult("us")
.run("select country from t2 where country = 'france'")
-.verifyResult("france");
+.verifyResult("france")
+.run("show partitions t2").verifyResults(new String[]
{"country=france", "country=india", "country=us"});
String hiveDumpLocation = tuple.dumpLocation + File.separator +
ReplUtils.REPL_HIVE_BASE_DIR;
// Ckpt should be set on bootstrapped db.
@@ -343,6 +344,8 @@ public class TestReplicationScenariosExternalTables extends
BaseReplicationAcros
.verifyResults(new String[] { "bangalore", "pune", "mumbai" })
.run("select place from t2 where country='australia'")
.verifyResults(new String[] { "sydney" })
+.run("show partitions t2")
+.verifyResults(new String[] {"country=australia", "country=india"})
.verifyReplTargetProperty(replicatedDbName);
Path customPartitionLocation =
@@ -364,6 +367,8 @@ public class TestReplicationScenariosExternalTables extends
BaseReplicationAcros
.run("use " + replicatedDbName)
.run("select place from t2 where country='france'")
.verifyResults(new String[] { "paris" })
+.run("show partitions t2")
+.verifyResults(new String[] {"country=australia", "country=france",
"country=india"})
.verifyReplTargetProperty(replicatedDbName);
// change the location of the partition via alter command
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
index 624f29b..c260a7d 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
@@ -166,7 +166,8 @@ public class
TestReplicationScenariosExternalTablesMetaDataOnly extends BaseRepl
.run("select country from t2 where country = 'us'")
.verifyResult(null)
.run("select country from t2 where country = 'france'")
-.verifyResult(null);
+.verifyResult(null)
+.run("show partitions t2").verifyResults(new String[]
{"country=france", "country=india", "country=us"})
[hive] branch master updated: HIVE-22860 : Support metadata only replication for external tables. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e0a0db3 HIVE-22860 : Support metadata only replication for external
tables. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
e0a0db3 is described below
commit e0a0db37fd6823ba3c3ff3374d775d3d151732fa
Author: Aasha Medhi
AuthorDate: Wed Feb 19 08:46:28 2020 +0530
HIVE-22860 : Support metadata only replication for external tables. (Aasha
Medhi, reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../TestReplicationScenariosExternalTables.java| 1 -
...cationScenariosExternalTablesMetaDataOnly.java} | 374 +++--
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 9 +-
.../hive/ql/exec/repl/ReplExternalTables.java | 3 +-
.../hive/ql/parse/ReplicationSemanticAnalyzer.java | 4 +-
.../hadoop/hive/ql/parse/ReplicationSpec.java | 10 +
.../hive/ql/parse/repl/dump/HiveWrapper.java | 2 +-
.../hive/ql/parse/repl/dump/TableExport.java | 12 +-
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 8 +
.../parse/repl/dump/events/AlterTableHandler.java | 2 +-
.../parse/repl/dump/events/CreateTableHandler.java | 3 +-
12 files changed, 91 insertions(+), 340 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d3cb60b..0eee582 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -478,6 +478,9 @@ public class HiveConf extends Configuration {
REPL_DUMP_METADATA_ONLY("hive.repl.dump.metadata.only", false,
"Indicates whether replication dump only metadata information or data
+ metadata. \n"
+ "This config makes hive.repl.include.external.tables config
ineffective."),
+
REPL_DUMP_METADATA_ONLY_FOR_EXTERNAL_TABLE("hive.repl.dump.metadata.only.for.external.table",
+false,
+"Indicates whether external table replication dump only metadata
information or data + metadata"),
REPL_BOOTSTRAP_ACID_TABLES("hive.repl.bootstrap.acid.tables", false,
"Indicates if repl dump should bootstrap the information about ACID
tables along with \n"
+ "incremental dump for replication. It is recommended to keep
this config parameter \n"
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
index 7a90dcc..200e02c 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
@@ -28,7 +28,6 @@ import
org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.CallerArg
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import
org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder;
-import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
similarity index 62%
copy from
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
copy to
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
index 7a90dcc..1d82467 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
@@ -24,14 +24,10 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore;
import
org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection;
-import
org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.CallerArguments;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import
org.
[hive] branch master updated: HIVE-22844 : Validate cm configs, add retries in fs apis for cm. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 21f801d HIVE-22844 : Validate cm configs, add retries in fs apis for
cm. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
21f801d is described below
commit 21f801dc82b93eb9c6ee449be15a003cd228e02d
Author: Aasha Medhi
AuthorDate: Tue Feb 18 08:45:09 2020 +0530
HIVE-22844 : Validate cm configs, add retries in fs apis for cm. (Aasha
Medhi, reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +-
.../MetastoreHousekeepingLeaderTestBase.java | 1 +
.../TestMetaStoreMultipleEncryptionZones.java | 83 +-
.../hadoop/hive/metastore/ReplChangeManager.java | 65 +++--
.../hadoop/hive/metastore/conf/MetastoreConf.java | 2 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 77 +---
6 files changed, 165 insertions(+), 65 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 2f695d4..d3cb60b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -455,7 +455,7 @@ public class HiveConf extends Configuration {
REPLCMENCRYPTEDDIR("hive.repl.cm.encryptionzone.rootdir", ".cmroot",
"Root dir for ChangeManager if encryption zones are enabled, used
for deleted files."),
REPLCMFALLBACKNONENCRYPTEDDIR("hive.repl.cm.nonencryptionzone.rootdir",
-"/user/${system:user.name}/cmroot/",
+"",
"Root dir for ChangeManager for non encrypted paths if
hive.repl.cmrootdir is encrypted."),
REPLCMINTERVAL("hive.repl.cm.interval","3600s",
new TimeValidator(TimeUnit.SECONDS),
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/MetastoreHousekeepingLeaderTestBase.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/MetastoreHousekeepingLeaderTestBase.java
index d89d67c..7ba0d3e 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/MetastoreHousekeepingLeaderTestBase.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/MetastoreHousekeepingLeaderTestBase.java
@@ -109,6 +109,7 @@ class MetastoreHousekeepingLeaderTestBase {
MetastoreConf.setBoolVar(conf, ConfVars.REPLCMENABLED, true);
String cmroot = "hdfs://" + miniDFS.getNameNode().getHostAndPort() +
"/cmroot";
MetastoreConf.setVar(conf, ConfVars.REPLCMDIR, cmroot);
+MetastoreConf.setVar(conf, ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR, cmroot);
threadNames.put(ReplChangeManager.CM_THREAD_NAME_PREFIX, false);
}
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
index 51bb787..41a1ce9 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
@@ -1364,8 +1364,10 @@ public class TestMetaStoreMultipleEncryptionZones {
"hdfs://" + miniDFSCluster.getNameNode().getHostAndPort()
+ HiveConf.ConfVars.METASTOREWAREHOUSE.defaultStrVal);
-String cmrootdirEncrypted = "hdfs://" +
miniDFSCluster.getNameNode().getHostAndPort() + "/cmroot";
+String cmrootdirEncrypted = "hdfs://" +
miniDFSCluster.getNameNode().getHostAndPort() + "/cmrootDirEncrypted";
encryptedHiveConf.set(HiveConf.ConfVars.REPLCMDIR.varname,
cmrootdirEncrypted);
+FileSystem cmrootdirEncryptedFs = new
Path(cmrootdirEncrypted).getFileSystem(hiveConf);
+cmrootdirEncryptedFs.mkdirs(new Path(cmrootdirEncrypted));
encryptedHiveConf.set(HiveConf.ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR.varname,
cmrootFallBack);
//Create cm in encrypted zone
@@ -1410,10 +1412,89 @@ public class TestMetaStoreMultipleEncryptionZones {
exceptionThrown = true;
}
assertFalse(exceptionThrown);
+cmrootdirEncryptedFs.delete(new Path(cmrootdirEncrypted), true);
ReplChangeManager.resetReplChangeManagerInstance();
initReplChangeManager();
}
+ @Test
+ public void testCmrootFallbackEncrypted() throws Exception {
+HiveConf encryptedHiveConf = new HiveConf(TestReplChangeManager.class);
+encryptedHiveConf.setBoolean(HiveConf.ConfVars.REPLCMENABLED.varname,
true);
+
encryptedHiveCo
[hive] branch master updated: HIVE-22890 : Repl load fails if table name contains _function. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new db97be2 HIVE-22890 : Repl load fails if table name contains
_function. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
db97be2 is described below
commit db97be2b7c029516e1e42cadf682f97a797dd5ad
Author: Aasha Medhi
AuthorDate: Mon Feb 17 10:21:09 2020 +0530
HIVE-22890 : Repl load fails if table name contains _function. (Aasha
Medhi, reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../TestReplicationScenariosExternalTables.java| 46 ++
.../events/filesystem/DatabaseEventsIterator.java | 2 +-
2 files changed, 47 insertions(+), 1 deletion(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
index 1815824..7a90dcc 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
@@ -855,6 +855,52 @@ public class TestReplicationScenariosExternalTables
extends BaseReplicationAcros
.verifyReplTargetProperty(replicatedDbName);
}
+ @Test
+ public void replicationWithTableNameContainsKeywords() throws Throwable {
+List loadWithClause = externalTableBasePathWithClause();
+
+WarehouseInstance.Tuple tuple = primary
+.run("use " + primaryDbName)
+.run("create external table t1_functions (id int)")
+.run("insert into table t1_functions values (1)")
+.run("insert into table t1_functions values (2)")
+.run("create external table t2_constraints (place string)
partitioned by (country string)")
+.run("insert into table t2_constraints partition(country='india')
values ('bangalore')")
+.run("insert into table t2_constraints partition(country='us')
values ('austin')")
+.run("insert into table t2_constraints partition(country='france')
values ('paris')")
+.dump(primaryDbName, null);
+
+replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
+.run("repl status " + replicatedDbName)
+.verifyResult(tuple.lastReplicationId)
+.run("use " + replicatedDbName)
+.run("show tables like 't1_functions'")
+.verifyResults(new String[] {"t1_functions"})
+.run("show tables like 't2_constraints'")
+.verifyResults(new String[] {"t2_constraints"})
+.run("select id from t1_functions")
+.verifyResults(new String[] {"1", "2"})
+.verifyReplTargetProperty(replicatedDbName);
+
+tuple = primary.run("use " + primaryDbName)
+.run("create external table t3_bootstrap (id int)")
+.run("insert into table t3_bootstrap values (10)")
+.run("insert into table t3_bootstrap values (20)")
+.run("create table t4_tables (id int)")
+.run("insert into table t4_tables values (10)")
+.run("insert into table t4_tables values (20)")
+.dump(primaryDbName, tuple.lastReplicationId);
+
+replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
+.run("use " + replicatedDbName)
+.run("show tables like 't3_bootstrap'")
+.verifyResults(new String[] {"t3_bootstrap"})
+.run("show tables like 't4_tables'")
+.verifyResults(new String[] {"t4_tables"})
+.verifyReplTargetProperty(replicatedDbName);
+ }
+
+
private List externalTableBasePathWithClause() throws IOException,
SemanticException {
return
ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE,
replica);
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
index 4c84797..72baee6 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
@@ -170,7 +170,7 @@ class DatabaseEventsIterator implements
Iterator {
}
String currentPath = next.toString();
-if (currentPath.contains(FUNCTIONS_ROOT_DIR_NAME)) {
+if (currentPath.contains(Path.SEPARATOR + FUNCTIONS_ROOT_DIR_NAME +
Path.SEPARATOR)) {
LOG.debug("functions directory: {}", next.toString());
return postProcessing(new FSFunctionEvent(next));
}
[hive] branch master updated: HIVE-22856 : Hive LLAP LlapArrowBatchRecordReader skipping remaining batches when ArrowStreamReader returns a 0 length batch. (Mahesh Kumar Behera, reviewed by Jason Dere
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 99cc8db HIVE-22856 : Hive LLAP LlapArrowBatchRecordReader skipping
remaining batches when ArrowStreamReader returns a 0 length batch. (Mahesh
Kumar Behera, reviewed by Jason Dere)
99cc8db is described below
commit 99cc8db52f32eac7d57c9df9ca9e2b6d57ace194
Author: Mahesh Kumar Behera
AuthorDate: Sun Feb 16 16:28:41 2020 +0530
HIVE-22856 : Hive LLAP LlapArrowBatchRecordReader skipping remaining
batches when ArrowStreamReader returns a 0 length batch. (Mahesh Kumar Behera,
reviewed by Jason Dere)
---
.../org/apache/hadoop/hive/llap/LlapArrowBatchRecordReader.java | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapArrowBatchRecordReader.java
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapArrowBatchRecordReader.java
index cb3d9cc..1ac7637 100644
---
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapArrowBatchRecordReader.java
+++
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapArrowBatchRecordReader.java
@@ -65,10 +65,8 @@ public class LlapArrowBatchRecordReader extends
LlapBaseRecordReader
0);
-if(vectorSchemaRoot.getFieldVectors().get(0).getValueCount() == 0) {
- //An empty batch will appear at the end of the stream
- return false;
-}
+// We should continue even if FieldVectors are empty. The next read
might have the
+// data. We should stop only when loadNextBatch returns false.
value.setVectorSchemaRoot(arrowStreamReader.getVectorSchemaRoot());
return true;
} else {
[hive] branch master updated: HIVE-22736 : Support replication across multiple encryption zones. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 7411d42 HIVE-22736 : Support replication across multiple encryption
zones. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
7411d42 is described below
commit 7411d42579ffa0bad96e8da731a1a35afc9ff614
Author: Aasha Medhi
AuthorDate: Fri Feb 7 07:16:24 2020 +0530
HIVE-22736 : Support replication across multiple encryption zones. (Aasha
Medhi, reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java |5 +
.../TestMetaStoreMultipleEncryptionZones.java | 1434
.../hive/metastore/TestReplChangeManager.java |1 +
.../cache/TestCachedStoreUpdateUsingEvents.java|2 +
.../hive/ql/metadata/TestAlterTableMetadata.java |1 +
.../java/org/apache/hive/jdbc/TestJdbcDriver2.java |4 +
.../org/apache/hive/jdbc/TestJdbcWithMiniHS2.java |1 +
.../ql/ddl/table/create/CreateTableOperation.java |1 -
.../table/storage/AlterTableArchiveOperation.java |4 +-
.../ddl/table/storage/AlterTableArchiveUtils.java |5 +-
.../storage/AlterTableSetLocationOperation.java|1 +
.../storage/AlterTableUnarchiveOperation.java |6 +-
.../org/apache/hadoop/hive/ql/metadata/Hive.java |4 +-
.../hive/ql/parse/ImportSemanticAnalyzer.java |4 +-
.../hadoop/hive/ql/txn/compactor/Cleaner.java |4 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java|2 +-
.../org/apache/hadoop/hive/shims/HadoopShims.java | 15 +-
standalone-metastore/metastore-common/pom.xml |5 +
.../hadoop/hive/metastore/ReplChangeManager.java | 187 ++-
.../hadoop/hive/metastore/conf/MetastoreConf.java |5 +
.../apache/hadoop/hive/metastore/utils/Retry.java | 52 +
.../hadoop/hive/metastore/utils/package-info.java | 22 +
.../hadoop/hive/metastore/utils/RetryTest.java | 57 +
.../hadoop/hive/metastore/utils/package-info.java | 22 +
.../hadoop/hive/metastore/HiveAlterHandler.java|7 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 120 +-
26 files changed, 1886 insertions(+), 85 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 12a022c..a120b45 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -452,6 +452,11 @@ public class HiveConf extends Configuration {
REPLCMRETIAN("hive.repl.cm.retain","24h",
new TimeValidator(TimeUnit.HOURS),
"Time to retain removed files in cmrootdir."),
+REPLCMENCRYPTEDDIR("hive.repl.cm.encryptionzone.rootdir", ".cmroot",
+"Root dir for ChangeManager if encryption zones are enabled, used
for deleted files."),
+REPLCMFALLBACKNONENCRYPTEDDIR("hive.repl.cm.nonencryptionzone.rootdir",
+"/user/${system:user.name}/cmroot/",
+"Root dir for ChangeManager for non encrypted paths if
hive.repl.cmrootdir is encrypted."),
REPLCMINTERVAL("hive.repl.cm.interval","3600s",
new TimeValidator(TimeUnit.SECONDS),
"Inteval for cmroot cleanup thread."),
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
new file mode 100644
index 000..51bb787
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
@@ -0,0 +1,1434 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigur
[hive] branch master updated: HIVE-22548 : Optimise Utilities.removeTempOrDuplicateFiles when moving files to final location. (Mahesh Kumar Behera, reviewed by Rajesh Balamohan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0dfecd5 HIVE-22548 : Optimise Utilities.removeTempOrDuplicateFiles
when moving files to final location. (Mahesh Kumar Behera, reviewed by Rajesh
Balamohan)
0dfecd5 is described below
commit 0dfecd5d14aee9de81754347c6728afe9d7134dc
Author: Mahesh Kumar Behera
AuthorDate: Thu Dec 19 17:30:08 2019 +0530
HIVE-22548 : Optimise Utilities.removeTempOrDuplicateFiles when moving
files to final location. (Mahesh Kumar Behera, reviewed by Rajesh Balamohan)
---
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 108 +++--
1 file changed, 77 insertions(+), 31 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 3d30d09..92e46fd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -63,6 +63,7 @@ import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -1598,18 +1599,65 @@ public final class Utilities {
fs, fileStats, null, dpLevels, numBuckets, hconf, null, 0, false,
filesKept, isBaseDir);
}
- private static boolean removeEmptyDpDirectory(FileSystem fs, Path path)
throws IOException {
+ private static FileStatus[] removeEmptyDpDirectory(FileSystem fs, Path path)
throws IOException {
+// listStatus is not required to be called to check if we need to delete
the directory or not,
+// delete does that internally. We are getting the file list as it is used
by the caller.
FileStatus[] items = fs.listStatus(path);
-// remove empty directory since DP insert should not generate empty
partitions.
-// empty directories could be generated by crashed Task/ScriptOperator
-if (items.length != 0) {
- return false;
+
+// Remove empty directory since DP insert should not generate empty
partitions.
+// Empty directories could be generated by crashed Task/ScriptOperator.
+if (items.length == 0) {
+ // delete() returns false in only two conditions
+ // 1. Tried to delete root
+ // 2. The file wasn't actually there (or deleted by some other thread)
+ // So return value is not checked for delete.
+ fs.delete(path, true);
}
-if (!fs.delete(path, true)) {
- LOG.error("Cannot delete empty directory {}", path);
- throw new IOException("Cannot delete empty directory " + path);
+return items;
+ }
+
+ // Returns the list of non empty sub-directories, deletes the empty sub
sub-directories.
+ private static Map getNonEmptySubDirs(FileSystem fs,
Configuration hConf, FileStatus[] parts)
+ throws IOException {
+int threadCount =
hConf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 15);
+final ExecutorService pool = (threadCount <= 0 ? null :
+ Executors.newFixedThreadPool(threadCount, new
ThreadFactoryBuilder().setDaemon(true).setNameFormat(
+"Remove-Temp-%d").build()));
+Map partStatusMap = new ConcurrentHashMap<>();
+List> futures = new LinkedList<>();
+
+for (FileStatus part : parts) {
+ Path path = part.getPath();
+ if (pool != null) {
+futures.add(pool.submit(() -> {
+ FileStatus[] items = removeEmptyDpDirectory(fs, path);
+ partStatusMap.put(path, items);
+ return null;
+}));
+ } else {
+partStatusMap.put(path, removeEmptyDpDirectory(fs, path));
+ }
}
-return true;
+
+if (null != pool) {
+ pool.shutdown();
+ try {
+for (Future future : futures) {
+ future.get();
+}
+ } catch (InterruptedException | ExecutionException e) {
+LOG.error("Exception in getting dir status", e);
+for (Future future : futures) {
+ future.cancel(true);
+}
+throw new IOException(e);
+ }
+}
+
+// Dump of its metrics
+LOG.debug("FS {}", fs);
+
+return partStatusMap;
}
public static List removeTempOrDuplicateFiles(FileSystem fs,
FileStatus[] fileStats,
@@ -1621,33 +1669,32 @@ public final class Utilities {
List result = new ArrayList();
HashMap taskIDToFile = null;
if (dpLevels > 0) {
- FileStatus[] parts = fileStats;
- for (int i = 0; i < parts.length; ++i) {
-assert parts[i].isDirectory() : "dynamic partition " +
parts[i].getPath()
-+ " is not a
[hive] branch master updated: HIVE-22379 : Reduce db lookups during dynamic partition loading. (Mahesh Kumar Behera reviewed by Rajesh Balamohan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 7c796af HIVE-22379 : Reduce db lookups during dynamic partition
loading. (Mahesh Kumar Behera reviewed by Rajesh Balamohan)
7c796af is described below
commit 7c796af642d3bc5d1ae72d2fb0fa31191b77ec45
Author: Mahesh Kumar Behera
AuthorDate: Thu Nov 28 13:39:37 2019 +0530
HIVE-22379 : Reduce db lookups during dynamic partition loading. (Mahesh
Kumar Behera reviewed by Rajesh Balamohan)
---
.../TestReplicationScenariosAcrossInstances.java | 25 +++--
.../hadoop/hive/metastore/HiveAlterHandler.java| 26 +-
.../apache/hadoop/hive/metastore/ObjectStore.java | 13 +--
.../metastore/InjectableBehaviourObjectStore.java | 22 ++
4 files changed, 71 insertions(+), 15 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 4f5cbbb..8df5fc8 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1387,27 +1387,28 @@ public class TestReplicationScenariosAcrossInstances
extends BaseReplicationAcro
// Inject a behavior where REPL LOAD failed when try to load table "t2"
and partition "uk".
// So, table "t2" will exist and partition "india" will exist, rest failed
as operation failed.
-BehaviourInjection getPartitionStub
-= new BehaviourInjection() {
- @Nullable
+BehaviourInjection, Boolean> alterPartitionStub
+= new BehaviourInjection, Boolean>() {
@Override
- public Partition apply(@Nullable Partition ptn) {
-if (ptn.getValues().get(0).equals("india")) {
- injectionPathCalled = true;
- LOG.warn("getPartition Stub called");
- return null;
+ public Boolean apply(List ptns) {
+for (Partition ptn : ptns) {
+ if (ptn.getValues().get(0).equals("india")) {
+injectionPathCalled = true;
+LOG.warn("getPartition Stub called");
+return false;
+ }
}
-return ptn;
+return true;
}
};
-InjectableBehaviourObjectStore.setGetPartitionBehaviour(getPartitionStub);
+
InjectableBehaviourObjectStore.setAlterPartitionsBehaviour(alterPartitionStub);
// Make sure that there's some order in which the objects are loaded.
List withConfigs =
Arrays.asList("'hive.repl.approx.max.load.tasks'='1'",
"'hive.in.repl.test.files.sorted'='true'");
replica.loadFailure(replicatedDbName, tuple.dumpLocation, withConfigs);
-InjectableBehaviourObjectStore.resetGetPartitionBehaviour(); // reset the
behaviour
-getPartitionStub.assertInjectionsPerformed(true, false);
+InjectableBehaviourObjectStore.setAlterPartitionsBehaviour(null); // reset
the behaviour
+alterPartitionStub.assertInjectionsPerformed(true, false);
replica.run("use " + replicatedDbName)
.run("repl status " + replicatedDbName)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 213dd9e..5eaf53f 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -64,6 +64,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.stream.Collectors;
+import java.util.LinkedList;
import static org.apache.hadoop.hive.metastore.HiveMetaHook.ALTERLOCATION;
import static
org.apache.hadoop.hive.metastore.HiveMetaHook.ALTER_TABLE_OPERATION_TYPE;
@@ -732,6 +734,27 @@ public class HiveAlterHandler implements AlterHandler {
environmentContext, null, -1, null);
}
+ private Map, Partition> getExistingPartitions(final RawStore
msdb,
+ final List new_parts, final Table tbl, final String catName,
+ final String dbname, final String name)
+ throws MetaException, NoSuchObjectException, InvalidOperationException {
+
+// Get list of partition values
+List partValues = new LinkedList<>();
+for (Partiti
[hive] branch master updated: HIVE-22512 : Use direct SQL to fetch column privileges in refreshPrivileges. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2b34ad7 HIVE-22512 : Use direct SQL to fetch column privileges in
refreshPrivileges. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
2b34ad7 is described below
commit 2b34ad7facc1aed74307f5c7991019116b043346
Author: Ashutosh Bapat
AuthorDate: Wed Nov 27 16:22:29 2019 +0530
HIVE-22512 : Use direct SQL to fetch column privileges in
refreshPrivileges. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 94 +-
.../apache/hadoop/hive/metastore/ObjectStore.java | 40 -
2 files changed, 129 insertions(+), 5 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 54fe1eb..9f2b436 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -55,10 +55,14 @@ import
org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec;
+import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
+import org.apache.hadoop.hive.metastore.api.HiveObjectType;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
@@ -159,8 +163,8 @@ class MetaStoreDirectSql {
private String DBS, TBLS, PARTITIONS, DATABASE_PARAMS, PARTITION_PARAMS,
SORT_COLS, SD_PARAMS,
SDS, SERDES, SKEWED_STRING_LIST_VALUES, SKEWED_VALUES, BUCKETING_COLS,
SKEWED_COL_NAMES,
SKEWED_COL_VALUE_LOC_MAP, COLUMNS_V2, PARTITION_KEYS, SERDE_PARAMS,
PART_COL_STATS, KEY_CONSTRAINTS,
- TAB_COL_STATS, PARTITION_KEY_VALS, PART_PRIVS, PART_COL_PRIVS,
SKEWED_STRING_LIST, CDS;
-
+ TAB_COL_STATS, PARTITION_KEY_VALS, PART_PRIVS, PART_COL_PRIVS,
SKEWED_STRING_LIST, CDS,
+ TBL_COL_PRIVS;
public MetaStoreDirectSql(PersistenceManager pm, Configuration conf, String
schema) {
this.pm = pm;
@@ -1280,6 +1284,92 @@ class MetaStoreDirectSql {
return result;
}
+ public List getTableAllColumnGrants(String catName,
String dbName,
+ String tableName,
String authorizer) throws MetaException {
+Query query = null;
+
+// These constants should match the SELECT clause of the query.
+final int authorizerIndex = 0;
+final int columnNameIndex = 1;
+final int createTimeIndex = 2;
+final int grantOptionIndex = 3;
+final int grantorIndex = 4;
+final int grantorTypeIndex = 5;
+final int principalNameIndex = 6;
+final int principalTypeIndex = 7;
+final int privilegeIndex = 8;
+
+// Retrieve the privileges from the object store. Just grab only the
required fields.
+String queryText = "select " +
+TBL_COL_PRIVS + ".\"AUTHORIZER\", " +
+TBL_COL_PRIVS + ".\"COLUMN_NAME\", " +
+TBL_COL_PRIVS + ".\"CREATE_TIME\", " +
+TBL_COL_PRIVS + ".\"GRANT_OPTION\", " +
+TBL_COL_PRIVS + ".\"GRANTOR\", " +
+TBL_COL_PRIVS + ".\"GRANTOR_TYPE\", " +
+TBL_COL_PRIVS + ".\"PRINCIPAL_NAME\", " +
+TBL_COL_PRIVS + ".\"PRINCIPAL_TYPE\", " +
+TBL_COL_PRIVS + ".\"TBL_COL_PRIV\" " +
+"FROM " + TBL_COL_PRIVS + " JOIN " + TBLS +
+" ON " + TBL_COL_PRIVS + ".\"TBL_ID\" = " + TBLS + ".\"TBL_ID\"" +
+" JOIN " + DBS + " ON " + TBLS + ".\"DB_ID\" = " + DBS +
".\"DB_ID\" " +
+" WHERE " + TBLS + ".\"TBL_NAME\" = ?" +
+" AND "
[hive] branch master updated: HIVE-22497 : Remove default value for Capabilities from HiveConf. (Naveen Gangam, reviewed by Sam An)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c032cbe HIVE-22497 : Remove default value for Capabilities from
HiveConf. (Naveen Gangam, reviewed by Sam An)
c032cbe is described below
commit c032cbe2d3fdfc9a107c91f3642d95560efd89a2
Author: Naveen Gangam
AuthorDate: Tue Nov 19 12:25:37 2019 +0530
HIVE-22497 : Remove default value for Capabilities from HiveConf. (Naveen
Gangam, reviewed by Sam An)
Signed-off-by: Mahesh Kumar Behera
---
.../src/java/org/apache/hadoop/hive/conf/HiveConf.java | 16 ++--
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 11 +++
2 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 9162253..a7687d5 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -745,10 +745,7 @@ public class HiveConf extends Configuration {
@Deprecated
METASTORE_CAPABILITY_CHECK("hive.metastore.client.capability.check", true,
"Whether to check client capabilities for potentially breaking API
usage."),
-METASTORE_CLIENT_CAPABILITIES("hive.metastore.client.capabilities",
"EXTWRITE,EXTREAD,HIVEBUCKET2,"
-+
"HIVEFULLACIDREAD,HIVEFULLACIDWRITE,HIVECACHEINVALIDATE,HIVEMANAGESTATS,"
-+ "HIVEMANAGEDINSERTWRITE,HIVEMANAGEDINSERTREAD,"
-+ "HIVESQL,HIVEMQT,HIVEONLYMQTWRITE", "Capabilities possessed by
HiveServer"),
+METASTORE_CLIENT_CAPABILITIES("hive.metastore.client.capabilities", "",
"Capabilities possessed by HiveServer"),
METASTORE_CLIENT_CACHE_ENABLED("hive.metastore.client.cache.enabled",
false,
"Whether to enable metastore client cache"),
METASTORE_CLIENT_CACHE_EXPIRY_TIME("hive.metastore.client.cache.expiry.time",
"120s",
@@ -5620,8 +5617,15 @@ public class HiveConf extends Configuration {
// metastore can be embedded within hiveserver2, in such cases
// the conf params in hiveserver2-site.xml will override whats defined
// in hivemetastore-site.xml
-if (isLoadHiveServer2Config() && hiveServer2SiteUrl != null) {
- addResource(hiveServer2SiteUrl);
+if (isLoadHiveServer2Config()) {
+ // set the hardcoded value first, so anything in hiveserver2-site.xml
can override it
+ set(ConfVars.METASTORE_CLIENT_CAPABILITIES.varname,
"EXTWRITE,EXTREAD,HIVEBUCKET2,HIVEFULLACIDREAD,"
+ +
"HIVEFULLACIDWRITE,HIVECACHEINVALIDATE,HIVEMANAGESTATS,HIVEMANAGEDINSERTWRITE,HIVEMANAGEDINSERTREAD,"
+ + "HIVESQL,HIVEMQT,HIVEONLYMQTWRITE");
+
+ if (hiveServer2SiteUrl != null) {
+addResource(hiveServer2SiteUrl);
+ }
}
String val = this.getVar(HiveConf.ConfVars.HIVE_ADDITIONAL_CONFIG_FILES);
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 459af5b..736c2b0 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -176,6 +176,17 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
fileMetadataBatchSize = MetastoreConf.getIntVar(
conf, ConfVars.BATCH_RETRIEVE_OBJECTS_MAX);
+if ((MetastoreConf.get(conf, "hive.metastore.client.capabilities")) !=
null) {
+ String[] capabilities = MetastoreConf.get(conf,
"hive.metastore.client.capabilities").split(",");
+ setProcessorCapabilities(capabilities);
+ String hostName = "unknown";
+ try {
+hostName = InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException ue) {
+ }
+ setProcessorIdentifier("HMSClient-" + "@" + hostName);
+}
+
String msUri = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS);
localMetaStore = MetastoreConf.isEmbeddedMetaStore(msUri);
if (localMetaStore) {
[hive] branch master updated: HIVE-22319 : Repl load fails to create partition if the dump is from old version. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6e6900e HIVE-22319 : Repl load fails to create partition if the dump
is from old version. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
6e6900e is described below
commit 6e6900e6f22d20e9785e3c17e9a52ae9b0bd03f6
Author: Mahesh Kumar Behera
AuthorDate: Fri Oct 11 11:31:07 2019 +0530
HIVE-22319 : Repl load fails to create partition if the dump is from old
version. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
---
data/files/repl_dump/test_hcube_2/tbl6/_metadata | 2 +-
.../files/repl_dump/test_hcube_2/tbl6/{data => fld1=1}/_files | 0
.../files/repl_dump/test_hcube_2/tbl6/{data => fld1=2}/_files | 0
.../ddl/table/partition/AlterTableAddPartitionOperation.java | 6 ++
.../hadoop/hive/ql/parse/repl/load/message/TableHandler.java | 1 -
.../ql/parse/repl/load/message/UpdatePartColStatHandler.java | 4
.../ql/parse/repl/load/message/UpdateTableColStatHandler.java | 4
ql/src/test/queries/clientpositive/repl_load_old_version.q| 2 +-
.../test/results/clientpositive/repl_load_old_version.q.out | 11 ---
9 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/data/files/repl_dump/test_hcube_2/tbl6/_metadata
b/data/files/repl_dump/test_hcube_2/tbl6/_metadata
index 75ead6a..16f5c7e 100644
--- a/data/files/repl_dump/test_hcube_2/tbl6/_metadata
+++ b/data/files/repl_dump/test_hcube_2/tbl6/_metadata
@@ -1 +1 @@
-{"version":"0.2","repl.scope":"all","repl.event.id":"replv2","repl.last.id":"548434","repl.noop":"false","repl.lazy":"true","repl.is.replace":"true","repl.valid.writeid.list":"test_hcube_2.tbl6:1:9223372036854775807::","repl.valid.txnid.list":"1915:9223372036854775807::1899,1907","table":"{\"1\":{\"str\":\"tbl6\"},\"2\":{\"str\":\"test_hcube_2\"},\"3\":{\"str\":\"hive\"},\"4\":{\"i32\":1569308637},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str
[...]
\ No newline at end of file
+{"version":"0.2","repl.scope":"all","repl.event.id":"replv2","repl.last.id":"326198","repl.noop":"false","repl.lazy":"true","repl.is.replace":"true","repl.valid.writeid.list":"test_hcube_2.tbl6:2:9223372036854775807::","repl.valid.txnid.list":"162684:9223372036854775807::","table":"{\"1\":{\"str\":\"tbl6\"},\"2\":{\"str\":\"test_hcube_2\"},\"3\":{\"str\":\"hive\"},\"4\":{\"i32\":1570685217},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"fl
[...]
diff --git a/data/files/repl_dump/test_hcube_2/tbl6/data/_files
b/data/files/repl_dump/test_hcube_2/tbl6/fld1=1/_files
similarity index 100%
copy from data/files/repl_dump/test_hcube_2/tbl6/data/_files
copy to data/files/repl_dump/test_hcube_2/tbl6/fld1=1/_files
diff --git a/data/files/repl_dump/test_hcube_2/tbl6/data/_files
b/data/files/repl_dump/test_hcube_2/tbl6/fld1=2/_files
similarity index 100%
rename from data/files/repl_dump/test_hcube_2/tbl6/data/_files
rename to data/files/repl_dump/test_hcube_2/tbl6/fld1=2/_files
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableAddPartitionOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableAddPartitionOperation.java
index d8597a8..0adccf6 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableAddPartitionOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableAddPartitionOperation.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
@@ -128,6 +129,11 @@ public class AlterTableAddPartitionOperation extends
DDLOperation
[hive] branch master updated: HIVE-22313 : Some of the HMS auth LDAP hive config names do not start with hive. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 37a0675 HIVE-22313 : Some of the HMS auth LDAP hive config names do
not start with hive. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
37a0675 is described below
commit 37a0675a03e66be2667a550d9c89f5566c12ec41
Author: Ashutosh Bapat
AuthorDate: Thu Oct 10 10:21:06 2019 +0530
HIVE-22313 : Some of the HMS auth LDAP hive config names do not start with
hive. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 3f69ad8..a496a59 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -657,13 +657,13 @@ public class MetastoreConf {
"The implementation may optionally implement Hadoop's\n" +
"org.apache.hadoop.conf.Configurable class to grab MetaStore's
Configuration object."),
METASTORE_PLAIN_LDAP_URL("metastore.authentication.ldap.url",
-"metastore.authentication.ldap.url", "",
+"hive.metastore.authentication.ldap.url", "",
"LDAP connection URL(s),\n" +
"this value could contain URLs to multiple LDAP servers instances for
HA,\n" +
"each LDAP URL is separated by a SPACE character. URLs are used in the
\n" +
" order specified until a connection is successful."),
METASTORE_PLAIN_LDAP_BASEDN("metastore.authentication.ldap.baseDN",
-"metastore.authentication.ldap.baseDN", "", "LDAP base DN"),
+"hive.metastore.authentication.ldap.baseDN", "", "LDAP base DN"),
METASTORE_PLAIN_LDAP_DOMAIN("metastore.authentication.ldap.Domain",
"hive.metastore.authentication.ldap.Domain", "", ""),
METASTORE_PLAIN_LDAP_GROUPDNPATTERN("metastore.authentication.ldap.groupDNPattern",
@@ -676,7 +676,7 @@ public class MetastoreConf {
"COMMA-separated list of LDAP Group names (short name not full DNs).\n" +
"For example: HiveAdmins,HadoopAdmins,Administrators"),
METASTORE_PLAIN_LDAP_USERDNPATTERN("metastore.authentication.ldap.userDNPattern",
-"metastore.authentication.ldap.userDNPattern", "",
+"hive.metastore.authentication.ldap.userDNPattern", "",
"COLON-separated list of patterns to use to find DNs for users in this
directory.\n" +
"Use %s where the actual group name is to be substituted for.\n" +
"For example: CN=%s,CN=Users,DC=subdomain,DC=domain,DC=com."),
@@ -689,7 +689,7 @@ public class MetastoreConf {
"LDAP attribute name whose values are unique in this LDAP
server.\n" +
"For example: uid or CN."),
METASTORE_PLAIN_LDAP_GROUPMEMBERSHIP_KEY("metastore.authentication.ldap.groupMembershipKey",
-"metastore.authentication.ldap.groupMembershipKey",
+"hive.metastore.authentication.ldap.groupMembershipKey",
"member",
"LDAP attribute name on the group object that contains the list of
distinguished names\n" +
"for the user, group, and contact objects that are members of the
group.\n" +
[hive] branch master updated: HIVE-22267 : update errata.txt
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new a254859 HIVE-22267 : update errata.txt a254859 is described below commit a254859f8d91893d4267940920d2dbb2d2ccc751 Author: Mahesh Kumar Behera AuthorDate: Tue Oct 8 20:59:38 2019 +0530 HIVE-22267 : update errata.txt --- errata.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/errata.txt b/errata.txt index 20df1ea..13f8b1a 100644 --- a/errata.txt +++ b/errata.txt @@ -99,3 +99,4 @@ e7081035bb9768bc014f0aba11417418ececbaf0 master HIVE-17109 https://issues.ap f33db1f68c68b552b9888988f818c03879749461 master HIVE-18617 https://issues.apache.org/jira/browse/HIVE-18617 1eea5a80ded2df33d57b2296b3bed98cb18383fd master HIVE-19157 https://issues.apache.org/jira/browse/HIVE-19157 4853a44b2fcfa702d23965ab0d3835b6b57954c4 master HIVE-21823 https://issues.apache.org/jira/browse/HIVE-21823 +bd432c9203584c531f51559d8c97c398202f0794 master HIVE-22267 https://issues.apache.org/jira/browse/HIVE-22267
[hive] branch master updated: HIVE-22234 : Hive replication fails with table already exist error when replicating from old version of hive. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 187e1c3 HIVE-22234 : Hive replication fails with table already exist
error when replicating from old version of hive. (Mahesh Kumar Behera reviewed
by Sankar Hariappan)
187e1c3 is described below
commit 187e1c3a2152ca2ea1534f890531b342ea89ae9f
Author: Mahesh Kumar Behera
AuthorDate: Mon Sep 30 09:55:56 2019 +0530
HIVE-22234 : Hive replication fails with table already exist error when
replicating from old version of hive. (Mahesh Kumar Behera reviewed by Sankar
Hariappan)
---
data/files/exported_table/_metadata| 2 +-
data/files/repl_dump/_dumpmetadata | 1 +
data/files/repl_dump/test_hcube_2/_metadata| 1 +
data/files/repl_dump/test_hcube_2/tbl/_metadata| 1 +
data/files/repl_dump/test_hcube_2/tbl/data/_files | 1 +
data/files/repl_dump/test_hcube_2/tbl1/_metadata | 1 +
data/files/repl_dump/test_hcube_2/tbl1/data/_files | 3 +
data/files/repl_dump/test_hcube_2/tbl2/_metadata | 1 +
data/files/repl_dump/test_hcube_2/tbl2/data/_files | 2 +
data/files/repl_dump/test_hcube_2/tbl3/_metadata | 1 +
data/files/repl_dump/test_hcube_2/tbl3/data/_files | 1 +
data/files/repl_dump/test_hcube_2/tbl4/_metadata | 1 +
data/files/repl_dump/test_hcube_2/tbl4/data/_files | 1 +
data/files/repl_dump/test_hcube_2/tbl5/_metadata | 1 +
data/files/repl_dump/test_hcube_2/tbl5/data/_files | 1 +
data/files/repl_dump/test_hcube_2/tbl6/_metadata | 1 +
data/files/repl_dump/test_hcube_2/tbl6/data/_files | 1 +
.../delta_001_001_/_orc_acid_version | 1 +
.../tbl/delta_001_001_/bucket_0| Bin 0 -> 611 bytes
.../delta_001_001_/_orc_acid_version | 1 +
.../tbl1/delta_001_001_/bucket_0 | Bin 0 -> 611 bytes
.../delta_002_002_/_orc_acid_version | 1 +
.../tbl1/delta_002_002_/bucket_0 | Bin 0 -> 620 bytes
.../delta_003_003_/_orc_acid_version | 1 +
.../tbl1/delta_003_003_/bucket_0 | Bin 0 -> 621 bytes
.../delta_001_001_/_orc_acid_version | 1 +
.../tbl2/delta_001_001_/bucket_0 | Bin 0 -> 611 bytes
.../delta_002_002_/_orc_acid_version | 1 +
.../tbl2/delta_002_002_/bucket_0 | Bin 0 -> 620 bytes
.../delta_001_001_/_orc_acid_version | 1 +
.../tbl3/delta_001_001_/bucket_0 | Bin 0 -> 611 bytes
.../delta_001_001_/_orc_acid_version | 1 +
.../tbl4/delta_001_001_/bucket_0 | Bin 0 -> 611 bytes
.../delta_001_001_/_orc_acid_version | 1 +
.../tbl5/delta_001_001_/bucket_0 | Bin 0 -> 611 bytes
.../delta_001_001_/_orc_acid_version | 1 +
.../tbl6/delta_001_001_/bucket_0 | Bin 0 -> 611 bytes
.../ql/ddl/table/creation/CreateTableDesc.java | 8 +-
.../bootstrap/events/filesystem/FSTableEvent.java | 3 +-
.../queries/clientpositive/repl_load_old_version.q | 10 +
.../clientpositive/repl_load_old_version.q.out | 80
.../hive/metastore/api/ColumnStatistics.java | 55 +--
.../apache/hadoop/hive/metastore/api/Table.java| 498 ++---
.../src/gen/thrift/gen-php/metastore/Types.php | 186
.../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 172 ---
.../src/gen/thrift/gen-rb/hive_metastore_types.rb | 61 ++-
.../src/main/thrift/hive_metastore.thrift | 54 +--
.../apache/hadoop/hive/metastore/ObjectStore.java | 8 +-
.../hadoop/hive/metastore/cache/SharedCache.java | 7 +-
.../apache/hadoop/hive/metastore/TestStats.java| 4 +-
50 files changed, 657 insertions(+), 520 deletions(-)
diff --git a/data/files/exported_table/_metadata
b/data/files/exported_table/_metadata
index 8d8f8bb..b37416e 100644
--- a/data/files/exported_table/_metadata
+++ b/data/files/exported_table/_metadata
@@ -1 +1 @@
-{"partitions":[],"table":"{\"2\":{\"str\":\"j1_41\"},\"3\":{\"str\":\"default\"},\"4\":{\"str\":\"johndee\"},\"5\":{\"i32\":1371900915},\"6\":{\"i32\":0},\"7\":{\"i32\":0},\"8\":{\"rec\":{\"1\":{\"lst\":[\"rec\",2,{\"1\":{\"str\":\"a\"},\"2\":{\"str\":\"string\"}},{\"1\":{\"str\":\"b\"},\"2\":{\"str\":\"int\"}}]},\"2\":{\"str\":\"hdfs://hivebase01:8020/user/hive/warehouse/j1_41\"},\"3\&qu
[hive] branch master updated: HIVE-22197 : Common Merge join throwing class cast exception. (Mahesh Kumar Behera reviewed by Vineet Garg)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 33ccc9b HIVE-22197 : Common Merge join throwing class cast exception.
(Mahesh Kumar Behera reviewed by Vineet Garg)
33ccc9b is described below
commit 33ccc9bef82f7acb7fa7cb7ab33a654b1790fd45
Author: Mahesh Kumar Behera
AuthorDate: Fri Sep 27 11:36:17 2019 +0530
HIVE-22197 : Common Merge join throwing class cast exception. (Mahesh Kumar
Behera reviewed by Vineet Garg)
---
.../hadoop/hive/ql/exec/DummyStoreOperator.java| 4 +-
.../clientpositive/merge_test_dummy_operator.q | 120 ++
.../clientpositive/merge_test_dummy_operator.q.out | 419 +
.../tez/merge_test_dummy_operator.q.out| 419 +
.../apache/hadoop/hive/shims/Hadoop23Shims.java| 1 +
5 files changed, 960 insertions(+), 3 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java
index 2c63001..7af70aa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hive.ql.plan.DummyStoreDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
/**
* For SortMerge joins, this is a dummy operator, which stores the row for the
@@ -99,8 +98,7 @@ public class DummyStoreOperator extends
Operator implements Seri
@Override
public void process(Object row, int tag) throws HiveException {
// Store the row. See comments above for why we need a new copy of the row.
-result.o = ObjectInspectorUtils.copyToStandardObject(row,
inputObjInspectors[0],
-ObjectInspectorCopyOption.WRITABLE);
+result.o = ObjectInspectorUtils.copyToStandardObject(row,
inputObjInspectors[0]);
}
@Override
diff --git a/ql/src/test/queries/clientpositive/merge_test_dummy_operator.q
b/ql/src/test/queries/clientpositive/merge_test_dummy_operator.q
new file mode 100644
index 000..f11e22c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/merge_test_dummy_operator.q
@@ -0,0 +1,120 @@
+--! qt:dataset:src
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.auto.convert.join=false;
+
+create table common_join_table (id string,
+col1 string,
+date_created date,
+col2 string,
+col3 string,
+time_stamp timestamp,
+col4 date,
+ col4key bigint,
+col5 date,
+col6 string,
+col7 string,
+col8 smallint);
+
+insert into common_join_table values ('id', '109515', null, 'test', 'test',
'2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test',
'2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test',
'2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test',
'2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test',
'2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test',
'2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test',
'2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test
[hive] branch master updated: HIVE-22092 : Fetch is failing with IllegalArgumentException: No ValidTxnList when refetch is done. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new fba9c20 HIVE-22092 : Fetch is failing with IllegalArgumentException:
No ValidTxnList when refetch is done. (Mahesh Kumar Behera reviewed by Sankar
Hariappan)
fba9c20 is described below
commit fba9c20ce9c9de919fb5d055d796d1e39f59e93e
Author: Mahesh Kumar Behera
AuthorDate: Wed Aug 14 10:10:10 2019 +0530
HIVE-22092 : Fetch is failing with IllegalArgumentException: No
ValidTxnList when refetch is done. (Mahesh Kumar Behera reviewed by Sankar
Hariappan)
---
.../java/org/apache/hive/jdbc/TestJdbcDriver2.java | 35 ++
.../org/apache/hadoop/hive/ql/exec/FetchTask.java | 7 -
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
index 02d4360..4010535 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
@@ -81,6 +81,8 @@ import java.util.regex.Pattern;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.junit.rules.TestName;
+import static java.sql.ResultSet.CONCUR_READ_ONLY;
+import static java.sql.ResultSet.TYPE_SCROLL_INSENSITIVE;
import static org.apache.hadoop.hive.conf.SystemVariables.SET_COLUMN_NAME;
import static org.apache.hadoop.hive.ql.exec.ExplainTask.EXPL_COLUMN_NAME;
import static org.junit.Assert.assertEquals;
@@ -3210,6 +3212,39 @@ public class TestJdbcDriver2 {
stmt1.close();
}
+ @Test
+ public void testResultNextAcidTable() throws Exception {
+Statement stmt = con.createStatement(TYPE_SCROLL_INSENSITIVE,
CONCUR_READ_ONLY);
+try {
+ stmt.execute("set " + ConfVars.HIVE_SUPPORT_CONCURRENCY.varname +
"=true");
+ stmt.execute("set " + ConfVars.HIVE_TXN_MANAGER.varname +
+ "=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
+ stmt.execute("create table tbl (fld int) tblproperties(" +
+
"'transactional'='true','transactional_properties'='insert_only')");
+ stmt.execute("insert into tbl values (1)");
+ stmt.execute("insert into tbl values (2)");
+ stmt.execute("insert into tbl values (3)");
+ ResultSet res = stmt.executeQuery("select * from tbl");
+ assertNotNull(res);
+ int numRows = 0;
+ while (res.next()) {
+numRows++;
+ }
+ assertEquals(numRows, 3);
+ res.beforeFirst();
+ while (res.next()) {
+numRows--;
+ }
+ assertEquals(numRows, 0);
+ stmt.execute("drop table tbl");
+} finally {
+ stmt.execute("set " + ConfVars.HIVE_SUPPORT_CONCURRENCY.varname +
"=false");
+ stmt.execute("set " + ConfVars.HIVE_TXN_MANAGER.varname +
+ "=org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager");
+ stmt.close();
+}
+ }
+
// Test that opening a JDBC connection to a non-existent database throws a
HiveSQLException
@Test(expected = HiveSQLException.class)
public void testConnectInvalidDatabase() throws SQLException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
index caa9d83..93b1158 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
@@ -51,6 +51,7 @@ public class FetchTask extends Task implements
Serializable {
private ListSinkOperator sink;
private int totalRows;
private static transient final Logger LOG =
LoggerFactory.getLogger(FetchTask.class);
+ JobConf job = null;
public FetchTask() {
super();
@@ -68,7 +69,11 @@ public class FetchTask extends Task implements
Serializable {
try {
// Create a file system handle
- JobConf job = new JobConf(conf);
+ if (job == null) {
+// The job config should be initilaized once per fetch task. In case
of refetch, we should use the
+// same config.
+job = new JobConf(conf);
+ }
Operator source = work.getSource();
if (source instanceof TableScanOperator) {
[hive] branch master updated: HIVE-21960 : Disable HMS tasks on replica databases. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0603700 HIVE-21960 : Disable HMS tasks on replica databases.
(Ashutosh Bapat reviewed by Mahesh Kumar Behera)
0603700 is described below
commit 0603700395827acdd819460fe110e35fe7c59f4a
Author: Ashutosh Bapat
AuthorDate: Fri Aug 2 10:14:54 2019 +0530
HIVE-21960 : Disable HMS tasks on replica databases. (Ashutosh Bapat
reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../parse/BaseReplicationScenariosAcidTables.java | 9 ++-
.../parse/TestReplicationScenariosAcidTables.java | 6 +-
.../TestReplicationScenariosAcrossInstances.java | 24 --
.../TestReplicationScenariosExternalTables.java| 42 ++
.../parse/TestTableLevelReplicationScenarios.java | 12 ++-
.../hadoop/hive/ql/parse/WarehouseInstance.java| 18 +
.../ddl/table/creation/CreateTableOperation.java | 9 +++
.../hadoop/hive/ql/parse/ReplicationSpec.java | 5 +-
.../hadoop/hive/ql/stats/StatsUpdaterThread.java | 14 +++-
.../hive/ql/stats/TestStatsUpdaterThread.java | 75 +-
.../apache/hadoop/hive/common/repl/ReplConst.java | 7 ++
.../hive/metastore/PartitionManagementTask.java| 15 +++-
.../hive/metastore/TestPartitionManagement.java| 89 ++
13 files changed, 290 insertions(+), 35 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
index e543695..5e869d2 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
@@ -192,7 +192,8 @@ public class BaseReplicationScenariosAcidTables {
.run("show tables")
.verifyResults(tableNames)
.run("repl status " + replicatedDbName)
-.verifyResult(lastReplId);
+.verifyResult(lastReplId)
+.verifyReplTargetProperty(replicatedDbName);
verifyNonAcidTableLoad(replicatedDbName);
if (includeAcid) {
verifyAcidTableLoad(replicatedDbName);
@@ -295,7 +296,8 @@ public class BaseReplicationScenariosAcidTables {
.run("show tables")
.verifyResults(tableNames)
.run("repl status " + dbName)
-.verifyResult(lastReplId);
+.verifyResult(lastReplId)
+.verifyReplTargetProperty(replicatedDbName);
verifyIncNonAcidLoad(dbName);
verifyIncAcidLoad(dbName);
}
@@ -308,7 +310,8 @@ public class BaseReplicationScenariosAcidTables {
.run("show tables")
.verifyResults(tableNames)
.run("repl status " + dbName)
-.verifyResult(lastReplId);
+.verifyResult(lastReplId)
+.verifyReplTargetProperty(replicatedDbName);
verifyInc2NonAcidLoad(dbName);
verifyInc2AcidLoad(dbName);
}
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
index 96b074d..e23fdd8 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
@@ -512,7 +512,8 @@ public class TestReplicationScenariosAcidTables extends
BaseReplicationScenarios
.run("repl status " + replicatedDbName)
.verifyResult("null")
.run("show tables like t2")
-.verifyResults(new String[] { });
+.verifyResults(new String[] { })
+.verifyReplTargetProperty(replicatedDbName);
// Retry with different dump should fail.
replica.loadFailure(replicatedDbName, tuple2.dumpLocation);
@@ -546,7 +547,8 @@ public class TestReplicationScenariosAcidTables extends
BaseReplicationScenarios
.run("select id from t1")
.verifyResults(Arrays.asList("1"))
.run("select name from t2 order by name")
-.verifyResults(Arrays.asList("bob", "carl"));
+.verifyResults(Arrays.asList("bob", "carl"))
+.verifyReplTargetProperty(replicatedDbName);
}
@Test
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
b/itests/hive-unit/src/test/java/org/apa
[hive] branch master updated: HIVE-22036 : HMS should identify events corresponding to replicated database for Atlas HMS hook. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ba26fcf HIVE-22036 : HMS should identify events corresponding to
replicated database for Atlas HMS hook. (Ashutosh Bapat reviewed by Mahesh
Kumar Behera)
ba26fcf is described below
commit ba26fcfdd6f27c148d8f4ff2e2c4001530cb039b
Author: Ashutosh Bapat
AuthorDate: Tue Jul 30 09:13:38 2019 +0530
HIVE-22036 : HMS should identify events corresponding to replicated
database for Atlas HMS hook. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../parse/ReplMetaStoreEventListenerTestImpl.java | 155
.../ql/parse/TestMetaStoreEventListenerInRepl.java | 199 +
.../hadoop/hive/ql/exec/repl/util/ReplUtils.java | 2 +-
.../apache/hadoop/hive/common/repl/ReplConst.java | 7 +
.../hadoop/hive/metastore/HiveAlterHandler.java| 10 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 51 --
.../hive/metastore/events/AlterDatabaseEvent.java | 10 +-
.../hive/metastore/events/AlterTableEvent.java | 8 +-
.../hive/metastore/events/CreateDatabaseEvent.java | 9 +-
.../hive/metastore/events/CreateTableEvent.java| 11 +-
.../hive/metastore/events/DropDatabaseEvent.java | 9 +-
.../hive/metastore/events/DropTableEvent.java | 10 +-
12 files changed, 459 insertions(+), 22 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplMetaStoreEventListenerTestImpl.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplMetaStoreEventListenerTestImpl.java
new file mode 100644
index 000..31fdb77
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplMetaStoreEventListenerTestImpl.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.events.AlterDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.ListenerEvent;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * ReplMetaStoreEventListenerTestImpl - Implementation of
MetaStoreEventListener to test
+ * isReplicated flag in some of the tests.
+ */
+public class ReplMetaStoreEventListenerTestImpl extends MetaStoreEventListener
{
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ReplMetaStoreEventListenerTestImpl.class);
+ static Map> replicatedDbsForEvents = new HashMap<>();
+ static Map> nonReplicatedDbsForEvents = new HashMap<>();
+ static Map> replicatedTablesForEvents = new HashMap<>();
+
+ public ReplMetaStoreEventListenerTestImpl(Configuration conf) {
+super(conf);
+ }
+
+ private void addNameToEventMap(Map> eventMap, String
name,
+ ListenerEvent event) {
+String eventType = event.getClass().getName();
+Set eventNames = eventMap.get(eventType);
+if (eventNames == null) {
+ eventNames = new HashSet<>();
+ eventMap.put(eventType, eventNames);
+}
+eventNames.add(name.toLowerCase());
+ }
+
+ @Override
+ public void onCreateDatabase(CreateDatabaseEvent createDatabaseEvent) {
+String dbName = createDatabaseEvent.getDatabase().getName().toLowerCase();
+if (createDatabaseEvent.isReplicated()) {
+ addNameToEventMap(replicatedDbsForEvents, dbName, createDatabaseEvent);
+} else {
+ addNameToEventMap(nonReplicatedDbsForEvents, dbName,
createDatabaseEvent);
+
[hive] branch master updated: HIVE-22009 : CTLV with user specified location is not honoured. ( Naresh P R reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 10554ae HIVE-22009 : CTLV with user specified location is not
honoured. ( Naresh P R reviewed by Mahesh Kumar Behera)
10554ae is described below
commit 10554ae8fa3d44b611a716e05a9e9fdda26a44f2
Author: Naresh P R
AuthorDate: Wed Jul 24 10:44:29 2019 +0530
HIVE-22009 : CTLV with user specified location is not honoured. ( Naresh P
R reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../org/apache/hive/jdbc/TestJdbcWithMiniHS2.java | 21 +
.../table/creation/CreateTableLikeOperation.java| 16 +++-
2 files changed, 32 insertions(+), 5 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
index 2e151ec..03a1926 100644
---
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
+++
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
@@ -1676,4 +1676,25 @@ public class TestJdbcWithMiniHS2 {
}
return extendedDescription;
}
+
+ @Test
+ public void testCustomPathsForCTLV() throws Exception {
+try (Statement stmt = conTestDb.createStatement()) {
+ // Initialize
+ stmt.execute("CREATE TABLE emp_table (id int, name string, salary int)");
+ stmt.execute("insert into emp_table values(1,'a',2)");
+ stmt.execute("CREATE VIEW emp_view AS SELECT * FROM emp_table WHERE
salary>1");
+ String customPath = System.getProperty("test.tmp.dir") + "/custom";
+
+ //Test External CTLV
+ String extPath = customPath + "/emp_ext_table";
+ stmt.execute("CREATE EXTERNAL TABLE emp_ext_table like emp_view STORED
AS PARQUET LOCATION '" + extPath + "'");
+ assertTrue(getDetailedTableDescription(stmt,
"emp_ext_table").contains(extPath));
+
+ //Test Managed CTLV
+ String mndPath = customPath + "/emp_mm_table";
+ stmt.execute("CREATE TABLE emp_mm_table like emp_view STORED AS ORC
LOCATION '" + mndPath + "'");
+ assertTrue(getDetailedTableDescription(stmt,
"emp_mm_table").contains(mndPath));
+}
+ }
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/creation/CreateTableLikeOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/creation/CreateTableLikeOperation.java
index 4837d44..8a11261 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/creation/CreateTableLikeOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/creation/CreateTableLikeOperation.java
@@ -94,6 +94,8 @@ public class CreateTableLikeOperation extends
DDLOperation
setExternalProperties(table);
}
+setUserSpecifiedLocation(table);
+
table.setFields(oldTable.getCols());
table.setPartCols(oldTable.getPartCols());
@@ -114,11 +116,7 @@ public class CreateTableLikeOperation extends
DDLOperation
table.setTableName(names[1]);
table.setOwner(SessionState.getUserFromAuthenticator());
-if (desc.getLocation() != null) {
- table.setDataLocation(new Path(desc.getLocation()));
-} else {
- table.unsetDataLocation();
-}
+setUserSpecifiedLocation(table);
setTableParameters(table);
@@ -138,6 +136,14 @@ public class CreateTableLikeOperation extends
DDLOperation
return table;
}
+ private void setUserSpecifiedLocation(Table table) {
+if (desc.getLocation() != null) {
+ table.setDataLocation(new Path(desc.getLocation()));
+} else {
+ table.unsetDataLocation();
+}
+ }
+
private void setTableParameters(Table tbl) throws HiveException {
Set retainer = new HashSet();
[hive] branch master updated: HIVE-21956 : Add the list of table selected by dump in the dump folder. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c7340c6 HIVE-21956 : Add the list of table selected by dump in the
dump folder. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
c7340c6 is described below
commit c7340c6f6e765ef6e499f7a3c399beab843cb6b0
Author: Mahesh Kumar Behera
AuthorDate: Tue Jul 16 12:04:08 2019 +0530
HIVE-21956 : Add the list of table selected by dump in the dump folder.
(Mahesh Kumar Behera reviewed by Sankar Hariappan)
---
.../parse/TestTableLevelReplicationScenarios.java | 107 -
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 102 +---
.../events/filesystem/BootstrapEventsIterator.java | 2 +-
.../events/filesystem/DatabaseEventsIterator.java | 6 +-
.../hadoop/hive/ql/exec/repl/util/ReplUtils.java | 18 +++-
5 files changed, 218 insertions(+), 17 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
index 09db38d..270e61a 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.parse;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
@@ -36,6 +37,11 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.HashSet;
+import java.util.Set;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import static org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables.FILE_NAME;
import static
org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.INC_BOOTSTRAP_ROOT_DIR_NAME;
@@ -153,6 +159,9 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
verifyBootstrapDirInIncrementalDump(tuple.dumpLocation,
bootstrappedTables);
}
+// If the policy contains '.'' means its table level replication.
+verifyTableListForPolicy(tuple.dumpLocation, replPolicy.contains(".'") ?
expectedTables : null);
+
replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
.run("use " + replicatedDbName)
.run("show tables")
@@ -194,6 +203,36 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
}
}
+ private void verifyTableListForPolicy(String dumpLocation, String[]
tableList) throws Throwable {
+FileSystem fileSystem = primary.miniDFSCluster.getFileSystem();
+Path tableListFile = new Path(dumpLocation,
ReplUtils.REPL_TABLE_LIST_DIR_NAME);
+tableListFile = new Path(tableListFile, primaryDbName.toLowerCase());
+
+if (tableList == null) {
+ Assert.assertFalse(fileSystem.exists(tableListFile));
+ return;
+} else {
+ Assert.assertTrue(fileSystem.exists(tableListFile));
+}
+
+BufferedReader reader = null;
+try {
+ InputStream inputStream = fileSystem.open(tableListFile);
+ reader = new BufferedReader(new InputStreamReader(inputStream));
+ Set tableNames = new HashSet<>(Arrays.asList(tableList));
+ int numTable = 0;
+ for (String line = reader.readLine(); line != null; line =
reader.readLine()) {
+numTable++;
+Assert.assertTrue(tableNames.contains(line));
+ }
+ Assert.assertEquals(numTable, tableList.length);
+} finally {
+ if (reader != null) {
+reader.close();
+ }
+}
+ }
+
@Test
public void testBasicBootstrapWithIncludeList() throws Throwable {
String[] originalNonAcidTables = new String[] {"t1", "t2"};
@@ -660,7 +699,7 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
.run("alter table out100 rename to in100") // this will add the
bootstrap
.run("drop table in100"); // table in100 is dropped, so no
bootstrap should happen.
-replicatedTables = new String[] {"in200", "in12", "in12", "in14"};
+replicatedTables = new String[] {"in200", "in12", "in11", "in14"};
bootstrapTables = new String[] {"in14", "in200"};
replicateAndVerify(replPolicy, null, lastReplId, null,
null, bootstrapTables, replicatedTables);
@@ -90
[hive] branch master updated: HIVE-21958 : The list of table expression in the inclusion and exclusion list should be separated by '|' instead of comma. (Mahesh Kumar Behera reviewed by Sankar Hariapp
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e2a8375 HIVE-21958 : The list of table expression in the inclusion
and exclusion list should be separated by '|' instead of comma. (Mahesh Kumar
Behera reviewed by Sankar Hariappan)
e2a8375 is described below
commit e2a8375fe3e8e3c601c770556b104bb47debd33e
Author: mbehera
AuthorDate: Wed Jul 10 09:09:20 2019 +0530
HIVE-21958 : The list of table expression in the inclusion and exclusion
list should be separated by '|' instead of comma. (Mahesh Kumar Behera
reviewed by Sankar Hariappan)
---
.../parse/TestReplicationWithTableMigrationEx.java | 4 +-
.../parse/TestTableLevelReplicationScenarios.java | 100 +++--
.../plugin/TestHiveAuthorizerCheckInvocation.java | 2 +-
.../org/apache/hadoop/hive/ql/parse/HiveParser.g | 18 +---
.../hive/ql/parse/ReplicationSemanticAnalyzer.java | 25 ++
.../hive/ql/parse/repl/load/DumpMetaData.java | 68 +-
.../ql/parse/TestReplicationSemanticAnalyzer.java | 20 ++---
.../apache/hadoop/hive/common/repl/ReplScope.java | 55 +---
8 files changed, 114 insertions(+), 178 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
index 3a86cb7..f0f1ce5 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
@@ -221,14 +221,14 @@ public class TestReplicationWithTableMigrationEx {
WarehouseInstance.Tuple tuple = primary
.run("use " + primaryDbName)
.run("create table t1 (i int, j int)")
-.dump(primaryDbName+".['t1']", null);
+.dump(primaryDbName+".'t1'", null);
replica.run("create database " + replicatedDbName);
replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation);
assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters()));
tuple = primary.run("use " + primaryDbName)
.run("insert into t1 values (1, 2)")
-.dump(primaryDbName+".['t1']", tuple.lastReplicationId);
+.dump(primaryDbName+".'t1'", tuple.lastReplicationId);
replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation);
assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters()));
}
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
index a3f56f7..09db38d 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
@@ -204,7 +204,7 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
createTables(originalMMAcidTables, CreateTableType.MM_ACID);
// Replicate and verify if only 2 tables are replicated to target.
-String replPolicy = primaryDbName + ".['t1', 't4', 't5']";
+String replPolicy = primaryDbName + ".'(t1)|(t4)|(t5)'";
String[] replicatedTables = new String[] {"t1", "t4", "t5"};
replicateAndVerify(replPolicy, null, null, null, replicatedTables);
}
@@ -215,7 +215,7 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
createTables(originalTables, CreateTableType.NON_ACID);
// Replicate and verify if only 3 tables are replicated to target.
-String replPolicy = primaryDbName + ".['t1*', 't3'].['t100']";
+String replPolicy = primaryDbName + ".'(t1*)|(t3)'.'t100'";
String[] replicatedTables = new String[] {"t1", "t11", "t3"};
replicateAndVerify(replPolicy, null, null, null, replicatedTables);
}
@@ -234,7 +234,7 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
createTables(originalMMAcidTables, CreateTableType.MM_ACID);
// Replicate and verify if only 2 tables are replicated to target.
-String replPolicy = primaryDbName + ".['t1', 't5']";
+String replPolicy = primaryDbNa
[hive] branch master updated: HIVE-21880 : Enable flaky test TestReplicationScenariosAcidTablesBootstrap.testBootstrapAcidTablesDuringIncrementalWithConcurrentWrites. (Ashutosh Bapat reviewed by Mahes
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8e64482 HIVE-21880 : Enable flaky test
TestReplicationScenariosAcidTablesBootstrap.testBootstrapAcidTablesDuringIncrementalWithConcurrentWrites.
(Ashutosh Bapat reviewed by Mahesh Kumar Behera)
8e64482 is described below
commit 8e6448213c35361567f29808da6e74997494c5b9
Author: Ashutosh Bapat
AuthorDate: Tue Jul 9 07:56:28 2019 +0530
HIVE-21880 : Enable flaky test
TestReplicationScenariosAcidTablesBootstrap.testBootstrapAcidTablesDuringIncrementalWithConcurrentWrites.
(Ashutosh Bapat reviewed by Mahesh Kumar Behera)
Signed-off-by: mbehera
---
.../hcatalog/listener/DbNotificationListener.java | 10 +++
...estReplicationScenariosAcidTablesBootstrap.java | 2 --
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 25
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 9 ++
.../apache/hadoop/hive/metastore/ObjectStore.java | 33 ++
5 files changed, 59 insertions(+), 20 deletions(-)
diff --git
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
index af01178..5f9d809 100644
---
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
+++
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants;
import org.apache.hadoop.hive.metastore.RawStore;
@@ -997,6 +998,14 @@ public class DbNotificationListener extends
TransactionalMetaStoreEventListener
stmt.execute("SET @@session.sql_mode=ANSI_QUOTES");
}
+ // Derby doesn't allow FOR UPDATE to lock the row being selected (See
https://db.apache
+ // .org/derby/docs/10.1/ref/rrefsqlj31783.html) . So lock the whole
table. Since there's
+ // only one row in the table, this shouldn't cause any performance
degradation.
+ if (sqlGenerator.getDbProduct() == DatabaseProduct.DERBY) {
+String lockingQuery = "lock table \"NOTIFICATION_SEQUENCE\" in
exclusive mode";
+LOG.info("Going to execute query <" + lockingQuery + ">");
+stmt.executeUpdate(lockingQuery);
+ }
String s = sqlGenerator.addForUpdateClause("select \"NEXT_EVENT_ID\" " +
" from \"NOTIFICATION_SEQUENCE\"");
LOG.debug("Going to execute query <" + s + ">");
@@ -1081,6 +1090,7 @@ public class DbNotificationListener extends
TransactionalMetaStoreEventListener
String.join(", ", params) + ")");
pst.execute();
+ event.setEventId(nextEventId);
// Set the DB_NOTIFICATION_EVENT_ID for future reference by other
listeners.
if (event.isSetEventId()) {
listenerEvent.putParameter(
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTablesBootstrap.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTablesBootstrap.java
index 94070e5..f475b1e 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTablesBootstrap.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTablesBootstrap.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.BeforeClass;
@@ -256,7 +255,6 @@ public class TestReplicationScenariosAcidTablesBootstrap
verifyCompactionQueue(tables, replicatedDbName, replicaConf);
}
- @Ignore("HIVE-21879: Disabling this testcase as it is flaky.")
@Test
public void testBootstrapAcidTablesDuringIncrementalWithConcurrentWrites()
throws Throwable {
// Dump and load bootstrap without ACID tables.
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index f947ee1..153f4b8 100644
---
a/standalone-metastore/metastore-com
[hive] branch master updated: HIVE-21886 : REPL - With table list - Handle rename events during replace policy. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new be6bf93 HIVE-21886 : REPL - With table list - Handle rename events
during replace policy. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
be6bf93 is described below
commit be6bf933a141d1b8bd6141a942b096949c4a5a4e
Author: mbehera
AuthorDate: Sun Jun 30 16:28:06 2019 +0530
HIVE-21886 : REPL - With table list - Handle rename events during replace
policy. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
---
.../parse/TestTableLevelReplicationScenarios.java | 313 ++---
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 1 +
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 2 +-
.../parse/repl/dump/events/AlterTableHandler.java | 83 --
.../parse/repl/dump/events/CommitTxnHandler.java | 6 +-
.../ptest2/conf/deployed/master-mr2.properties | 3 +-
6 files changed, 286 insertions(+), 122 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
index 546678b..a3f56f7 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
@@ -196,27 +196,27 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
@Test
public void testBasicBootstrapWithIncludeList() throws Throwable {
-String[] originalNonAcidTables = new String[] {"t1", "t2" };
-String[] originalFullAcidTables = new String[] {"t3", "t4" };
-String[] originalMMAcidTables = new String[] {"t5" };
+String[] originalNonAcidTables = new String[] {"t1", "t2"};
+String[] originalFullAcidTables = new String[] {"t3", "t4"};
+String[] originalMMAcidTables = new String[] {"t5"};
createTables(originalNonAcidTables, CreateTableType.NON_ACID);
createTables(originalFullAcidTables, CreateTableType.FULL_ACID);
createTables(originalMMAcidTables, CreateTableType.MM_ACID);
// Replicate and verify if only 2 tables are replicated to target.
String replPolicy = primaryDbName + ".['t1', 't4', 't5']";
-String[] replicatedTables = new String[] {"t1", "t4", "t5" };
+String[] replicatedTables = new String[] {"t1", "t4", "t5"};
replicateAndVerify(replPolicy, null, null, null, replicatedTables);
}
@Test
public void testBasicBootstrapWithIncludeAndExcludeList() throws Throwable {
-String[] originalTables = new String[] {"t1", "t11", "t2", "t3", "t100" };
+String[] originalTables = new String[] {"t1", "t11", "t2", "t3", "t100"};
createTables(originalTables, CreateTableType.NON_ACID);
// Replicate and verify if only 3 tables are replicated to target.
String replPolicy = primaryDbName + ".['t1*', 't3'].['t100']";
-String[] replicatedTables = new String[] {"t1", "t11", "t3" };
+String[] replicatedTables = new String[] {"t1", "t11", "t3"};
replicateAndVerify(replPolicy, null, null, null, replicatedTables);
}
@@ -226,16 +226,16 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
.dump(primaryDbName, null);
replica.load(replicatedDbName, tupleBootstrap.dumpLocation);
-String[] originalNonAcidTables = new String[] {"t1", "t2" };
-String[] originalFullAcidTables = new String[] {"t3", "t4" };
-String[] originalMMAcidTables = new String[] {"t5" };
+String[] originalNonAcidTables = new String[] {"t1", "t2"};
+String[] originalFullAcidTables = new String[] {"t3", "t4"};
+String[] originalMMAcidTables = new String[] {"t5"};
createTables(originalNonAcidTables, CreateTableType.NON_ACID);
createTables(originalFullAcidTables, CreateTableType.FULL_ACID);
createTables(originalMMAcidTables, CreateTableType.MM_ACID);
// Replicate and verify if only 2 tables are replicated to target.
String replPolicy = primaryDbName + ".['t1', 't5']";
-String[] replicatedTables = new String[] {"t1", "t5" };
+String[] replicatedTables = new String[] {"t1", "t5&quo
[hive] branch master updated: HIVE-21764 : REPL DUMP should detect and bootstrap any rename table events where old table was excluded but renamed table is included. (Mahesh Kumar Behera reviewed by Sa
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 11f7856 HIVE-21764 : REPL DUMP should detect and bootstrap any rename
table events where old table was excluded but renamed table is included.
(Mahesh Kumar Behera reviewed by Sankar Hariappan)
11f7856 is described below
commit 11f78562ab36333cc1d0a3f6051d9846c9c92132
Author: mbehera
AuthorDate: Sat Jun 22 16:41:28 2019 +0530
HIVE-21764 : REPL DUMP should detect and bootstrap any rename table events
where old table was excluded but renamed table is included. (Mahesh Kumar
Behera reviewed by Sankar Hariappan)
---
.../parse/TestTableLevelReplicationScenarios.java | 205 -
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 30 ++-
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 3 -
.../apache/hadoop/hive/ql/parse/repl/DumpType.java | 6 +
.../hive/ql/parse/repl/dump/TableExport.java | 3 +-
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 39 ++--
.../ql/parse/repl/dump/events/AbortTxnHandler.java | 4 +
.../events/AbstractConstraintEventHandler.java | 1 +
.../repl/dump/events/AddPartitionHandler.java | 4 +-
.../repl/dump/events/AlterPartitionHandler.java| 4 +-
.../parse/repl/dump/events/AlterTableHandler.java | 79 +++-
.../parse/repl/dump/events/CommitTxnHandler.java | 3 +
.../parse/repl/dump/events/CreateTableHandler.java | 4 +-
.../parse/repl/dump/events/DropTableHandler.java | 7 +
.../ql/parse/repl/dump/events/EventHandler.java| 22 ++-
.../ql/parse/repl/dump/events/InsertHandler.java | 4 +-
.../ql/parse/repl/dump/events/OpenTxnHandler.java | 4 +
.../repl/dump/events/UpdatePartColStatHandler.java | 4 +-
.../dump/events/UpdateTableColStatHandler.java | 4 +-
.../ql/parse/repl/dump/io/TableSerializer.java | 3 +-
.../parse/repl/load/message/DropTableHandler.java | 17 +-
.../messaging/event/filters/ReplEventFilter.java | 9 +-
22 files changed, 410 insertions(+), 49 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
index f67fc81..546678b 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
@@ -117,7 +117,8 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
List dumpWithClause,
List loadWithClause,
String[] expectedTables) throws Throwable {
-return replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause,
loadWithClause, null, expectedTables);
+return replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause,
loadWithClause,
+null, expectedTables);
}
private String replicateAndVerify(String replPolicy, String oldReplPolicy,
String lastReplId,
@@ -125,6 +126,16 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
List loadWithClause,
String[] bootstrappedTables,
String[] expectedTables) throws Throwable {
+return replicateAndVerify(replPolicy, oldReplPolicy, lastReplId,
dumpWithClause, loadWithClause,
+bootstrappedTables, expectedTables, null);
+ }
+
+ private String replicateAndVerify(String replPolicy, String oldReplPolicy,
String lastReplId,
+List dumpWithClause,
+List loadWithClause,
+String[] bootstrappedTables,
+String[] expectedTables,
+String[] records) throws Throwable {
if (dumpWithClause == null) {
dumpWithClause = new ArrayList<>();
}
@@ -138,7 +149,7 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
}
WarehouseInstance.Tuple tuple = primary.dump(replPolicy, oldReplPolicy,
lastReplId, dumpWithClause);
-if (oldReplPolicy != null) {
+if (bootstrappedTables != null) {
verifyBootstrapDirInIncrementalDump(tuple.dumpLocation,
bootstrappedTables);
}
@@ -146,6 +157,15 @@ public class TestTableLevelReplicationScenarios extends
BaseReplicationScenarios
.run("use " + replicatedDbName)
.run("show tables")
.verifyResults(expectedTables);
+
+if (records == null) {
+ records = new String[] {&qu
[hive] branch master updated: HIVE-21841 : Leader election in HMS to run housekeeping tasks. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c3d0502 HIVE-21841 : Leader election in HMS to run housekeeping
tasks. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
c3d0502 is described below
commit c3d05022f79ed8ef6c61ca9c088f6a8a992d6e8d
Author: Ashutosh Bapat
AuthorDate: Thu Jun 20 09:48:36 2019 +0530
HIVE-21841 : Leader election in HMS to run housekeeping tasks. (Ashutosh
Bapat reviewed by Mahesh Kumar Behera)
Signed-off-by: mbehera
---
.../MetastoreHousekeepingLeaderTestBase.java | 192 +
.../MetastoreTaskThreadAlwaysTestImpl.java | 60 +++
.../RemoteMetastoreTaskThreadTestImpl1.java| 60 +++
.../RemoteMetastoreTaskThreadTestImpl2.java| 60 +++
.../metastore/TestMetastoreHousekeepingLeader.java | 61 +++
...TestMetastoreHousekeepingLeaderEmptyConfig.java | 62 +++
.../TestMetastoreHousekeepingNonLeader.java| 71
.../hadoop/hive/ql/stats/StatsUpdaterThread.java | 5 +-
.../hadoop/hive/metastore/ReplChangeManager.java | 3 +-
.../hadoop/hive/metastore/conf/MetastoreConf.java | 9 +
.../hadoop/hive/metastore/HiveMetaStore.java | 103 ---
.../hadoop/hive/metastore/MetaStoreTestUtils.java | 35 +++-
.../hive/metastore/TestMarkPartitionRemote.java| 2 +-
13 files changed, 686 insertions(+), 37 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/MetastoreHousekeepingLeaderTestBase.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/MetastoreHousekeepingLeaderTestBase.java
new file mode 100644
index 000..d89d67c
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/MetastoreHousekeepingLeaderTestBase.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.apache.hadoop.hive.ql.stats.StatsUpdaterThread;
+import org.apache.hadoop.hive.ql.txn.compactor.Cleaner;
+import org.apache.hadoop.hive.ql.txn.compactor.Initiator;
+import org.apache.hadoop.hive.ql.txn.compactor.Worker;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Base class for HMS leader config testing.
+ */
+class MetastoreHousekeepingLeaderTestBase {
+ private static final Logger LOG =
LoggerFactory.getLogger(MetastoreHousekeepingLeaderTestBase.class);
+ private static HiveMetaStoreClient client;
+ private static Configuration conf = MetastoreConf.newMetastoreConf();
+ private static Warehouse warehouse;
+ private static boolean isServerStarted = false;
+ private static int port;
+ private static MiniDFSCluster miniDFS;
+ // How long should we wait for the housekeeping threads to start in ms.
+ private static final long SLEEP_INTERVAL_FOR_THREADS_TO_START = 1;
+ // Threads using ThreadPool will start after the configured interval. So,
start them some time
+ // before we check the existence of threads.
+ private static final long REMOTE_TASKS_INTERVAL =
SLEEP_INTERVAL_FOR_THREADS_TO_START - 3000;
+ static final String METASTORE_THREAD_TASK_FREQ_CONF =
"metastore.leader.test.task.freq";
+
+ static Map threadNames = new HashMap<>();
+ static Map threadClasses = new HashMap<>();
+
+ void internalSetup(final String leaderHostName) throws Exception {
+MetaStoreTestUtils.setConfForStandloneMode(conf);
+MetastoreConf.setVar(conf, ConfVars.THRIFT_BIND_HOST, "localhost");
+MetastoreConf.setVar(conf,
ConfVars.METASTORE_HOUSEKEEPING_LEADER_HOSTNAME,
[hive] branch master updated: HIVE-21844 : HMS schema Upgrade Script is failing with NPE. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9c00ee0 HIVE-21844 : HMS schema Upgrade Script is failing with NPE.
(Mahesh Kumar Behera reviewed by Sankar Hariappan)
9c00ee0 is described below
commit 9c00ee0d3043a7fdb7576baf8aea0805439eeb0a
Author: mbehera
AuthorDate: Mon Jun 10 21:32:28 2019 +0530
HIVE-21844 : HMS schema Upgrade Script is failing with NPE. (Mahesh Kumar
Behera reviewed by Sankar Hariappan)
---
beeline/src/java/org/apache/hive/beeline/ColorBuffer.java | 3 +++
beeline/src/java/org/apache/hive/beeline/Commands.java| 4 +++-
.../src/main/sql/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql | 4 ++--
.../src/main/sql/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql | 4 ++--
.../src/main/sql/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql | 4 ++--
.../src/main/sql/mysql/upgrade-2.2.0-to-2.3.0.mysql.sql | 4 ++--
.../src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 4 ++--
.../src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql | 4 ++--
.../src/main/sql/mysql/upgrade-3.1.0-to-3.2.0.mysql.sql | 4 ++--
.../src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql | 4 ++--
10 files changed, 22 insertions(+), 17 deletions(-)
diff --git a/beeline/src/java/org/apache/hive/beeline/ColorBuffer.java
b/beeline/src/java/org/apache/hive/beeline/ColorBuffer.java
index 3bb6436..1730d49 100644
--- a/beeline/src/java/org/apache/hive/beeline/ColorBuffer.java
+++ b/beeline/src/java/org/apache/hive/beeline/ColorBuffer.java
@@ -78,6 +78,9 @@ final class ColorBuffer implements Comparable {
}
ColorBuffer center(String str, int len) {
+if (str == null) {
+ str = "";
+}
StringBuilder buf = new StringBuilder(str);
while (buf.length() < len) {
buf.append(" ");
diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java
b/beeline/src/java/org/apache/hive/beeline/Commands.java
index 90cae9f..fd0af2c 100644
--- a/beeline/src/java/org/apache/hive/beeline/Commands.java
+++ b/beeline/src/java/org/apache/hive/beeline/Commands.java
@@ -1052,7 +1052,9 @@ public class Commands {
logThread.interrupt();
}
logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
- showRemainingLogsIfAny(stmnt);
+ if (stmnt != null) {
+showRemainingLogsIfAny(stmnt);
+ }
}
if (stmnt != null) {
stmnt.close();
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
index 1d85e2c..20ddd1a 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
@@ -1,4 +1,4 @@
-SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS ' ';
+SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS MESSAGE;
--SOURCE 021-HIVE-7018.mysql.sql;
ALTER TABLE `TBLS` DROP FOREIGN KEY `TBLS_FK3`;
ALTER TABLE `TBLS` DROP KEY `TBLS_N51`;
@@ -71,5 +71,5 @@ CREATE TABLE AUX_TABLE (
UPDATE VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release
version 2.0.0' where VER_ID=1;
-SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS ' ';
+SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS MESSAGE;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql
b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql
index 456d4cd..22a3c37 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql
@@ -1,4 +1,4 @@
-SELECT 'Upgrading MetaStore schema from 2.0.0 to 2.1.0' AS ' ';
+SELECT 'Upgrading MetaStore schema from 2.0.0 to 2.1.0' AS MESSAGE;
--SOURCE 034-HIVE-13076.mysql.sql;
CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS`
@@ -38,5 +38,5 @@ ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES
varchar(2048);
ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES varchar(2048);
UPDATE VERSION SET SCHEMA_VERSION='2.1.0', VERSION_COMMENT='Hive release
version 2.1.0' where VER_ID=1;
-SELECT 'Finished upgrading MetaStore schema from 2.0.0 to 2.1.0' AS ' ';
+SELECT 'Finished upgrading MetaStore schema from 2.0.0 to 2.1.0' AS MESSAGE;
diff --git
a/standalone-metastore/metastore-server
[hive] branch master updated: HIVE-21852 : Testcases are randomly failing with ArrayIndexOutOfBoundException. (Naresh P R reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ec37799 HIVE-21852 : Testcases are randomly failing with
ArrayIndexOutOfBoundException. (Naresh P R reviewed by Mahesh Kumar Behera)
ec37799 is described below
commit ec3779978797051fdb345172536aafcd50f1b4ae
Author: Naresh P R
AuthorDate: Mon Jun 10 11:42:22 2019 +0530
HIVE-21852 : Testcases are randomly failing with
ArrayIndexOutOfBoundException. (Naresh P R reviewed by Mahesh Kumar Behera)
Signed-off-by: mbehera
---
.../metastore/MetastoreDefaultTransformer.java | 51 +++---
1 file changed, 25 insertions(+), 26 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
index 7ecbef8..4b543c8 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.metastore;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_NONE;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READONLY;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READWRITE;
-import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_WRITEONLY;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -62,25 +61,25 @@ public class MetastoreDefaultTransformer implements
IMetaStoreMetadataTransforme
HIVECACHEINVALIDATE,
CONNECTORWRITE));
- private static List ACIDWRITELIST = new
ArrayList<>(Arrays.asList(HIVEFULLACIDWRITE));
- private static List INSERTONLYWRITELIST = new
ArrayList<>(Arrays.asList(HIVEMANAGEDINSERTWRITE));
+ private List acidWriteList = new
ArrayList<>(Arrays.asList(HIVEFULLACIDWRITE));
+ private List insertOnlyWriteList = new
ArrayList<>(Arrays.asList(HIVEMANAGEDINSERTWRITE));
private static List MQTLIST = new
ArrayList<>(Arrays.asList(HIVEFULLACIDREAD, HIVEONLYMQTWRITE,
HIVEMANAGESTATS, HIVEMQT, CONNECTORREAD));
- private static List ACIDLIST = new ArrayList<>();
- private static List INSERTONLYLIST = new ArrayList<>();
+ private List acidList = new ArrayList<>();
+ private List insertOnlyList = new ArrayList<>();
public MetastoreDefaultTransformer(IHMSHandler handler) throws
HiveMetaException {
this.hmsHandler = handler;
-ACIDWRITELIST.addAll(ACIDCOMMONWRITELIST);
-ACIDLIST.addAll(ACIDWRITELIST);
-ACIDLIST.add(HIVEFULLACIDREAD);
-ACIDLIST.add(CONNECTORREAD);
+acidWriteList.addAll(ACIDCOMMONWRITELIST);
+acidList.addAll(acidWriteList);
+acidList.add(HIVEFULLACIDREAD);
+acidList.add(CONNECTORREAD);
-INSERTONLYWRITELIST.addAll(ACIDCOMMONWRITELIST);
-INSERTONLYLIST.addAll(INSERTONLYWRITELIST);
-INSERTONLYLIST.add(HIVEMANAGEDINSERTREAD);
-INSERTONLYLIST.add(CONNECTORREAD);
+insertOnlyWriteList.addAll(ACIDCOMMONWRITELIST);
+insertOnlyList.addAll(insertOnlyWriteList);
+insertOnlyList.add(HIVEMANAGEDINSERTREAD);
+insertOnlyList.add(CONNECTORREAD);
}
@Override
@@ -129,7 +128,7 @@ public class MetastoreDefaultTransformer implements
IMetaStoreMetadataTransforme
String txnal = params.get("transactional");
if (txnal == null || txnal.equalsIgnoreCase("FALSE")) { //
non-ACID MANAGED table
table.setAccessType(ACCESSTYPE_READONLY);
- generated.addAll(ACIDWRITELIST);
+ generated.addAll(acidWriteList);
}
if (txnal != null && txnal.equalsIgnoreCase("TRUE")) { // ACID
table
@@ -137,7 +136,7 @@ public class MetastoreDefaultTransformer implements
IMetaStoreMetadataTransforme
processorCapabilities.contains(HIVEFULLACIDREAD) ||
processorCapabilities.contains(HIVEMANAGEDINSERTREAD))) {
table.setAccessType(ACCESSTYPE_NONE); // clients have no
access to ACID tables without capabilities
-generated.addAll(ACIDLIST);
+generated.addAll(acidList);
}
String txntype = params.get("transactional_properties");
@@ -145,30 +144,30 @@ public class MetastoreDefaultTransformer implements
IMetaStoreMetadataTransforme
// MGD table is insert only, not full ACID
[hive] branch master updated: HIVE-21731 : Hive import fails, post upgrade of source 3.0 cluster, to a target 4.0 cluster with strict managed table set to true. (Mahesh Kumar Behera reviewed by Sankar
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8b8e702 HIVE-21731 : Hive import fails, post upgrade of source 3.0
cluster, to a target 4.0 cluster with strict managed table set to true. (Mahesh
Kumar Behera reviewed by Sankar Hariappan)
8b8e702 is described below
commit 8b8e7029169f486abd8da5786b6fd5eb1d2aba86
Author: Mahesh Kumar Behera
AuthorDate: Mon May 20 09:41:26 2019 +0530
HIVE-21731 : Hive import fails, post upgrade of source 3.0 cluster, to a
target 4.0 cluster with strict managed table set to true. (Mahesh Kumar Behera
reviewed by Sankar Hariappan)
---
.../TestReplicationScenariosExternalTables.java| 29 --
.../parse/TestReplicationWithTableMigration.java | 67 ++
.../hadoop/hive/ql/parse/WarehouseInstance.java| 6 +-
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 2 +-
.../exec/repl/bootstrap/load/table/LoadTable.java | 17 +-
.../parse/repl/dump/events/AlterTableHandler.java | 9 +++
6 files changed, 96 insertions(+), 34 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
index c6f8e40..c069831 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
@@ -663,35 +663,6 @@ public class TestReplicationScenariosExternalTables
extends BaseReplicationAcros
}
@Test
- public void
retryIncBootstrapExternalTablesFromDifferentDumpWithoutCleanTablesConfig()
throws Throwable {
-List dumpWithClause = Collections.singletonList(
-"'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname +
"'='false'"
-);
-List loadWithClause = externalTableBasePathWithClause();
-
-WarehouseInstance.Tuple tupleBootstrapWithoutExternal = primary
-.dump(primaryDbName, null, dumpWithClause);
-
-replica.load(replicatedDbName, tupleBootstrapWithoutExternal.dumpLocation,
loadWithClause);
-
-dumpWithClause = Arrays.asList("'" +
HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
-"'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname +
"'='true'");
-WarehouseInstance.Tuple tupleIncWithExternalBootstrap = primary.run("use "
+ primaryDbName)
-.run("create external table t1 (id int)")
-.run("insert into table t1 values (1)")
-.run("create table t2 as select * from t1")
-.dump(primaryDbName,
tupleBootstrapWithoutExternal.lastReplicationId, dumpWithClause);
-WarehouseInstance.Tuple tupleNewIncWithExternalBootstrap
-= primary.dump(primaryDbName,
tupleBootstrapWithoutExternal.lastReplicationId, dumpWithClause);
-
-replica.load(replicatedDbName, tupleIncWithExternalBootstrap.dumpLocation,
loadWithClause);
-
-// Re-bootstrapping from different bootstrap dump without clean tables
config should fail.
-replica.loadFailure(replicatedDbName,
tupleNewIncWithExternalBootstrap.dumpLocation, loadWithClause,
-ErrorMsg.REPL_BOOTSTRAP_LOAD_PATH_NOT_VALID.getErrorCode());
- }
-
- @Test
public void testExternalTableDataPath() throws Exception {
HiveConf conf = primary.getConf();
Path basePath = new Path("/");
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java
index 41f2b9d..a5904d6 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.parse.repl.PathBuilder;
import static
org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION;
+import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.After;
@@ -114,6 +115,8 @@ public class TestReplicationWithTableMigration {
put("hive.support.concurrency", "false");
put("hive.txn.manager",
"org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager");
put("hive.strict.managed.tables", "false");
[hive] branch master updated: HIVE-21722 : REPL:: logs are missing in hiveStatement.getQueryLog output during parallel execution mode. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new de69a0f HIVE-21722 : REPL:: logs are missing in
hiveStatement.getQueryLog output during parallel execution mode. (Mahesh Kumar
Behera reviewed by Sankar Hariappan)
de69a0f is described below
commit de69a0f9d973822895f3ff0744ee3af768118268
Author: Mahesh Kumar Behera
AuthorDate: Tue May 14 14:01:32 2019 +0530
HIVE-21722 : REPL:: logs are missing in hiveStatement.getQueryLog output
during parallel execution mode. (Mahesh Kumar Behera reviewed by Sankar
Hariappan)
---
.../java/org/apache/hive/jdbc/TestJdbcDriver2.java | 129 -
.../org/apache/hadoop/hive/ql/exec/TaskRunner.java | 2 +
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 2 +-
.../hadoop/hive/ql/exec/repl/ReplStateLogTask.java | 7 ++
.../ql/exec/repl/bootstrap/load/LoadFunction.java | 2 +-
.../incremental/IncrementalLoadTasksBuilder.java | 2 +-
6 files changed, 139 insertions(+), 5 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
index 8f19b7d..654bdf8 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
@@ -20,6 +20,7 @@ package org.apache.hive.jdbc;
import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
@@ -27,6 +28,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.exec.repl.ReplDumpWork;
import org.apache.hadoop.hive.ql.processors.DfsProcessor;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
import org.apache.hive.common.util.HiveVersionInfo;
@@ -77,6 +79,7 @@ import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.junit.rules.TestName;
import static org.apache.hadoop.hive.conf.SystemVariables.SET_COLUMN_NAME;
import static org.apache.hadoop.hive.ql.exec.ExplainTask.EXPL_COLUMN_NAME;
@@ -121,6 +124,7 @@ public class TestJdbcDriver2 {
private static final float floatCompareDelta = 0.0001f;
@Rule public ExpectedException thrown = ExpectedException.none();
+ @Rule public final TestName testName = new TestName();
private static Connection getConnection(String postfix) throws SQLException {
Connection con1;
@@ -2767,10 +2771,10 @@ public class TestJdbcDriver2 {
incrementalLogs.addAll(statement.getQueryLog());
Thread.sleep(500);
} catch (SQLException e) {
-LOG.error("Failed getQueryLog. Error message: " + e.getMessage());
+LOG.info("Failed getQueryLog. Error message: " + e.getMessage());
fail("error in getting log thread");
} catch (InterruptedException e) {
-LOG.error("Getting log thread is interrupted. Error message: " +
e.getMessage());
+LOG.info("Getting log thread is interrupted. Error message: " +
e.getMessage());
fail("error in getting log thread");
}
}
@@ -2793,6 +2797,126 @@ public class TestJdbcDriver2 {
verifyFetchedLog(incrementalLogs, expectedLogs);
}
+ private static int next = 0;
+ private synchronized void advanceDumpDir() {
+next++;
+ReplDumpWork.injectNextDumpDirForTest(String.valueOf(next));
+ }
+
+ /**
+ * Test getting query log method in Jdbc for REPL commands
+ * @throws Exception
+ */
+ @Test
+ public void testGetQueryLogForReplCommands() throws Exception {
+// Prepare
+String primaryDb = testName.getMethodName() + "_" +
System.currentTimeMillis();
+String replicaDb = primaryDb + "_replica";
+String primaryTblName = primaryDb + ".t1";
+Path replDir = new Path(conf.get("test.data.files"));
+HiveStatement stmt = (HiveStatement) con.createStatement();
+assertNotNull("Statement is null", stmt);
+
+replDir = new Path(replDir, primaryDb + "_repl");
+FileSystem fs = FileSystem.get(replDir.toUri(), conf);
+fs.mkdirs(replDir);
+
+try {
+ // Prepare
+ stmt.execute("set hive.exec.parallel = true");
+ stmt.execute("set hive.server2.logging.operation.level = execution");
+ stmt.execute("set hive.metastore.transactional.event.list
[hive] branch master updated: HIVE-21717 : Rename is failing for directory in move task. (Mahesh Kumar Behera reviewed by Sankar Hariappan
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5c425dc HIVE-21717 : Rename is failing for directory in move task.
(Mahesh Kumar Behera reviewed by Sankar Hariappan
5c425dc is described below
commit 5c425dca2fc95ed718d13513de7a7319e20c5261
Author: Mahesh Kumar Behera
AuthorDate: Mon May 13 11:33:29 2019 +0530
HIVE-21717 : Rename is failing for directory in move task. (Mahesh Kumar
Behera reviewed by Sankar Hariappan
---
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 40 --
1 file changed, 15 insertions(+), 25 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 55ae535..27ec7dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -4301,6 +4301,19 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
}
+ private static void deleteAndRename(FileSystem destFs, Path destFile,
FileStatus srcStatus, Path destPath)
+ throws IOException {
+if (destFs.exists(destFile)) {
+ // rename cannot overwrite non empty destination directory, so deleting
the destination before renaming.
+ destFs.delete(destFile);
+ LOG.info("Deleting destination file" + destFile.toUri());
+}
+if(!destFs.rename(srcStatus.getPath(), destFile)) {
+ throw new IOException("rename for src path: " + srcStatus.getPath() + "
to dest:"
+ + destPath + " returned false");
+}
+ }
+
//it is assumed that parent directory of the destf should already exist when
this
//method is called. when the replace value is true, this method works a
little different
//from mv command if the destf is a directory, it replaces the destf instead
of moving under
@@ -4386,37 +4399,14 @@ private void constructOneLBLocationMap(FileStatus fSta,
"Unable to move source " + srcStatus.getPath() + " to
destination " + destFile;
if (null == pool) {
-boolean success = false;
-if (destFs instanceof DistributedFileSystem) {
- ((DistributedFileSystem)destFs).rename(srcStatus.getPath(),
destFile, Options.Rename.OVERWRITE);
- success = true;
-} else {
- destFs.delete(destFile, false);
- success = destFs.rename(srcStatus.getPath(), destFile);
-}
-if(!success) {
- throw new IOException("rename for src path: " +
srcStatus.getPath() + " to dest:"
- + destf + " returned false");
-}
+deleteAndRename(destFs, destFile, srcStatus, destf);
} else {
futures.add(pool.submit(new Callable() {
@Override
public Void call() throws HiveException {
SessionState.setCurrentSessionState(parentSession);
try {
- boolean success = false;
- if (destFs instanceof DistributedFileSystem) {
-
((DistributedFileSystem)destFs).rename(srcStatus.getPath(), destFile,
Options.Rename.OVERWRITE);
-success = true;
- } else {
-destFs.delete(destFile, false);
-success = destFs.rename(srcStatus.getPath(), destFile);
- }
- if (!success) {
-throw new IOException(
-"rename for src path: " + srcStatus.getPath() + "
to dest path:"
-+ destFile + " returned false");
- }
+ deleteAndRename(destFs, destFile, srcStatus, destf);
} catch (Exception e) {
throw getHiveException(e, poolMsg);
}
[hive] branch master updated: HIVE-20967 : Handle alter events when replicate to cluster with hive.strict.managed.tables enabled. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9a06923 HIVE-20967 : Handle alter events when replicate to cluster
with hive.strict.managed.tables enabled. (Ashutosh Bapat reviewed by Mahesh
Kumar Behera)
9a06923 is described below
commit 9a0692330104d610e8102f509f061a2cc8498e18
Author: Ashutosh Bapat
AuthorDate: Thu May 9 11:09:11 2019 +0530
HIVE-20967 : Handle alter events when replicate to cluster with
hive.strict.managed.tables enabled. (Ashutosh Bapat reviewed by Mahesh Kumar
Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../parse/BaseReplicationScenariosAcidTables.java | 6 +-
.../TestReplicationScenariosExternalTables.java| 17 +
.../parse/TestReplicationWithTableMigrationEx.java | 42 +++
.../org/apache/hadoop/hive/ql/exec/DDLTask.java| 4 +-
.../clientnegative/strict_managed_tables6.q| 10 +++
.../clientnegative/strict_managed_tables6.q.out| 30
.../apache/hadoop/hive/metastore/HiveMetaHook.java | 2 +
.../hadoop/hive/metastore/HiveAlterHandler.java| 85 +++---
8 files changed, 179 insertions(+), 17 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
index bfc4abd..e543695 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
@@ -140,10 +140,8 @@ public class BaseReplicationScenariosAcidTables {
"\"transactional_properties\"=\"insert_only\")")
.run("insert into t3 values(11)")
.run("insert into t3 values(22)")
-.run("create table t5 (id int) stored as orc ")
-.run("insert into t5 values(), ()")
-.run("alter table t5 set tblproperties
(\"transactional\"=\"true\")")
-.run("insert into t5 values()");
+.run("create table t5 (id int) stored as orc tblproperties
(\"transactional\"=\"true\")")
+.run("insert into t5 values(), (), ()");
acidTableNames.add("t1");
acidTableNames.add("t2");
acidTableNames.add("t3");
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
index 0b7b82e..74854db 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
@@ -364,6 +364,23 @@ public class TestReplicationScenariosExternalTables
extends BaseReplicationAcros
.run("use " + replicatedDbName)
.run("select place from t2 where country='france'")
.verifyResults(new String[] {});
+
+// Changing location of one of the partitions shouldn't result in changing
location of other
+// partitions as well as that of the table.
+assertTablePartitionLocation(primaryDbName + ".t2", replicatedDbName +
".t2");
+
+// Changing location of the external table, should result in changes to
the location of
+// partition residing within the table location and not the partitions
located outside.
+String tmpLocation2 = "/tmp/" + System.nanoTime() + "_2";
+primary.miniDFSCluster.getFileSystem().mkdirs(new Path(tmpLocation2), new
FsPermission("777"));
+
+tuple = primary.run("use " + primaryDbName)
+.run("insert into table t2 partition(country='france') values
('lyon')")
+.run("alter table t2 set location '" + tmpLocation2 + "'")
+.dump(primaryDbName, tuple.lastReplicationId);
+
+replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause);
+assertTablePartitionLocation(primaryDbName + ".t2", replicatedDbName +
".t2");
}
@Test
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
index 5f59a2a..7619718 100644
---
a/itests/hive-unit/src/te
[hive] branch master updated: HIVE-21678 : CTAS creating a Full ACID partitioned table fails because of no writeId. (Ashutosh Bapat reviewed by Mahesh Kumar Behera, Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 71985c2 HIVE-21678 : CTAS creating a Full ACID partitioned table
fails because of no writeId. (Ashutosh Bapat reviewed by Mahesh Kumar Behera,
Sankar Hariappan)
71985c2 is described below
commit 71985c23b9c01a764d88274f21f9ce014e5e1a7e
Author: Ashutosh Bapat
AuthorDate: Wed May 8 14:34:01 2019 +0530
HIVE-21678 : CTAS creating a Full ACID partitioned table fails because of
no writeId. (Ashutosh Bapat reviewed by Mahesh Kumar Behera, Sankar Hariappan)
Signed-off-by: Mahesh Kumar Behera
---
.../hadoop/hive/ql/parse/ReplicationTestUtils.java | 4 +--
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +++--
ql/src/test/queries/clientpositive/ctas.q | 18 +-
ql/src/test/results/clientpositive/ctas.q.out | 40 ++
ql/src/test/results/clientpositive/llap/ctas.q.out | 40 ++
.../test/results/clientpositive/spark/ctas.q.out | 40 ++
6 files changed, 137 insertions(+), 13 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java
index bde6570..4caa884 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java
@@ -136,10 +136,10 @@ public class ReplicationTestUtils {
String tableNameCTAS = tableName + "_CTAS";
String tableNameCTASMM = tableName + "_CTASMM";
-/*insertRecords(primary, primaryDbName, primaryDbNameExtra,
+insertRecords(primary, primaryDbName, primaryDbNameExtra,
tableName, tableNameCTAS, false,
OperationType.REPL_TEST_ACID_CTAS);
selectStmtList.add("select key from " + tableNameCTAS + " order by key");
-expectedValues.add(new String[]{"1", "2", "3", "4", "5"});*/
+expectedValues.add(new String[]{"1", "2", "3", "4", "5"});
insertRecords(primary, primaryDbName, primaryDbNameExtra,
tableNameMM, tableNameCTASMM, true,
OperationType.REPL_TEST_ACID_CTAS);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index f436611..a3da075 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -7477,8 +7477,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
fileSinkColInfos = new ArrayList<>();
destTableIsTemporary = tblDesc.isTemporary();
destTableIsMaterialization = tblDesc.isMaterialization();
-if (AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) {
- isMmTable = isMmCtas = true;
+if (AcidUtils.isTablePropertyTransactional(tblDesc.getTblProps())) {
try {
if (ctx.getExplainConfig() != null) {
writeId = 0L; // For explain plan, txn won't be opened and
doesn't make sense to allocate write id
@@ -7498,7 +7497,10 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
} catch (LockException ex) {
throw new SemanticException("Failed to allocate write Id", ex);
}
- tblDesc.setInitialMmWriteId(writeId);
+ if (AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) {
+isMmTable = isMmCtas = true;
+tblDesc.setInitialMmWriteId(writeId);
+ }
}
} else if (viewDesc != null) {
fieldSchemas = new ArrayList<>();
diff --git a/ql/src/test/queries/clientpositive/ctas.q
b/ql/src/test/queries/clientpositive/ctas.q
index c4fdda1..cd0bf5a 100644
--- a/ql/src/test/queries/clientpositive/ctas.q
+++ b/ql/src/test/queries/clientpositive/ctas.q
@@ -61,11 +61,13 @@ create table nzhang_ctas6 (key string, `to` string);
insert overwrite table nzhang_ctas6 select key, value from src tablesample (10
rows);
create table nzhang_ctas7 as select key, `to` from nzhang_ctas6;
-
-
-
-
-
-
-
-
+-- ACID CTAS
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.stats.autogather=false;
+
+create table acid_ctas_part partitioned by (k)
+ stored as orc TBLPROPERTIES ('transactional'='true')
+ as select key k, value from src order by k limit 5;
+select k, value from acid_ctas_part;
diff --git a/ql/src/test/results/clien
[hive] branch master updated: HIVE-21700 : Hive incremental load going OOM while adding load task to the leaf nodes of the DAG. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 875ba82 HIVE-21700 : Hive incremental load going OOM while adding
load task to the leaf nodes of the DAG. (Mahesh Kumar Behera, reviewed by
Sankar Hariappan)
875ba82 is described below
commit 875ba8231b824a500bd0844613285aadd64e651e
Author: Mahesh Kumar Behera
AuthorDate: Wed May 8 11:26:29 2019 +0530
HIVE-21700 : Hive incremental load going OOM while adding load task to the
leaf nodes of the DAG. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
---
.../hadoop/hive/ql/exec/util/DAGTraversal.java | 20 ++--
1 file changed, 18 insertions(+), 2 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/util/DAGTraversal.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/util/DAGTraversal.java
index cb5dc2e..40f5f55 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/util/DAGTraversal.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/util/DAGTraversal.java
@@ -23,7 +23,9 @@ import org.apache.hadoop.hive.ql.exec.Task;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* The dag traversal done here is written to be not recursion based as large
DAG's will lead to
@@ -33,7 +35,21 @@ public class DAGTraversal {
public static void traverse(List> tasks,
Function function) {
List> listOfTasks = new ArrayList<>(tasks);
while (!listOfTasks.isEmpty()) {
- List> children = new ArrayList<>();
+ // HashSet will make sure that no duplicate children are added. If a
task is added multiple
+ // time to the children list then it may cause the list to grow
exponentially. Lets take an example of
+ // incremental load with 2 events. The DAG will look some thing similar
as below.
+ //
+ // --->ev1.task1--
--->ev2.task1--
+ /// \/
\
+ // evTaskRoot-->*>ev1.task2---*-->
ev1.barrierTask-->*>ev2.task2---*->ev2.barrierTask---
+ //\ /
+ // --->ev1.task3--
+ //
+ // While traversing the DAG, if the filter is not added then
ev1.barrierTask will be added 3 times in
+ // the children list and in next iteration ev2.task1 will be added 3
times and ev2.task2 will be added
+ // 3 times. So in next iteration ev2.barrierTask will be added 6 times.
As it goes like this, the next barrier
+ // task will be added 12-15 times and may reaches millions with large
number of events.
+ Set> children = new HashSet<>();
for (Task task : listOfTasks) {
// skip processing has to be done first before continuing
if (function.skipProcessing(task)) {
@@ -48,7 +64,7 @@ public class DAGTraversal {
}
function.process(task);
}
- listOfTasks = children;
+ listOfTasks = new ArrayList<>(children);
}
}
[hive] branch master updated: HIVE-21694 : Hive driver wait time is fixed for task getting executed in parallel. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5c2c838 HIVE-21694 : Hive driver wait time is fixed for task getting
executed in parallel. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
5c2c838 is described below
commit 5c2c838930f9ab695209dfde3b265918475e2f07
Author: Mahesh Kumar Behera
AuthorDate: Wed May 8 08:43:08 2019 +0530
HIVE-21694 : Hive driver wait time is fixed for task getting executed in
parallel. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
---
.../java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java| 2 +-
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 2 +-
ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java | 5 +
ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java | 7 ++-
4 files changed, 13 insertions(+), 3 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index aeafe85..01f8cda 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -293,7 +293,7 @@ public class WarehouseInstance implements Closeable {
}
WarehouseInstance loadWithoutExplain(String replicatedDbName, String
dumpLocation) throws Throwable {
-run("REPL LOAD " + replicatedDbName + " FROM '" + dumpLocation + "'");
+run("REPL LOAD " + replicatedDbName + " FROM '" + dumpLocation + "' with
('hive.exec.parallel'='true')");
return this;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 3a20130..255c65a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -2692,7 +2692,7 @@ public class Driver implements IDriver {
console.printInfo("Launching Job " + cxt.getCurJobNo() + " out of " +
jobs);
}
tsk.initialize(queryState, plan, cxt, ctx.getOpContext());
-TaskRunner tskRun = new TaskRunner(tsk);
+TaskRunner tskRun = new TaskRunner(tsk, cxt);
cxt.launching(tskRun);
// Launch Task
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
index 2dd83fb..d5392ab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
@@ -99,6 +99,11 @@ public class DriverContext {
return null;
}
+ public synchronized void releaseRunnable() {
+//release the waiting poller.
+notify();
+ }
+
/**
* Polls running tasks to see if a task has ended.
*
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
index 8904139..6f6d721 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.slf4j.Logger;
@@ -46,10 +47,13 @@ public class TaskRunner extends Thread {
private static transient final Logger LOG =
LoggerFactory.getLogger(TaskRunner.class);
- public TaskRunner(Task tsk) {
+ private final DriverContext driverCtx;
+
+ public TaskRunner(Task tsk, DriverContext ctx) {
this.tsk = tsk;
this.result = new TaskResult();
ss = SessionState.get();
+driverCtx = ctx;
}
public Task getTask() {
@@ -102,6 +106,7 @@ public class TaskRunner extends Thread {
LOG.error("Error in executeTask", t);
}
result.setExitVal(exitVal);
+driverCtx.releaseRunnable();
if (tsk.getException() != null) {
result.setTaskError(tsk.getException());
}
[hive] branch master updated: HIVE-21598 : CTAS on ACID table during incremental does not replicate data (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new df5a8f1 HIVE-21598 : CTAS on ACID table during incremental does not
replicate data (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
df5a8f1 is described below
commit df5a8f124174f580df51d6217822852fcc3fb1ee
Author: Ashutosh Bapat
AuthorDate: Fri May 3 09:25:44 2019 +0530
HIVE-21598 : CTAS on ACID table during incremental does not replicate data
(Ashutosh Bapat reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera
---
.../parse/BaseReplicationScenariosAcidTables.java | 387 +
...estReplAcidTablesBootstrapWithJsonMessage.java} | 11 +-
.../parse/TestReplAcidTablesWithJsonMessage.java | 2 -
.../parse/TestReplicationScenariosAcidTables.java | 626 +
...estReplicationScenariosAcidTablesBootstrap.java | 334 +++
.../hive/ql/ddl/table/CreateTableOperation.java| 26 +
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 4 +-
.../ptest2/conf/deployed/master-mr2.properties | 4 +-
8 files changed, 782 insertions(+), 612 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
new file mode 100644
index 000..bfc4abd
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.ql.DriverFactory;
+import org.apache.hadoop.hive.ql.IDriver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.Utils;
+
+import org.junit.rules.TestName;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.AfterClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Collections;
+import java.util.Map;
+
+import static
org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION;
+
+/**
+ * TestReplicationScenariosAcidTablesBase - base class for replication for
ACID tables tests
+ */
+public class BaseReplicationScenariosAcidTables {
+
+ @Rule
+ public final TestName testName = new TestName();
+
+ protected static final Logger LOG =
LoggerFactory.getLogger(TestReplicationScenarios.class);
+ static WarehouseInstance primary;
+ static WarehouseInstance replica, replicaNonAcid;
+ static HiveConf conf;
+ String primaryDbName, replicatedDbName;
+ List acidTableNames = new LinkedList<>();
+ private List nonAcidTableNames = new LinkedList<>();
+
+ static void internalBeforeClassSetup(Map overrides, Class
clazz)
+ throws Exception {
+conf = new HiveConf(clazz);
+conf.set("dfs.client.use.datanode.hostname", "true");
+conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() +
".hosts", "*");
+MiniDFSCluster miniDFSCluster =
+new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
+HashMap acidEnableConf = new HashMap() {{
+ put("fs.defaultFS", miniDFSCluster.getFileSystem().getUri().toString());
+ put("hive.support.concu
[hive] branch master updated: HIVE-21430 : INSERT into a dynamically partitioned table with hive.stats.autogather = false throws a MetaException. (Ashutosh Bapat, reviewed by Mahesh Kumar Behera )
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4c87512 HIVE-21430 : INSERT into a dynamically partitioned table with
hive.stats.autogather = false throws a MetaException. (Ashutosh Bapat, reviewed
by Mahesh Kumar Behera )
4c87512 is described below
commit 4c875127f6f07761af4e1ee146f7bc13738b6276
Author: Ashutosh Bapat
AuthorDate: Fri Mar 22 12:15:52 2019 +0530
HIVE-21430 : INSERT into a dynamically partitioned table with
hive.stats.autogather = false throws a MetaException. (Ashutosh Bapat, reviewed
by Mahesh Kumar Behera )
Signed-off-by: Mahesh Kumar Behera
---
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 42 +-
.../hive/ql/stats/TestStatsUpdaterThread.java | 25 +
2 files changed, 51 insertions(+), 16 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 33d157d..4350dc8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1882,6 +1882,20 @@ public class Hive {
return getDatabase(currentDb);
}
+ private TableSnapshot getTableSnapshot(Table tbl, Long writeId) throws
LockException {
+TableSnapshot tableSnapshot = null;
+if ((writeId != null) && (writeId > 0)) {
+ ValidWriteIdList writeIds =
AcidUtils.getTableValidWriteIdListWithTxnList(
+ conf, tbl.getDbName(), tbl.getTableName());
+ tableSnapshot = new TableSnapshot(writeId, writeIds.writeToString());
+} else {
+ // Make sure we pass in the names, so we can get the correct snapshot
for rename table.
+ tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, tbl.getDbName(),
tbl.getTableName(),
+ true);
+}
+return tableSnapshot;
+ }
+
/**
* Load a directory into a Hive Table Partition - Alters existing content of
* the partition with the contents of loadPath. - If the partition does not
@@ -1935,17 +1949,7 @@ public class Hive {
inheritLocation, isSkewedStoreAsSubdir, isSrcLocal,
isAcidIUDoperation,
resetStatistics, writeId, stmtId, isInsertOverwrite, isTxnTable,
newFiles);
-AcidUtils.TableSnapshot tableSnapshot = null;
-if (isTxnTable) {
- if ((writeId != null) && (writeId > 0)) {
-ValidWriteIdList writeIds =
AcidUtils.getTableValidWriteIdListWithTxnList(
-conf, tbl.getDbName(), tbl.getTableName());
-tableSnapshot = new TableSnapshot(writeId, writeIds.writeToString());
- } else {
-// Make sure we pass in the names, so we can get the correct snapshot
for rename table.
-tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, tbl.getDbName(),
tbl.getTableName(), true);
- }
-}
+AcidUtils.TableSnapshot tableSnapshot = isTxnTable ? getTableSnapshot(tbl,
writeId) : null;
if (tableSnapshot != null) {
newTPart.getTPartition().setWriteId(tableSnapshot.getWriteId());
}
@@ -2363,7 +2367,8 @@ public class Hive {
private void setStatsPropAndAlterPartitions(boolean resetStatistics, Table
tbl,
List partitions,
- long writeId) throws TException {
+ AcidUtils.TableSnapshot
tableSnapshot)
+ throws TException {
if (partitions.isEmpty()) {
return;
}
@@ -2377,9 +2382,15 @@ public class Hive {
LOG.debug(sb.toString());
}
+String validWriteIdList = null;
+long writeId = 0L;
+if (tableSnapshot != null) {
+ validWriteIdList = tableSnapshot.getValidWriteIdList();
+ writeId = tableSnapshot.getWriteId();
+}
getSynchronizedMSC().alter_partitions(tbl.getCatName(), tbl.getDbName(),
tbl.getTableName(),
partitions.stream().map(Partition::getTPartition).collect(Collectors.toList()),
-ec, null, writeId);
+ec, validWriteIdList, writeId);
}
/**
@@ -2626,6 +2637,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
boolean isTxnTable = AcidUtils.isTransactionalTable(tbl);
+AcidUtils.TableSnapshot tableSnapshot = isTxnTable ? getTableSnapshot(tbl,
writeId) : null;
for (Entry entry : partitionDetailsMap.entrySet())
{
tasks.add(() -> {
@@ -2645,8 +2657,6 @@ private void constructOneLBLocationMap(FileStatus fSta,
// if the partition already existed before the loading, no need to
add it again to the
// metastore
- AcidUtils.TableSnapshot tableSnapshot =
AcidUtils.getTableSnapshot(conf,
- partition.getTable(), true);
[hive] branch master updated: HIVE-21446 : Hive Server going OOM during hive external table replications. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 38682a4 HIVE-21446 : Hive Server going OOM during hive external table
replications. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
38682a4 is described below
commit 38682a414708d810937012bae4b0c97deca5ef07
Author: Mahesh Kumar Behera
AuthorDate: Thu Mar 21 20:27:38 2019 +0530
HIVE-21446 : Hive Server going OOM during hive external table replications.
(Mahesh Kumar Behera, reviewed by Sankar Hariappan)
---
.../org/apache/hadoop/hive/common/FileUtils.java | 10 +-
.../apache/hadoop/hive/common/TestFileUtils.java | 11 +-
.../apache/hadoop/hive/ql/exec/ReplCopyTask.java | 2 +-
.../ql/exec/repl/ExternalTableCopyTaskBuilder.java | 86
.../hadoop/hive/ql/parse/repl/CopyUtils.java | 150 ++---
.../hive/ql/parse/repl/dump/io/FileOperations.java | 5 +-
.../hadoop/hive/ql/parse/repl/TestCopyUtils.java | 8 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java| 5 +-
.../org/apache/hadoop/hive/shims/HadoopShims.java | 5 +-
9 files changed, 182 insertions(+), 100 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index 23a3a6b..8b03faa 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -636,18 +636,18 @@ public final class FileUtils {
}
public static boolean distCp(FileSystem srcFS, List srcPaths, Path dst,
- boolean deleteSource, String doAsUser,
+ boolean deleteSource, UserGroupInformation proxyUser,
HiveConf conf, HadoopShims shims) throws IOException {
LOG.debug("copying srcPaths : {}, to DestPath :{} ,with doAs: {}",
-StringUtils.join(",", srcPaths), dst.toString(), doAsUser);
+StringUtils.join(",", srcPaths), dst.toString(), proxyUser);
boolean copied = false;
-if (doAsUser == null){
+if (proxyUser == null){
copied = shims.runDistCp(srcPaths, dst, conf);
} else {
- copied = shims.runDistCpAs(srcPaths, dst, conf, doAsUser);
+ copied = shims.runDistCpAs(srcPaths, dst, conf, proxyUser);
}
if (copied && deleteSource) {
- if (doAsUser != null) {
+ if (proxyUser != null) {
// if distcp is done using doAsUser, delete also should be done using
same user.
//TODO : Need to change the delete execution within doAs if doAsUser
is given.
throw new IOException("Distcp is called with doAsUser and delete
source set as true");
diff --git a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
index b45832e..9b5748e 100644
--- a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
+++ b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
import org.junit.Test;
@@ -239,14 +240,16 @@ public class TestFileUtils {
FileSystem fs = copySrc.getFileSystem(conf);
String doAsUser = conf.getVar(HiveConf.ConfVars.HIVE_DISTCP_DOAS_USER);
+UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
+doAsUser, UserGroupInformation.getLoginUser());
HadoopShims shims = mock(HadoopShims.class);
-when(shims.runDistCpAs(Collections.singletonList(copySrc), copyDst, conf,
doAsUser)).thenReturn(true);
+when(shims.runDistCpAs(Collections.singletonList(copySrc), copyDst, conf,
proxyUser)).thenReturn(true);
when(shims.runDistCp(Collections.singletonList(copySrc), copyDst,
conf)).thenReturn(false);
// doAs when asked
-Assert.assertTrue(FileUtils.distCp(fs, Collections.singletonList(copySrc),
copyDst, false, doAsUser, conf, shims));
-verify(shims).runDistCpAs(Collections.singletonList(copySrc), copyDst,
conf, doAsUser);
+Assert.assertTrue(FileUtils.distCp(fs, Collections.singletonList(copySrc),
copyDst, false, proxyUser, conf, shims));
+verify(shims).runDistCpAs(Collections.singletonList(copySrc), copyDst,
conf, proxyUser);
// don't doAs when not asked
Assert.assertFalse(FileUtils.distCp(fs,
Collections.singletonList(copySrc), copyDst, true, null, conf, shims));
verify(shims).runDistCp(Collections.singletonList(copySrc), copyDst, conf);
@@ -254,7 +257,7 @@ public class TestFileUtils {
// When distcp is done with doAs, the delete should also be done as doAs.
But in current code its broken. This
// should be fixed. For no
[hive] branch master updated: HIVE-21325 : Hive external table replication failed with Permission denied issue. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4274f48 HIVE-21325 : Hive external table replication failed with
Permission denied issue. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
4274f48 is described below
commit 4274f48a250db9624a48421147fa5700dc255e86
Author: Mahesh Kumar Behera
AuthorDate: Mon Mar 11 17:02:28 2019 +0530
HIVE-21325 : Hive external table replication failed with Permission denied
issue. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
---
...BasedMetastoreAuthorizationProviderWithACL.java | 11 ++-
.../TestMetastoreAuthorizationProvider.java| 18
.../ql/exec/repl/ExternalTableCopyTaskBuilder.java | 107 +++--
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 49 --
.../StorageBasedAuthorizationProvider.java | 22 +
.../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
6 files changed, 165 insertions(+), 44 deletions(-)
diff --git
a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java
b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java
index 62c109c..c9253c1 100644
---
a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java
+++
b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java
@@ -49,7 +49,12 @@ public class
TestStorageBasedMetastoreAuthorizationProviderWithACL
protected static Path warehouseDir = null;
protected UserGroupInformation userUgi = null;
protected String testUserName = "test_user";
+ protected String proxyUserName = null;
+ @Override
+ protected String getProxyUserName() {
+return proxyUserName;
+ }
@Override
protected boolean isTestEnabled() {
@@ -74,10 +79,10 @@ public class
TestStorageBasedMetastoreAuthorizationProviderWithACL
// Hadoop FS ACLs do not work with LocalFileSystem, so set up MiniDFS.
HiveConf conf = super.createHiveConf();
-String currentUserName = Utils.getUGI().getShortUserName();
+proxyUserName = Utils.getUGI().getShortUserName();
conf.set("dfs.namenode.acls.enabled", "true");
-conf.set("hadoop.proxyuser." + currentUserName + ".groups", "*");
-conf.set("hadoop.proxyuser." + currentUserName + ".hosts", "*");
+conf.set("hadoop.proxyuser." + proxyUserName + ".groups", "*");
+conf.set("hadoop.proxyuser." + proxyUserName + ".hosts", "*");
dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
FileSystem fs = dfs.getFileSystem();
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
index 0e08e81..0fc677b 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.Collections;
import junit.framework.TestCase;
@@ -85,6 +86,10 @@ public class TestMetastoreAuthorizationProvider extends
TestCase {
return new HiveConf(this.getClass());
}
+ protected String getProxyUserName() {
+return null;
+ }
+
@Override
protected void setUp() throws Exception {
@@ -304,6 +309,19 @@ public class TestMetastoreAuthorizationProvider extends
TestCase {
ret = driver.run("alter table "+tblName+" add partition (b='2011')");
assertEquals(0,ret.getResponseCode());
+String proxyUserName = getProxyUserName();
+if (proxyUserName != null) {
+ // for storage based authorization, user having proxy privilege should
be allowed to do operation
+ // even if the file permission is not there.
+ InjectableDummyAuthenticator.injectUserName(proxyUserName);
+
InjectableDummyAuthenticator.injectGroupNames(Collections.singletonList(proxyUserName));
+ InjectableDummyAuthenticator.injectMode(true);
+ disallowCreateInTbl(tbl.getTableName(), proxyUserName,
tbl.getSd().getLocation());
+ ret = driver.run("alter table "+tblName+" add partition (b='2012')");
+ assertEquals(0, ret.getResponseCode());
+ InjectableDummyAuth
[hive] branch master updated: HIVE-21314 : Hive Replication not retaining the owner in the replicated table. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0413fec HIVE-21314 : Hive Replication not retaining the owner in the
replicated table. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
0413fec is described below
commit 0413fec095c9282fa97a097f1736880a0caebebe
Author: Mahesh Kumar Behera
AuthorDate: Wed Mar 6 16:51:38 2019 +0530
HIVE-21314 : Hive Replication not retaining the owner in the replicated
table. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
---
.../parse/TestReplicationWithTableMigrationEx.java | 67 ++
.../org/apache/hadoop/hive/ql/exec/DDLTask.java| 3 +
.../bootstrap/events/filesystem/FSTableEvent.java | 3 +
.../hive/ql/parse/ImportSemanticAnalyzer.java | 3 +-
.../hadoop/hive/ql/plan/CreateTableDesc.java | 13 +
.../apache/hadoop/hive/ql/plan/CreateViewDesc.java | 13 +
.../hadoop/hive/ql/plan/ImportTableDesc.java | 13 +
7 files changed, 114 insertions(+), 1 deletion(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
index 4637da1..5f59a2a 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
@@ -348,4 +348,71 @@ public class TestReplicationWithTableMigrationEx {
replica.load(replicatedDbName, tuple.dumpLocation, withClause);
assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters()));
}
+
+ private void verifyUserName(String userName) throws Throwable {
+assertTrue(userName.equalsIgnoreCase(primary.getTable(primaryDbName,
"tbl_own").getOwner()));
+assertTrue(userName.equalsIgnoreCase(replica.getTable(replicatedDbName,
"tbl_own").getOwner()));
+assertTrue(userName.equalsIgnoreCase(primary.getTable(primaryDbName,
"tacid").getOwner()));
+assertTrue(userName.equalsIgnoreCase(replica.getTable(replicatedDbName,
"tacid").getOwner()));
+assertTrue(userName.equalsIgnoreCase(primary.getTable(primaryDbName,
"tacidpart").getOwner()));
+assertTrue(userName.equalsIgnoreCase(replica.getTable(replicatedDbName,
"tacidpart").getOwner()));
+assertTrue(userName.equalsIgnoreCase(primary.getTable(primaryDbName,
"tbl_part").getOwner()));
+assertTrue(userName.equalsIgnoreCase(replica.getTable(replicatedDbName,
"tbl_part").getOwner()));
+assertTrue(userName.equalsIgnoreCase(primary.getTable(primaryDbName,
"view_own").getOwner()));
+assertTrue(userName.equalsIgnoreCase(replica.getTable(replicatedDbName,
"view_own").getOwner()));
+ }
+
+ private void alterUserName(String userName) throws Throwable {
+primary.run("use " + primaryDbName)
+.run("alter table tbl_own set owner USER " + userName)
+.run("alter table tacid set owner USER " + userName)
+.run("alter table tacidpart set owner USER " + userName)
+.run("alter table tbl_part set owner USER " + userName)
+.run("alter table view_own set owner USER " + userName);
+ }
+
+ @Test
+ public void testOnwerPropagation() throws Throwable {
+primary.run("use " + primaryDbName)
+.run("create table tbl_own (fld int)")
+.run("create table tacid (id int) clustered by(id) into 3 buckets
stored as orc ")
+.run("create table tacidpart (place string) partitioned by
(country string) clustered by(place) "
++ "into 3 buckets stored as orc ")
+.run("create table tbl_part (fld int) partitioned by (country
string)")
+.run("insert into tbl_own values (1)")
+.run("create view view_own as select * from tbl_own");
+
+// test bootstrap
+alterUserName("hive");
+WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, null);
+replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation);
+verifyUserName("hive");
+
+// test incremental
+alterUserName("hive1");
+tuple = primary.dump(primaryDbName, tuple.lastReplicationId);
+replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation);
+verifyUserName("hive1");
+ }
+
+ @Test
+ public void testOnwerPropagationInc() throws Throwable {
+WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, null);
+replica.loadWithoutExplain
[hive] branch master updated: HIVE-21197 : Hive replication can add duplicate data during migration to a target with hive.strict.managed.tables enabled (Mahesh Kumar Behera, reviewed by Sankar Hariapp
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 20abb4e HIVE-21197 : Hive replication can add duplicate data during
migration to a target with hive.strict.managed.tables enabled (Mahesh Kumar
Behera, reviewed by Sankar Hariappan)
20abb4e is described below
commit 20abb4e06b6a613a9b039f5de1b3bbcbefaf
Author: Mahesh Kumar Behera
AuthorDate: Tue Feb 26 19:22:55 2019 +0530
HIVE-21197 : Hive replication can add duplicate data during migration to a
target with hive.strict.managed.tables enabled (Mahesh Kumar Behera, reviewed
by Sankar Hariappan)
---
.../TestReplicationScenariosAcrossInstances.java | 18 +-
.../parse/TestReplicationWithTableMigrationEx.java | 351 +
.../hadoop/hive/ql/parse/WarehouseInstance.java| 8 +-
.../hive/ql/txn/compactor/TestCompactor.java | 56
.../org/apache/hadoop/hive/ql/exec/DDLTask.java| 36 +++
.../apache/hadoop/hive/ql/exec/ReplCopyTask.java | 119 +--
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 20 +-
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 5 +-
.../ql/exec/repl/bootstrap/load/LoadDatabase.java | 23 +-
.../repl/bootstrap/load/table/TableContext.java| 9 +
.../incremental/IncrementalLoadTasksBuilder.java | 7 +
.../hadoop/hive/ql/exec/repl/util/ReplUtils.java | 15 +
.../org/apache/hadoop/hive/ql/parse/EximUtil.java | 3 +-
.../hive/ql/parse/ImportSemanticAnalyzer.java | 36 ++-
.../hadoop/hive/ql/parse/ReplicationSpec.java | 11 +
.../ql/parse/repl/dump/io/TableSerializer.java | 3 +-
.../repl/load/message/AlterDatabaseHandler.java| 3 +-
.../parse/repl/load/message/CommitTxnHandler.java | 5 +
.../org/apache/hadoop/hive/ql/plan/DDLWork.java| 16 +
.../apache/hadoop/hive/ql/plan/ReplCopyWork.java | 10 +
.../plan/ReplRemoveFirstIncLoadPendFlagDesc.java | 65
.../hive/ql/txn/compactor/CompactorThread.java | 9 +
.../hadoop/hive/ql/txn/compactor/Initiator.java| 12 +
.../ql/txn/compactor/MetaStoreCompactorThread.java | 14 +
.../ql/txn/compactor/RemoteCompactorThread.java| 14 +
.../metastore/InjectableBehaviourObjectStore.java | 25 ++
26 files changed, 837 insertions(+), 56 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 3639ab1..69d2648 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -610,6 +610,10 @@ public class TestReplicationScenariosAcrossInstances
extends BaseReplicationAcro
.run("show tables")
.verifyResults(new String[] { "t1" });
+
assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase("default").getParameters()));
+
assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase(primaryDbName).getParameters()));
+
assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase(dbOne).getParameters()));
+
replica.load("", incrementalTuple.dumpLocation)
.run("show databases")
.verifyResults(new String[] { "default", primaryDbName, dbOne, dbTwo })
@@ -620,6 +624,11 @@ public class TestReplicationScenariosAcrossInstances
extends BaseReplicationAcro
.run("show tables")
.verifyResults(new String[] { "t1", "t2" });
+
assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase("default").getParameters()));
+
assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(primaryDbName).getParameters()));
+
assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(dbOne).getParameters()));
+
assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(dbTwo).getParameters()));
+
/*
Start of cleanup
*/
@@ -1012,7 +1021,7 @@ public class TestReplicationScenariosAcrossInstances
extends BaseReplicationAcro
.dump(primaryDbName, null);
// Bootstrap Repl A -> B
-WarehouseInstance.Tuple tupleReplica = replica.load(replicatedDbName,
tuplePrimary.dumpLocation)
+replica.load(replicatedDbName, tuplePrimary.dumpLocation)
.run("repl status " + replicatedDbName)
.verifyResult(tuplePrimary.lastReplicationId)
.run("show tblproperties t1('custom.property')")
@@ -1020,9 +1029,14 @@ public class TestReplicationScenariosAcrossInstances
extends BaseReplicationAcro
.dumpFailure(replicatedDbName, null)
[hive] 02/02: HIVE-21260 : Hive replication to a target with hive.strict.managed.tables enabled is failing when used HMS on postgres. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 7facd2c6afd35311b7dbaa37da6fa00bd0a2e0be
Author: Mahesh Kumar Behera
AuthorDate: Wed Feb 20 14:31:02 2019 +0530
HIVE-21260 : Hive replication to a target with hive.strict.managed.tables
enabled is failing when used HMS on postgres. (Mahesh Kumar Behera, reviewed by
Sankar Hariappan)
---
.../hadoop/hive/ql/parse/WarehouseInstance.java| 26 +++
.../org/apache/hadoop/hive/ql/exec/DDLTask.java| 1 +
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 2 ++
.../hadoop/hive/ql/parse/ReplicationSpec.java | 7 +
.../hadoop/hive/metastore/txn/TxnHandler.java | 30 +++---
5 files changed, 51 insertions(+), 15 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index bd3a557..c0d416c 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -155,6 +155,32 @@ public class WarehouseInstance implements Closeable {
MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf, true);
+// Add the below mentioned dependency in metastore/pom.xml file. For
postgres need to copy postgresql-42.2.1.jar to
+//
.m2//repository/postgresql/postgresql/9.3-1102.jdbc41/postgresql-9.3-1102.jdbc41.jar.
+/*
+
+ mysql
+ mysql-connector-java
+ 8.0.15
+
+
+
+ postgresql
+ postgresql
+ 9.3-1102.jdbc41
+
+*/
+
+/*hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
"jdbc:mysql://localhost:3306/APP");
+hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER,
"com.mysql.jdbc.Driver");
+hiveConf.setVar(HiveConf.ConfVars.METASTOREPWD, "hivepassword");
+hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME,
"hiveuser");*/
+
+
/*hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,"jdbc:postgresql://localhost/app");
+hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER,
"org.postgresql.Driver");
+hiveConf.setVar(HiveConf.ConfVars.METASTOREPWD, "password");
+hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME,
"postgres");*/
+
driver = DriverFactory.newDriver(hiveConf);
SessionState.start(new CliSessionState(hiveConf));
client = new HiveMetaStoreClient(hiveConf);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index cb7fdf7..b02cdf8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4712,6 +4712,7 @@ public class DDLTask extends Task implements
Serializable {
if (existingTable != null){
if
(crtTbl.getReplicationSpec().allowEventReplacementInto(existingTable.getParameters())){
crtTbl.setReplaceMode(true); // we replace existing table.
+ ReplicationSpec.copyLastReplId(existingTable.getParameters(),
tbl.getParameters());
} else {
LOG.debug("DDLTask: Create Table is skipped as table {} is newer
than update",
crtTbl.getTableName());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 5cfd0a8..7343eed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -132,6 +132,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.session.CreateTableAutomaticGrant;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
@@ -2972,6 +2973,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
org.apache.hadoop.hive.metastore.api.Partition ptn =
getMSC().getPartition(addPartitionDesc.getDbName(),
addPartitionDesc.getTableName(), p.getValues());
if
(addPartitionDesc.getReplicationSpec().allowReplacementInto(ptn.getParameters())){
+ ReplicationSpec.copyLastReplId(ptn.getParameters(),
p.getParameters());
partsToAlter.add(p);
} // else ptn already exists, but we do nothing with it.
} catch (NoSuchObjectException nsoe){
diff --git a/ql/src/java/org/apach
[hive] 01/02: Revert "Hive replication to a target with hive.strict.managed.tables enabled is failing when used HMS on postgres. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)"
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 010838702ae00fa0395fcb2d7c0b137ac3e15970
Author: Mahesh Kumar Behera
AuthorDate: Wed Feb 20 14:30:11 2019 +0530
Revert "Hive replication to a target with hive.strict.managed.tables
enabled is failing when used HMS on postgres. (Mahesh Kumar Behera, reviewed by
Sankar Hariappan)"
This reverts commit 104fa19a4a45aaaf7fae929c92e99977d1796a51.
---
.../hadoop/hive/ql/parse/WarehouseInstance.java| 26 ---
.../org/apache/hadoop/hive/ql/exec/DDLTask.java| 1 -
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 2 --
.../hadoop/hive/ql/parse/ReplicationSpec.java | 7 -
.../hadoop/hive/metastore/txn/TxnHandler.java | 30 +++---
5 files changed, 15 insertions(+), 51 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index c0d416c..bd3a557 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -155,32 +155,6 @@ public class WarehouseInstance implements Closeable {
MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf, true);
-// Add the below mentioned dependency in metastore/pom.xml file. For
postgres need to copy postgresql-42.2.1.jar to
-//
.m2//repository/postgresql/postgresql/9.3-1102.jdbc41/postgresql-9.3-1102.jdbc41.jar.
-/*
-
- mysql
- mysql-connector-java
- 8.0.15
-
-
-
- postgresql
- postgresql
- 9.3-1102.jdbc41
-
-*/
-
-/*hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
"jdbc:mysql://localhost:3306/APP");
-hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER,
"com.mysql.jdbc.Driver");
-hiveConf.setVar(HiveConf.ConfVars.METASTOREPWD, "hivepassword");
-hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME,
"hiveuser");*/
-
-
/*hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,"jdbc:postgresql://localhost/app");
-hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER,
"org.postgresql.Driver");
-hiveConf.setVar(HiveConf.ConfVars.METASTOREPWD, "password");
-hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME,
"postgres");*/
-
driver = DriverFactory.newDriver(hiveConf);
SessionState.start(new CliSessionState(hiveConf));
client = new HiveMetaStoreClient(hiveConf);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index b02cdf8..cb7fdf7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4712,7 +4712,6 @@ public class DDLTask extends Task implements
Serializable {
if (existingTable != null){
if
(crtTbl.getReplicationSpec().allowEventReplacementInto(existingTable.getParameters())){
crtTbl.setReplaceMode(true); // we replace existing table.
- ReplicationSpec.copyLastReplId(existingTable.getParameters(),
tbl.getParameters());
} else {
LOG.debug("DDLTask: Create Table is skipped as table {} is newer
than update",
crtTbl.getTableName());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 7343eed..5cfd0a8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -132,7 +132,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType;
-import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.session.CreateTableAutomaticGrant;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
@@ -2973,7 +2972,6 @@ private void constructOneLBLocationMap(FileStatus fSta,
org.apache.hadoop.hive.metastore.api.Partition ptn =
getMSC().getPartition(addPartitionDesc.getDbName(),
addPartitionDesc.getTableName(), p.getValues());
if
(addPartitionDesc.getReplicationSpec().allowReplacementInto(ptn.getParameters())){
- ReplicationSpec.copyLastReplId(ptn.getParameters(),
p.getParameters());
partsToAlter.add(p);
} // else ptn already exists, but we do nothing with it.
[hive] branch master updated (acec83f -> 7facd2c)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from acec83f HIVE-21254: Pre-upgrade tool should handle exceptions and skip db/tables (Prasanth Jayachandran reviewed by Jason Dere, Eugene Koifman, Ashutosh Chauhan) new 0108387 Revert "Hive replication to a target with hive.strict.managed.tables enabled is failing when used HMS on postgres. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)" new 7facd2c HIVE-21260 : Hive replication to a target with hive.strict.managed.tables enabled is failing when used HMS on postgres. (Mahesh Kumar Behera, reviewed by Sankar Hariappan) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[hive] branch master updated: Hive replication to a target with hive.strict.managed.tables enabled is failing when used HMS on postgres. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 104fa19 Hive replication to a target with hive.strict.managed.tables
enabled is failing when used HMS on postgres. (Mahesh Kumar Behera, reviewed by
Sankar Hariappan)
104fa19 is described below
commit 104fa19a4a45aaaf7fae929c92e99977d1796a51
Author: Mahesh Kumar Behera
AuthorDate: Tue Feb 19 12:20:31 2019 +0530
Hive replication to a target with hive.strict.managed.tables enabled is
failing when used HMS on postgres. (Mahesh Kumar Behera, reviewed by Sankar
Hariappan)
---
.../hadoop/hive/ql/parse/WarehouseInstance.java| 26 +++
.../org/apache/hadoop/hive/ql/exec/DDLTask.java| 1 +
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 2 ++
.../hadoop/hive/ql/parse/ReplicationSpec.java | 7 +
.../hadoop/hive/metastore/txn/TxnHandler.java | 30 +++---
5 files changed, 51 insertions(+), 15 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index bd3a557..c0d416c 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -155,6 +155,32 @@ public class WarehouseInstance implements Closeable {
MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf, true);
+// Add the below mentioned dependency in metastore/pom.xml file. For
postgres need to copy postgresql-42.2.1.jar to
+//
.m2//repository/postgresql/postgresql/9.3-1102.jdbc41/postgresql-9.3-1102.jdbc41.jar.
+/*
+
+ mysql
+ mysql-connector-java
+ 8.0.15
+
+
+
+ postgresql
+ postgresql
+ 9.3-1102.jdbc41
+
+*/
+
+/*hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
"jdbc:mysql://localhost:3306/APP");
+hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER,
"com.mysql.jdbc.Driver");
+hiveConf.setVar(HiveConf.ConfVars.METASTOREPWD, "hivepassword");
+hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME,
"hiveuser");*/
+
+
/*hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,"jdbc:postgresql://localhost/app");
+hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER,
"org.postgresql.Driver");
+hiveConf.setVar(HiveConf.ConfVars.METASTOREPWD, "password");
+hiveConf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME,
"postgres");*/
+
driver = DriverFactory.newDriver(hiveConf);
SessionState.start(new CliSessionState(hiveConf));
client = new HiveMetaStoreClient(hiveConf);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index cb7fdf7..b02cdf8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4712,6 +4712,7 @@ public class DDLTask extends Task implements
Serializable {
if (existingTable != null){
if
(crtTbl.getReplicationSpec().allowEventReplacementInto(existingTable.getParameters())){
crtTbl.setReplaceMode(true); // we replace existing table.
+ ReplicationSpec.copyLastReplId(existingTable.getParameters(),
tbl.getParameters());
} else {
LOG.debug("DDLTask: Create Table is skipped as table {} is newer
than update",
crtTbl.getTableName());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 5cfd0a8..7343eed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -132,6 +132,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.session.CreateTableAutomaticGrant;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
@@ -2972,6 +2973,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
org.apache.hadoop.hive.metastore.api.Partition ptn =
getMSC().getPartition(addPartitionDesc.getDbName(),
addPartitionDesc.getTableName(), p.getValues());
if
(addPartitionDesc.getReplicationSpec().allowReplacementInto(ptn.getParameter
[hive] branch master updated: HIVE-21055 : REPL LOAD command executing copy in serial mode even if parallel execution is enabled using WITH clause. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b0d01eb HIVE-21055 : REPL LOAD command executing copy in serial mode
even if parallel execution is enabled using WITH clause. (Mahesh Kumar Behera,
reviewed by Sankar Hariappan)
b0d01eb is described below
commit b0d01eb68a604bb6efb68455193803f0651dc265
Author: Mahesh Kumar Behera
AuthorDate: Wed Feb 13 12:55:38 2019 +0530
HIVE-21055 : REPL LOAD command executing copy in serial mode even if
parallel execution is enabled using WITH clause. (Mahesh Kumar Behera, reviewed
by Sankar Hariappan)
---
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 2 +-
ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java| 5 +
ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java| 4
.../hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java | 6 +-
4 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 95a4653..cac14a6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -2696,7 +2696,7 @@ public class Driver implements IDriver {
cxt.launching(tskRun);
// Launch Task
-if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.EXECPARALLEL) &&
tsk.canExecuteInParallel()) {
+if (HiveConf.getBoolVar(tsk.getConf(), HiveConf.ConfVars.EXECPARALLEL) &&
tsk.canExecuteInParallel()) {
// Launch it in the parallel mode, as a separate thread only for MR tasks
if (LOG.isInfoEnabled()){
LOG.info("Starting task [" + tsk + "] in parallel");
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 4f2a116..1c2522d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -870,4 +870,9 @@ public class MoveTask extends Task implements
Serializable {
public String getName() {
return "MOVE";
}
+
+ @Override
+ public boolean canExecuteInParallel() {
+return false;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
index b39771f..3308797 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
@@ -435,6 +435,10 @@ public abstract class Task
implements Serializable, Node
this.conf = conf;
}
+ public HiveConf getConf() {
+return this.conf;
+ }
+
public void setWork(T work) {
this.work = work;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
index 5302191..3e0c969 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
@@ -382,11 +382,7 @@ public class IncrementalLoadTasksBuilder {
}
// Link import tasks to the barrier task which will in-turn linked with
repl state update tasks
-for (Task t : importTasks){
- t.addDependentTask(barrierTask);
- log.debug("Added {}:{} as a precursor of barrier task {}:{}",
- t.getClass(), t.getId(), barrierTask.getClass(),
barrierTask.getId());
-}
+DAGTraversal.traverse(importTasks, new AddDependencyToLeaves(barrierTask));
// At least one task would have been added to update the repl state
return tasks;
[hive] branch master updated: HIVE-21063 : Support statistics in cachedStore for transactional table. (Mahesh Kumar Behera, reviewed by Sankar Hariappan, Daniel Dai)
This is an automated email from the ASF dual-hosted git repository. mahesh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 0223cbf HIVE-21063 : Support statistics in cachedStore for transactional table. (Mahesh Kumar Behera, reviewed by Sankar Hariappan, Daniel Dai) 0223cbf is described below commit 0223cbf15ae2b27d4879627760a4dab4d7cc713f Author: Mahesh Kumar Behera AuthorDate: Fri Feb 8 11:51:00 2019 +0530 HIVE-21063 : Support statistics in cachedStore for transactional table. (Mahesh Kumar Behera, reviewed by Sankar Hariappan, Daniel Dai) --- .../hcatalog/listener/DbNotificationListener.java | 4 +- .../TestCachedStoreUpdateUsingEvents.java | 714 ++--- .../hadoop/hive/ql/stats/StatsUpdaterThread.java | 4 +- .../hadoop/hive/metastore/HiveMetaStore.java | 25 +- .../apache/hadoop/hive/metastore/ObjectStore.java | 10 +- .../hadoop/hive/metastore/cache/CachedStore.java | 157 +++-- .../hadoop/hive/metastore/cache/SharedCache.java | 245 ++- .../events/UpdatePartitionColumnStatEvent.java | 12 +- .../events/UpdateTableColumnStatEvent.java | 13 +- .../hive/metastore/messaging/MessageBuilder.java | 8 +- .../UpdatePartitionColumnStatMessage.java | 2 - .../messaging/UpdateTableColumnStatMessage.java| 2 - .../json/JSONUpdatePartitionColumnStatMessage.java | 10 +- .../json/JSONUpdateTableColumnStatMessage.java | 10 +- .../src/main/sql/derby/hive-schema-4.0.0.derby.sql | 2 + .../sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql | 3 + .../src/main/sql/mssql/hive-schema-4.0.0.mssql.sql | 2 + .../sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql | 3 + .../src/main/sql/mysql/hive-schema-4.0.0.mysql.sql | 2 + .../sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql | 3 + .../main/sql/oracle/hive-schema-4.0.0.oracle.sql | 2 + .../sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql | 3 + .../sql/postgres/hive-schema-4.0.0.postgres.sql| 2 + .../postgres/upgrade-3.2.0-to-4.0.0.postgres.sql | 3 + .../hadoop/hive/metastore/TestHiveMetaStore.java | 10 +- 25 files changed, 1004 insertions(+), 247 deletions(-) diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index 8404e3e..963b227 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -759,7 +759,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener .buildUpdateTableColumnStatMessage(updateTableColumnStatEvent.getColStats(), updateTableColumnStatEvent.getTableObj(), updateTableColumnStatEvent.getTableParameters(), -updateTableColumnStatEvent.getValidWriteIds(), updateTableColumnStatEvent.getWriteId()); +updateTableColumnStatEvent.getWriteId()); NotificationEvent event = new NotificationEvent(0, now(), EventType.UPDATE_TABLE_COLUMN_STAT.toString(), msgEncoder.getSerializer().serialize(msg)); ColumnStatisticsDesc statDesc = updateTableColumnStatEvent.getColStats().getStatsDesc(); @@ -789,7 +789,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener updatePartColStatEvent.getPartVals(), updatePartColStatEvent.getPartParameters(), updatePartColStatEvent.getTableObj(), -updatePartColStatEvent.getValidWriteIds(), updatePartColStatEvent.getWriteId()); +updatePartColStatEvent.getWriteId()); NotificationEvent event = new NotificationEvent(0, now(), EventType.UPDATE_PARTITION_COLUMN_STAT.toString(), msgEncoder.getSerializer().serialize(msg)); ColumnStatisticsDesc statDesc = updatePartColStatEvent.getPartColStats().getStatsDesc(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestCachedStoreUpdateUsingEvents.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestCachedStoreUpdateUsingEvents.java index 83f12a5..cdfc60c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestCachedStoreUpdateUsingEvents.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestCachedStoreUpdateUsingEvents.java @@ -4,6 +4,8 @@ import java.util.*; import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.conf.HiveConf
[3/3] hive git commit: HIVE-18661 : CachedStore: Use metastore notification log events to update cache. (Mahesh Kumar Behera, reviewed by Daniel Dai)
HIVE-18661 : CachedStore: Use metastore notification log events to update cache. (Mahesh Kumar Behera, reviewed by Daniel Dai) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ef7c3963 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ef7c3963 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ef7c3963 Branch: refs/heads/master Commit: ef7c3963be035635ef75ee202b9826b9beb407d7 Parents: 87f8ecc Author: Mahesh Kumar Behera Authored: Tue Dec 18 13:26:56 2018 +0530 Committer: Mahesh Kumar Behera Committed: Tue Dec 18 13:26:56 2018 +0530 -- .../listener/DbNotificationListener.java| 72 +++ .../listener/TestDbNotificationListener.java| 83 +-- .../TestCachedStoreUpdateUsingEvents.java | 535 +++ .../TestReplicationScenariosAcidTables.java | 2 +- .../hadoop/hive/ql/parse/repl/DumpType.java | 28 + .../dump/events/DeletePartColStatHandler.java | 48 ++ .../dump/events/DeleteTableColStatHandler.java | 48 ++ .../repl/dump/events/EventHandlerFactory.java | 4 + .../dump/events/UpdatePartColStatHandler.java | 48 ++ .../dump/events/UpdateTableColStatHandler.java | 48 ++ .../load/message/DeletePartColStatHandler.java | 43 ++ .../load/message/DeleteTableColStatHandler.java | 43 ++ .../load/message/UpdatePartColStatHandler.java | 43 ++ .../load/message/UpdateTableColStatHandler.java | 43 ++ .../metastore/api/NotificationEventRequest.java | 163 +- .../src/gen/thrift/gen-php/metastore/Types.php | 49 ++ .../gen/thrift/gen-py/hive_metastore/ttypes.py | 23 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 4 +- .../hive/metastore/conf/MetastoreConf.java | 2 + .../src/main/thrift/hive_metastore.thrift | 1 + .../hadoop/hive/metastore/HiveAlterHandler.java | 92 ++-- .../hadoop/hive/metastore/HiveMetaStore.java| 99 +++- .../hive/metastore/MetaStoreEventListener.java | 47 ++ .../metastore/MetaStoreListenerNotifier.java| 12 + .../hadoop/hive/metastore/ObjectStore.java | 20 +- .../hive/metastore/cache/CachedStore.java | 493 ++--- .../hive/metastore/cache/SharedCache.java | 11 +- .../events/DeletePartitionColumnStatEvent.java | 81 +++ .../events/DeleteTableColumnStatEvent.java | 64 +++ .../events/UpdatePartitionColumnStatEvent.java | 93 .../events/UpdateTableColumnStatEvent.java | 84 +++ .../DeletePartitionColumnStatMessage.java | 36 ++ .../messaging/DeleteTableColumnStatMessage.java | 30 ++ .../hive/metastore/messaging/EventMessage.java | 6 +- .../metastore/messaging/MessageBuilder.java | 38 ++ .../messaging/MessageDeserializer.java | 28 + .../UpdatePartitionColumnStatMessage.java | 42 ++ .../messaging/UpdateTableColumnStatMessage.java | 39 ++ .../JSONDeletePartitionColumnStatMessage.java | 102 .../json/JSONDeleteTableColumnStatMessage.java | 85 +++ .../messaging/json/JSONMessageDeserializer.java | 40 ++ .../JSONUpdatePartitionColumnStatMessage.java | 133 + .../json/JSONUpdateTableColumnStatMessage.java | 121 + .../messaging/json/gzip/DeSerializer.java | 24 + .../hive/metastore/TestHiveAlterHandler.java| 6 +- 45 files changed, 2998 insertions(+), 158 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java -- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index fe101d3..fa7ab25 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent; @@ -79,6 +80,10 @@ import org.apache.hadoop.hive.metastore.events.AbortTxnEvent; import org.apache.hadoop.hive.metastore.events.AllocWriteIdEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; +import
[1/3] hive git commit: HIVE-18661 : CachedStore: Use metastore notification log events to update cache. (Mahesh Kumar Behera, reviewed by Daniel Dai)
Repository: hive
Updated Branches:
refs/heads/master 87f8eccf9 -> ef7c3963b
http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java
new file mode 100644
index 000..a61b98c
--- /dev/null
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.events;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.IHMSHandler;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * UpdatePartitionColumnStatEvent
+ * Event generated for partition column stat update event.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class UpdatePartitionColumnStatEvent extends ListenerEvent {
+ private ColumnStatistics partColStats;
+ private String validWriteIds;
+ private long writeId;
+ private Map parameters;
+ private List partVals;
+
+ /**
+ * @param statsObj Columns statistics Info.
+ * @param partVals partition names
+ * @param parameters table parameters to be updated after stats are updated.
+ * @param validWriteIds valid write id list for the query.
+ * @param writeId writeId for the query.
+ * @param handler handler that is firing the event
+ */
+ public UpdatePartitionColumnStatEvent(ColumnStatistics statsObj,
List partVals, Map parameters,
+String validWriteIds, long writeId,
IHMSHandler handler) {
+super(true, handler);
+this.partColStats = statsObj;
+this.validWriteIds = validWriteIds;
+this.writeId = writeId;
+this.parameters = parameters;
+this.partVals = partVals;
+ }
+
+ /**
+ * @param statsObj Columns statistics Info.
+ * @param partVals partition names
+ * @param handler handler that is firing the event
+ */
+ public UpdatePartitionColumnStatEvent(ColumnStatistics statsObj,
List partVals, IHMSHandler handler) {
+super(true, handler);
+this.partColStats = statsObj;
+this.partVals = partVals;
+this.validWriteIds = null;
+this.writeId = 0;
+this.parameters = null;
+ }
+
+ public ColumnStatistics getPartColStats() {
+return partColStats;
+ }
+
+ public String getValidWriteIds() {
+return validWriteIds;
+ }
+
+ public long getWriteId() {
+return writeId;
+ }
+
+ public Map getPartParameters() {
+return parameters;
+ }
+
+ public List getPartVals() {
+return partVals;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java
new file mode 100644
index 000..cf23617
--- /dev/null
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with
[2/3] hive git commit: HIVE-18661 : CachedStore: Use metastore notification log events to update cache. (Mahesh Kumar Behera, reviewed by Daniel Dai)
http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
--
diff --git
a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index cb899d7..9e4f3c2 100644
---
a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++
b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -1135,6 +1135,7 @@ struct CreationMetadata {
struct NotificationEventRequest {
1: required i64 lastEvent,
2: optional i32 maxEvents,
+3: optional list eventTypeSkipList,
}
struct NotificationEvent {
http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 0ea46f8..617c7bc 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -274,7 +274,7 @@ public class HiveAlterHandler implements AlterHandler {
part.setDbName(newDbName);
part.setTableName(newTblName);
ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb,
catName, dbname, name,
-part.getValues(), part.getSd().getCols(), oldt, part, null);
+part.getValues(), part.getSd().getCols(), oldt, part, null,
null);
if (colStats != null) {
columnStatsNeedUpdated.put(part, colStats);
}
@@ -312,7 +312,7 @@ public class HiveAlterHandler implements AlterHandler {
}
} else {
alterTableUpdateTableColumnStats(
- msdb, oldt, newt, environmentContext, writeIdList);
+ msdb, oldt, newt, environmentContext, writeIdList, conf, null);
}
} else {
// operations other than table rename
@@ -332,7 +332,7 @@ public class HiveAlterHandler implements AlterHandler {
List oldCols = part.getSd().getCols();
part.getSd().setCols(newt.getSd().getCols());
ColumnStatistics colStats =
updateOrGetPartitionColumnStats(msdb, catName, dbname, name,
- part.getValues(), oldCols, oldt, part, null);
+ part.getValues(), oldCols, oldt, part, null, null);
assert(colStats == null);
if (cascade) {
msdb.alterPartition(
@@ -349,11 +349,11 @@ public class HiveAlterHandler implements AlterHandler {
} else {
LOG.warn("Alter table not cascaded to partitions.");
alterTableUpdateTableColumnStats(
-msdb, oldt, newt, environmentContext, writeIdList);
+msdb, oldt, newt, environmentContext, writeIdList, conf, null);
}
} else {
alterTableUpdateTableColumnStats(
- msdb, oldt, newt, environmentContext, writeIdList);
+ msdb, oldt, newt, environmentContext, writeIdList, conf, null);
}
}
@@ -481,7 +481,7 @@ public class HiveAlterHandler implements AlterHandler {
// PartitionView does not have SD. We do not need update its column
stats
if (oldPart.getSd() != null) {
updateOrGetPartitionColumnStats(msdb, catName, dbname, name,
new_part.getValues(),
- oldPart.getSd().getCols(), tbl, new_part, null);
+ oldPart.getSd().getCols(), tbl, new_part, null, null);
}
msdb.alterPartition(
catName, dbname, name, new_part.getValues(), new_part,
validWriteIds);
@@ -620,7 +620,7 @@ public class HiveAlterHandler implements AlterHandler {
String newPartName = Warehouse.makePartName(tbl.getPartitionKeys(),
new_part.getValues());
ColumnStatistics cs = updateOrGetPartitionColumnStats(msdb, catName,
dbname, name, oldPart.getValues(),
- oldPart.getSd().getCols(), tbl, new_part, null);
+ oldPart.getSd().getCols(), tbl, new_part, null, null);
msdb.alterPartition(catName, dbname, name, part_vals, new_part,
validWriteIds);
if (cs != null) {
cs.getStatsDesc().setPartName(newPartName);
@@ -727,7 +727,7 @@ public class HiveAlterHandler implements AlterHandler {
// PartitionView does not have SD and we do not need to update its
column stats
if (oldTmpPart.getSd() != null) {
updateOrGetPartitionColumnStats(msdb, catName, dbname, name,
oldTmpPa
hive git commit: HIVE-21023 : Add test for replication to a target with hive.strict.managed.tables enabled. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
Repository: hive
Updated Branches:
refs/heads/master e8e0396c1 -> 7da8f3d36
HIVE-21023 : Add test for replication to a target with
hive.strict.managed.tables enabled. (Mahesh Kumar Behera, reviewed by Sankar
Hariappan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7da8f3d3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7da8f3d3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7da8f3d3
Branch: refs/heads/master
Commit: 7da8f3d36ee2b6c508ea2ab8c241df52107ac74e
Parents: e8e0396
Author: Mahesh Kumar Behera
Authored: Fri Dec 14 18:26:09 2018 +0530
Committer: Mahesh Kumar Behera
Committed: Fri Dec 14 18:26:09 2018 +0530
--
...ationScenariosIncrementalLoadAcidTables.java | 128 +---
.../TestReplicationScenariosMigration.java | 33 ++
.../TestReplicationWithTableMigration.java | 328 +++
3 files changed, 363 insertions(+), 126 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/7da8f3d3/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
index b71cfa4..97775b3 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
@@ -17,20 +17,13 @@
*/
package org.apache.hadoop.hive.ql.parse;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import
org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
-import org.apache.hadoop.hive.ql.parse.repl.PathBuilder;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.hive.ql.parse.WarehouseInstance;
import static
org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION;
-import static org.apache.hadoop.hive.ql.io.AcidUtils.isFullAcidTable;
-import static org.apache.hadoop.hive.ql.io.AcidUtils.isTransactionalTable;
-
import org.apache.hadoop.hive.ql.parse.ReplicationTestUtils;
import org.junit.rules.TestName;
@@ -48,9 +41,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+
import com.google.common.collect.Lists;
/**
@@ -62,7 +53,7 @@ public class
TestReplicationScenariosIncrementalLoadAcidTables {
protected static final Logger LOG =
LoggerFactory.getLogger(TestReplicationScenariosIncrementalLoadAcidTables.class);
static WarehouseInstance primary;
- private static WarehouseInstance replica, replicaNonAcid, replicaMigration,
primaryMigration;
+ private static WarehouseInstance replica, replicaNonAcid;
private static HiveConf conf;
private String primaryDbName, replicatedDbName, primaryDbNameExtra;
@@ -105,36 +96,6 @@ public class
TestReplicationScenariosIncrementalLoadAcidTables {
put("hive.metastore.client.capability.check", "false");
}};
replicaNonAcid = new WarehouseInstance(LOG, miniDFSCluster,
overridesForHiveConf1);
-
-HashMap overridesForHiveConfReplicaMigration = new
HashMap() {{
- put("fs.defaultFS", miniDFSCluster.getFileSystem().getUri().toString());
- put("hive.support.concurrency", "true");
- put("hive.txn.manager",
"org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
- put("hive.metastore.client.capability.check", "false");
- put("hive.repl.bootstrap.dump.open.txn.timeout", "1s");
- put("hive.exec.dynamic.partition.mode", "nonstrict");
- put("hive.strict.checks.bucketing", "false");
- put("hive.mapred.mode", "nonstrict");
- put("mapred.input.dir.recursive", "true");
- put("hive.metastore.disallow.incompatible.col.type.changes", "false");
- put("hive.strict.managed.tables", "true");
-}};
-replicaMigration = new WarehouseInstance(LOG, miniDFSCluster,
overridesForHiveC
[10/11] hive git commit: HIVE-20966 : Support bootstrap and incremental replication to a target with hive.strict.managed.tables enabled.(Mahesh Kumar Behera reviewed by Sankar Hariappan)
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
index 314ca48..b71cfa4 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
@@ -17,13 +17,21 @@
*/
package org.apache.hadoop.hive.ql.parse;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import
org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.parse.repl.PathBuilder;
import org.apache.hadoop.hive.shims.Utils;
-
+import org.apache.hadoop.hive.ql.parse.WarehouseInstance;
import static
org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION;
+import static org.apache.hadoop.hive.ql.io.AcidUtils.isFullAcidTable;
+import static org.apache.hadoop.hive.ql.io.AcidUtils.isTransactionalTable;
+
+import org.apache.hadoop.hive.ql.parse.ReplicationTestUtils;
import org.junit.rules.TestName;
@@ -40,7 +48,9 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import com.google.common.collect.Lists;
/**
@@ -52,14 +62,9 @@ public class
TestReplicationScenariosIncrementalLoadAcidTables {
protected static final Logger LOG =
LoggerFactory.getLogger(TestReplicationScenariosIncrementalLoadAcidTables.class);
static WarehouseInstance primary;
- private static WarehouseInstance replica, replicaNonAcid;
+ private static WarehouseInstance replica, replicaNonAcid, replicaMigration,
primaryMigration;
private static HiveConf conf;
private String primaryDbName, replicatedDbName, primaryDbNameExtra;
- private enum OperationType {
-REPL_TEST_ACID_INSERT, REPL_TEST_ACID_INSERT_SELECT, REPL_TEST_ACID_CTAS,
-REPL_TEST_ACID_INSERT_OVERWRITE, REPL_TEST_ACID_INSERT_IMPORT,
REPL_TEST_ACID_INSERT_LOADLOCAL,
-REPL_TEST_ACID_INSERT_UNION
- }
@BeforeClass
public static void classLevelSetup() throws Exception {
@@ -67,7 +72,7 @@ public class
TestReplicationScenariosIncrementalLoadAcidTables {
overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(),
GzipJSONMessageEncoder.class.getCanonicalName());
-internalBeforeClassSetup(overrides,
TestReplicationScenariosAcidTables.class);
+internalBeforeClassSetup(overrides,
TestReplicationScenariosIncrementalLoadAcidTables.class);
}
static void internalBeforeClassSetup(Map overrides, Class
clazz)
@@ -100,6 +105,36 @@ public class
TestReplicationScenariosIncrementalLoadAcidTables {
put("hive.metastore.client.capability.check", "false");
}};
replicaNonAcid = new WarehouseInstance(LOG, miniDFSCluster,
overridesForHiveConf1);
+
+HashMap overridesForHiveConfReplicaMigration = new
HashMap() {{
+ put("fs.defaultFS", miniDFSCluster.getFileSystem().getUri().toString());
+ put("hive.support.concurrency", "true");
+ put("hive.txn.manager",
"org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
+ put("hive.metastore.client.capability.check", "false");
+ put("hive.repl.bootstrap.dump.open.txn.timeout", "1s");
+ put("hive.exec.dynamic.partition.mode", "nonstrict");
+ put("hive.strict.checks.bucketing", "false");
+ put("hive.mapred.mode", "nonstrict");
+ put("mapred.input.dir.recursive", "true");
+ put("hive.metastore.disallow.incompatible.col.type.changes", "false");
+ put("hive.strict.managed.tables", "true");
+}};
+replicaMigration = new WarehouseInstance(LOG, miniDFSCluster,
overridesForHiveConfReplicaMigration);
+
+HashMap overridesForHiveConfPrimaryMigration = new
HashMap() {{
+ put("fs.defaultFS", miniDFSCluster.getFileSystem().getUri().toString());
+ put("hive.metastore.client.capability.check", "false");
+ put("hive.repl.bootstrap.dump.open.txn.timeout", "1s");
+ put("hive.exec.dynamic.partition.mode", "nonstrict");
+ put("hive.strict.checks.bucketing", "false");
+ put("hive.mapred.mode", "nonstrict");
+ put("mapred.input.dir.recursive", "true");
+ put("hive.metastore.dis
[08/11] hive git commit: HIVE-20966 : Support bootstrap and incremental replication to a target with hive.strict.managed.tables enabled.(Mahesh Kumar Behera reviewed by Sankar Hariappan)
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java
--
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java
index 95ea81f..e6b0fd5 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/CommitTxnRequest.java
@@ -42,6 +42,7 @@ import org.slf4j.LoggerFactory;
private static final org.apache.thrift.protocol.TField
REPL_POLICY_FIELD_DESC = new org.apache.thrift.protocol.TField("replPolicy",
org.apache.thrift.protocol.TType.STRING, (short)2);
private static final org.apache.thrift.protocol.TField
WRITE_EVENT_INFOS_FIELD_DESC = new
org.apache.thrift.protocol.TField("writeEventInfos",
org.apache.thrift.protocol.TType.LIST, (short)3);
private static final org.apache.thrift.protocol.TField KEY_VALUE_FIELD_DESC
= new org.apache.thrift.protocol.TField("keyValue",
org.apache.thrift.protocol.TType.STRUCT, (short)4);
+ private static final org.apache.thrift.protocol.TField
REPL_LAST_ID_INFO_FIELD_DESC = new
org.apache.thrift.protocol.TField("replLastIdInfo",
org.apache.thrift.protocol.TType.STRUCT, (short)5);
private static final Map, SchemeFactory> schemes =
new HashMap, SchemeFactory>();
static {
@@ -53,13 +54,15 @@ import org.slf4j.LoggerFactory;
private String replPolicy; // optional
private List writeEventInfos; // optional
private CommitTxnKeyValue keyValue; // optional
+ private ReplLastIdInfo replLastIdInfo; // optional
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
TXNID((short)1, "txnid"),
REPL_POLICY((short)2, "replPolicy"),
WRITE_EVENT_INFOS((short)3, "writeEventInfos"),
-KEY_VALUE((short)4, "keyValue");
+KEY_VALUE((short)4, "keyValue"),
+REPL_LAST_ID_INFO((short)5, "replLastIdInfo");
private static final Map byName = new HashMap();
@@ -82,6 +85,8 @@ import org.slf4j.LoggerFactory;
return WRITE_EVENT_INFOS;
case 4: // KEY_VALUE
return KEY_VALUE;
+case 5: // REPL_LAST_ID_INFO
+ return REPL_LAST_ID_INFO;
default:
return null;
}
@@ -124,7 +129,7 @@ import org.slf4j.LoggerFactory;
// isset id assignments
private static final int __TXNID_ISSET_ID = 0;
private byte __isset_bitfield = 0;
- private static final _Fields optionals[] =
{_Fields.REPL_POLICY,_Fields.WRITE_EVENT_INFOS,_Fields.KEY_VALUE};
+ private static final _Fields optionals[] =
{_Fields.REPL_POLICY,_Fields.WRITE_EVENT_INFOS,_Fields.KEY_VALUE,_Fields.REPL_LAST_ID_INFO};
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData>
metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new
EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -137,6 +142,8 @@ import org.slf4j.LoggerFactory;
new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT
, "WriteEventInfo";
tmpMap.put(_Fields.KEY_VALUE, new
org.apache.thrift.meta_data.FieldMetaData("keyValue",
org.apache.thrift.TFieldRequirementType.OPTIONAL,
new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
CommitTxnKeyValue.class)));
+tmpMap.put(_Fields.REPL_LAST_ID_INFO, new
org.apache.thrift.meta_data.FieldMetaData("replLastIdInfo",
org.apache.thrift.TFieldRequirementType.OPTIONAL,
+new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT
, "ReplLastIdInfo")));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CommitTxnRequest.class,
metaDataMap);
}
@@ -171,6 +178,9 @@ import org.slf4j.LoggerFactory;
if (other.isSetKeyValue()) {
this.keyValue = new CommitTxnKeyValue(other.keyValue);
}
+if (other.isSetReplLastIdInfo()) {
+ this.replLastIdInfo = other.replLastIdInfo;
+}
}
public CommitTxnRequest deepCopy() {
@@ -184,6 +194,7 @@ import org.slf4j.LoggerFactory;
this.replPolicy = null;
this.writeEventInfos = null;
this.keyValue = null;
+this.replLastIdInfo = null;
}
public long getTxnid() {
@@ -292,6 +303,29 @@ import org.slf4j.LoggerFactory;
}
}
+ public ReplLastIdInfo getReplLastIdInfo() {
+return this.r
[06/11] hive git commit: HIVE-20966 : Support bootstrap and incremental replication to a target with hive.strict.managed.tables enabled.(Mahesh Kumar Behera reviewed by Sankar Hariappan)
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
--
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
index 52a2b71..b6a0893 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
@@ -43489,13 +43489,13 @@ import org.slf4j.LoggerFactory;
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list1000 =
iprot.readListBegin();
- struct.success = new ArrayList(_list1000.size);
- String _elem1001;
- for (int _i1002 = 0; _i1002 < _list1000.size; ++_i1002)
+ org.apache.thrift.protocol.TList _list1008 =
iprot.readListBegin();
+ struct.success = new ArrayList(_list1008.size);
+ String _elem1009;
+ for (int _i1010 = 0; _i1010 < _list1008.size; ++_i1010)
{
-_elem1001 = iprot.readString();
-struct.success.add(_elem1001);
+_elem1009 = iprot.readString();
+struct.success.add(_elem1009);
}
iprot.readListEnd();
}
@@ -43530,9 +43530,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
struct.success.size()));
-for (String _iter1003 : struct.success)
+for (String _iter1011 : struct.success)
{
- oprot.writeString(_iter1003);
+ oprot.writeString(_iter1011);
}
oprot.writeListEnd();
}
@@ -43571,9 +43571,9 @@ import org.slf4j.LoggerFactory;
if (struct.isSetSuccess()) {
{
oprot.writeI32(struct.success.size());
-for (String _iter1004 : struct.success)
+for (String _iter1012 : struct.success)
{
- oprot.writeString(_iter1004);
+ oprot.writeString(_iter1012);
}
}
}
@@ -43588,13 +43588,13 @@ import org.slf4j.LoggerFactory;
BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
{
-org.apache.thrift.protocol.TList _list1005 = new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
iprot.readI32());
-struct.success = new ArrayList(_list1005.size);
-String _elem1006;
-for (int _i1007 = 0; _i1007 < _list1005.size; ++_i1007)
+org.apache.thrift.protocol.TList _list1013 = new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
iprot.readI32());
+struct.success = new ArrayList(_list1013.size);
+String _elem1014;
+for (int _i1015 = 0; _i1015 < _list1013.size; ++_i1015)
{
- _elem1006 = iprot.readString();
- struct.success.add(_elem1006);
+ _elem1014 = iprot.readString();
+ struct.success.add(_elem1014);
}
}
struct.setSuccessIsSet(true);
@@ -44248,13 +44248,13 @@ import org.slf4j.LoggerFactory;
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list1008 =
iprot.readListBegin();
- struct.success = new ArrayList(_list1008.size);
- String _elem1009;
- for (int _i1010 = 0; _i1010 < _list1008.size; ++_i1010)
+ org.apache.thrift.protocol.TList _list1016 =
iprot.readListBegin();
+ struct.success = new ArrayList(_list1016.size);
+ String _elem1017;
+ for (int _i1018 = 0; _i1018 < _list1016.size; ++_i1018)
{
-_elem1009 = iprot.readString();
-struct.success.add(_elem1009);
+_elem1017 = iprot.readString();
+struct.success.add(_elem1017);
}
iprot.readListEnd();
}
@@ -44289,9 +44289,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(SUCCESS_FIE
[11/11] hive git commit: HIVE-20966 : Support bootstrap and incremental replication to a target with hive.strict.managed.tables enabled.(Mahesh Kumar Behera reviewed by Sankar Hariappan)
HIVE-20966 : Support bootstrap and incremental replication to a target with hive.strict.managed.tables enabled.(Mahesh Kumar Behera reviewed by Sankar Hariappan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/10cfba20 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/10cfba20 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/10cfba20 Branch: refs/heads/master Commit: 10cfba20074149bf786206f004e2260d6c3119eb Parents: f5618d9 Author: Mahesh Kumar Behera Authored: Mon Dec 10 17:38:36 2018 +0530 Committer: Mahesh Kumar Behera Committed: Mon Dec 10 17:38:36 2018 +0530 -- .../org/apache/hadoop/hive/conf/HiveConf.java |6 +- .../json/JSONAlterPartitionMessage.java |2 +- .../messaging/json/JSONAlterTableMessage.java |2 +- .../hive/ql/parse/ReplicationTestUtils.java | 489 ...ncrementalLoadAcidTablesWithJsonMessage.java |2 + .../ql/parse/TestReplWithJsonMessageFormat.java |2 +- .../hive/ql/parse/TestReplicationScenarios.java | 40 +- .../TestReplicationScenariosAcidTables.java |2 +- ...ationScenariosIncrementalLoadAcidTables.java | 632 ++--- .../hadoop/hive/ql/parse/WarehouseInstance.java | 63 +- .../metastore/SynchronizedMetaStoreClient.java |6 + .../java/org/apache/hadoop/hive/ql/Driver.java |7 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 23 +- .../apache/hadoop/hive/ql/exec/MoveTask.java| 12 + .../apache/hadoop/hive/ql/exec/ReplTxnTask.java | 33 + .../events/filesystem/FSTableEvent.java | 34 +- .../bootstrap/load/table/LoadPartitions.java| 28 +- .../repl/bootstrap/load/table/LoadTable.java| 41 +- .../IncrementalLoadTasksBuilder.java| 83 +- .../hive/ql/exec/repl/util/ReplUtils.java | 42 + .../hadoop/hive/ql/lockmgr/DbTxnManager.java| 34 +- .../apache/hadoop/hive/ql/metadata/Hive.java| 34 +- .../apache/hadoop/hive/ql/parse/EximUtil.java |8 + .../hive/ql/parse/ImportSemanticAnalyzer.java | 73 +- .../ql/parse/ReplicationSemanticAnalyzer.java | 11 + .../hadoop/hive/ql/parse/ReplicationSpec.java |8 + .../parse/repl/load/UpdatedMetaDataTracker.java | 10 + .../repl/load/message/DropPartitionHandler.java |3 +- .../load/message/RenamePartitionHandler.java|4 +- .../repl/load/message/RenameTableHandler.java | 13 +- .../parse/repl/load/message/TableHandler.java | 11 + .../load/message/TruncatePartitionHandler.java | 13 +- .../repl/load/message/TruncateTableHandler.java | 10 +- .../hadoop/hive/ql/plan/LoadTableDesc.java |4 + .../apache/hadoop/hive/ql/plan/ReplTxnWork.java | 22 +- .../ql/util/HiveStrictManagedMigration.java | 139 +- .../metastore/api/AddDynamicPartitions.java | 32 +- .../api/AllocateTableWriteIdsRequest.java | 68 +- .../api/AllocateTableWriteIdsResponse.java | 36 +- .../metastore/api/AlterPartitionsRequest.java | 36 +- .../metastore/api/ClearFileMetadataRequest.java | 32 +- .../hive/metastore/api/ClientCapabilities.java | 32 +- .../hive/metastore/api/CommitTxnRequest.java| 116 +- .../hive/metastore/api/CompactionRequest.java | 44 +- .../hive/metastore/api/CreationMetadata.java| 32 +- .../metastore/api/FindSchemasByColsResp.java| 36 +- .../hive/metastore/api/FireEventRequest.java| 32 +- .../metastore/api/GetAllFunctionsResponse.java | 36 +- .../api/GetFileMetadataByExprRequest.java | 32 +- .../api/GetFileMetadataByExprResult.java| 48 +- .../metastore/api/GetFileMetadataRequest.java | 32 +- .../metastore/api/GetFileMetadataResult.java| 44 +- .../metastore/api/GetPartitionsFilterSpec.java | 32 +- .../api/GetPartitionsProjectionSpec.java| 32 +- .../metastore/api/GetPartitionsRequest.java | 32 +- .../metastore/api/GetPartitionsResponse.java| 36 +- .../hive/metastore/api/GetTablesRequest.java| 32 +- .../hive/metastore/api/GetTablesResult.java | 36 +- .../metastore/api/GetValidWriteIdsRequest.java | 32 +- .../metastore/api/GetValidWriteIdsResponse.java | 36 +- .../api/HeartbeatTxnRangeResponse.java | 64 +- .../metastore/api/InsertEventRequestData.java | 96 +- .../hadoop/hive/metastore/api/LockRequest.java | 36 +- .../api/NotificationEventResponse.java | 36 +- .../metastore/api/PutFileMetadataRequest.java | 64 +- .../metastore/api/RenamePartitionRequest.java | 32 +- .../hive/metastore/api/ReplLastIdInfo.java | 958 +++ .../api/ReplTblWriteIdStateRequest.java | 32 +- .../hive/metastore/api/SchemaVersion.java | 36 +- .../hive/metastore/api/ShowCompactResponse.java | 36 +- .../hive/metastore/api/ShowLocksResponse.java | 36 +- .../hive/metastore/api/TableValidWriteIds.java | 32 +- .../hive/metastore
[04/11] hive git commit: HIVE-20966 : Support bootstrap and incremental replication to a target with hive.strict.managed.tables enabled.(Mahesh Kumar Behera reviewed by Sankar Hariappan)
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
--
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
index efeaec7..3170798 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
@@ -15952,14 +15952,14 @@ class ThriftHiveMetastore_get_databases_result {
case 0:
if ($ftype == TType::LST) {
$this->success = array();
-$_size882 = 0;
-$_etype885 = 0;
-$xfer += $input->readListBegin($_etype885, $_size882);
-for ($_i886 = 0; $_i886 < $_size882; ++$_i886)
+$_size889 = 0;
+$_etype892 = 0;
+$xfer += $input->readListBegin($_etype892, $_size889);
+for ($_i893 = 0; $_i893 < $_size889; ++$_i893)
{
- $elem887 = null;
- $xfer += $input->readString($elem887);
- $this->success []= $elem887;
+ $elem894 = null;
+ $xfer += $input->readString($elem894);
+ $this->success []= $elem894;
}
$xfer += $input->readListEnd();
} else {
@@ -15995,9 +15995,9 @@ class ThriftHiveMetastore_get_databases_result {
{
$output->writeListBegin(TType::STRING, count($this->success));
{
- foreach ($this->success as $iter888)
+ foreach ($this->success as $iter895)
{
-$xfer += $output->writeString($iter888);
+$xfer += $output->writeString($iter895);
}
}
$output->writeListEnd();
@@ -16128,14 +16128,14 @@ class ThriftHiveMetastore_get_all_databases_result {
case 0:
if ($ftype == TType::LST) {
$this->success = array();
-$_size889 = 0;
-$_etype892 = 0;
-$xfer += $input->readListBegin($_etype892, $_size889);
-for ($_i893 = 0; $_i893 < $_size889; ++$_i893)
+$_size896 = 0;
+$_etype899 = 0;
+$xfer += $input->readListBegin($_etype899, $_size896);
+for ($_i900 = 0; $_i900 < $_size896; ++$_i900)
{
- $elem894 = null;
- $xfer += $input->readString($elem894);
- $this->success []= $elem894;
+ $elem901 = null;
+ $xfer += $input->readString($elem901);
+ $this->success []= $elem901;
}
$xfer += $input->readListEnd();
} else {
@@ -16171,9 +16171,9 @@ class ThriftHiveMetastore_get_all_databases_result {
{
$output->writeListBegin(TType::STRING, count($this->success));
{
- foreach ($this->success as $iter895)
+ foreach ($this->success as $iter902)
{
-$xfer += $output->writeString($iter895);
+$xfer += $output->writeString($iter902);
}
}
$output->writeListEnd();
@@ -17174,18 +17174,18 @@ class ThriftHiveMetastore_get_type_all_result {
case 0:
if ($ftype == TType::MAP) {
$this->success = array();
-$_size896 = 0;
-$_ktype897 = 0;
-$_vtype898 = 0;
-$xfer += $input->readMapBegin($_ktype897, $_vtype898, $_size896);
-for ($_i900 = 0; $_i900 < $_size896; ++$_i900)
+$_size903 = 0;
+$_ktype904 = 0;
+$_vtype905 = 0;
+$xfer += $input->readMapBegin($_ktype904, $_vtype905, $_size903);
+for ($_i907 = 0; $_i907 < $_size903; ++$_i907)
{
- $key901 = '';
- $val902 = new \metastore\Type();
- $xfer += $input->readString($key901);
- $val902 = new \metastore\Type();
- $xfer += $val902->read($input);
- $this->success[$key901] = $val902;
+ $key908 = '';
+ $val909 = new \metastore\Type();
+ $xfer += $input->readString($key908);
+ $val909 = new \metastore\Type();
+ $xfer += $val909->read($input);
+ $this->success[$key908] = $val909;
}
$xfer += $input->readMapEnd();
} else {
@@ -17221,10 +17221,10 @@ class ThriftHiveMetastore_get_type_all_result {
{
$output->writeMapBegin(TType::STRING, TType::STRUCT,
count($this->success));
{
- foreach ($this->success as $kiter903 => $viter904)
+ foreach ($this->success as $kiter910 => $viter911)
{
-$xfer += $output->writeString($kiter903);
-$xfer
