[2/2] hive git commit: HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman, reviewed by Sergey Shelukhin)
HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman,
reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3e535f9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3e535f9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3e535f9
Branch: refs/heads/branch-3
Commit: a3e535f944d852209ca299e703860780fbd53955
Parents: 8584947
Author: Eugene Koifman
Authored: Thu Apr 19 09:21:41 2018 -0700
Committer: Eugene Koifman
Committed: Thu Apr 19 09:21:41 2018 -0700
--
.../apache/hadoop/hive/common/JavaUtils.java| 3 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 2 +-
.../apache/hadoop/hive/ql/exec/ExportTask.java | 1 +
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 4 +
.../apache/hadoop/hive/ql/metadata/Hive.java| 9 +-
.../ql/metadata/SessionHiveMetaStoreClient.java | 217
.../hive/ql/parse/BaseSemanticAnalyzer.java | 7 +
.../hive/ql/parse/ExportSemanticAnalyzer.java | 19 +-
.../hive/ql/parse/ImportSemanticAnalyzer.java | 116 ++--
.../hive/ql/parse/SemanticAnalyzerFactory.java | 3 +
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 258 -
.../apache/hadoop/hive/ql/plan/CopyWork.java| 6 +-
.../apache/hadoop/hive/ql/plan/ExportWork.java | 28 +-
.../hadoop/hive/ql/plan/ImportTableDesc.java| 2 +-
.../hadoop/hive/ql/session/SessionState.java| 10 +-
.../hadoop/hive/ql/TestTxnAddPartition.java | 2 +-
.../org/apache/hadoop/hive/ql/TestTxnExIm.java | 538 +++
.../apache/hadoop/hive/ql/TestTxnLoadData.java | 2 +-
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 24 +-
.../hive/metastore/HiveMetaStoreClient.java | 2 +-
.../hadoop/hive/metastore/ObjectStore.java | 18 +-
.../hive/metastore/utils/MetaStoreUtils.java| 20 +
22 files changed, 1201 insertions(+), 90 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
index 75c07b4..7894ec1 100644
--- a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
@@ -228,10 +228,11 @@ public final class JavaUtils {
@Override
public boolean accept(Path path) {
String name = path.getName();
+ //todo: what if this is a base?
if (!name.startsWith(DELTA_PREFIX + "_")) return false;
String idStr = name.substring(DELTA_PREFIX.length() + 1,
DELTA_PREFIX.length() + 1 + DELTA_DIGITS_LEN);
try {
-Long.parseLong(idStr);
+Long.parseLong(idStr);//what for? sanity check?
} catch (NumberFormatException ex) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 5b26b84..9a487cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -3965,7 +3965,7 @@ public class DDLTask extends Task implements
Serializable {
* how this is desirable.
*
* As of HIVE-14993, WriteEntity with different WriteType must be considered
different.
- * So WriteEntity create in DDLTask cause extra output in golden files, but
only because
+ * So WriteEntity created in DDLTask cause extra output in golden files, but
only because
* DDLTask sets a different WriteType for the same Entity.
*
* In the spirit of bug-for-bug compatibility, this method ensures we only
add new
http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
index 91af814..aba6591 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
@@ -52,6 +52,7 @@ public class ExportTask extends Task implements
Serializable {
conf, false);
Hive db = getHive();
LOG.debug("Exporting data to: {}", exportPaths.getExportRootDir());
+ work.acidPostProcess(db);
TableExport tableExport = new TableExport(
exportPaths, work.getTableSpec(), work.getReplicationSpec(), db,
[2/2] hive git commit: HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman, reviewed by Sergey Shelukhin)
HIVE-18739 - Add support for Import/Export from Acid table (Eugene Koifman,
reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/699c5768
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/699c5768
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/699c5768
Branch: refs/heads/master
Commit: 699c5768c88967abd507122d775bd5955ca45218
Parents: bd6b582
Author: Eugene Koifman
Authored: Tue Apr 17 18:23:13 2018 -0700
Committer: Eugene Koifman
Committed: Tue Apr 17 18:23:48 2018 -0700
--
.../apache/hadoop/hive/common/JavaUtils.java| 3 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 2 +-
.../apache/hadoop/hive/ql/exec/ExportTask.java | 1 +
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 4 +
.../apache/hadoop/hive/ql/metadata/Hive.java| 9 +-
.../ql/metadata/SessionHiveMetaStoreClient.java | 217
.../hive/ql/parse/BaseSemanticAnalyzer.java | 7 +
.../hive/ql/parse/ExportSemanticAnalyzer.java | 19 +-
.../hive/ql/parse/ImportSemanticAnalyzer.java | 111 ++--
.../hive/ql/parse/SemanticAnalyzerFactory.java | 3 +
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 258 -
.../apache/hadoop/hive/ql/plan/CopyWork.java| 6 +-
.../apache/hadoop/hive/ql/plan/ExportWork.java | 28 +-
.../hadoop/hive/ql/plan/ImportTableDesc.java| 2 +-
.../hadoop/hive/ql/session/SessionState.java| 10 +-
.../hadoop/hive/ql/TestTxnAddPartition.java | 2 +-
.../org/apache/hadoop/hive/ql/TestTxnExIm.java | 538 +++
.../apache/hadoop/hive/ql/TestTxnLoadData.java | 2 +-
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 24 +-
.../hive/metastore/HiveMetaStoreClient.java | 2 +-
.../hadoop/hive/metastore/ObjectStore.java | 18 +-
.../hive/metastore/utils/MetaStoreUtils.java| 20 +
22 files changed, 1197 insertions(+), 89 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/699c5768/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
index 75c07b4..7894ec1 100644
--- a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
@@ -228,10 +228,11 @@ public final class JavaUtils {
@Override
public boolean accept(Path path) {
String name = path.getName();
+ //todo: what if this is a base?
if (!name.startsWith(DELTA_PREFIX + "_")) return false;
String idStr = name.substring(DELTA_PREFIX.length() + 1,
DELTA_PREFIX.length() + 1 + DELTA_DIGITS_LEN);
try {
-Long.parseLong(idStr);
+Long.parseLong(idStr);//what for? sanity check?
} catch (NumberFormatException ex) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/699c5768/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 2466da9..c8cb8a4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -3969,7 +3969,7 @@ public class DDLTask extends Task implements
Serializable {
* how this is desirable.
*
* As of HIVE-14993, WriteEntity with different WriteType must be considered
different.
- * So WriteEntity create in DDLTask cause extra output in golden files, but
only because
+ * So WriteEntity created in DDLTask cause extra output in golden files, but
only because
* DDLTask sets a different WriteType for the same Entity.
*
* In the spirit of bug-for-bug compatibility, this method ensures we only
add new
http://git-wip-us.apache.org/repos/asf/hive/blob/699c5768/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
index 91af814..aba6591 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
@@ -52,6 +52,7 @@ public class ExportTask extends Task implements
Serializable {
conf, false);
Hive db = getHive();
LOG.debug("Exporting data to: {}", exportPaths.getExportRootDir());
+ work.acidPostProcess(db);
TableExport tableExport = new TableExport(
exportPaths, work.getTableSpec(), work.getReplicationSpec(), db,
