This is an automated email from the ASF dual-hosted git repository. lpinter pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new d237a30728 HIVE-26228: Implement Iceberg table rollback feature (#3287) (Laszlo Pinter, reviewed by Adam Szita and Peter Vary) d237a30728 is described below commit d237a307286ee0431a9a9ad148896041b5a13ebd Author: László Pintér <47777102+lcspin...@users.noreply.github.com> AuthorDate: Fri Jun 3 08:26:58 2022 +0200 HIVE-26228: Implement Iceberg table rollback feature (#3287) (Laszlo Pinter, reviewed by Adam Szita and Peter Vary) --- .../src/test/results/negative/hbase_ddl.q.out | 2 +- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 2 +- .../iceberg/mr/hive/HiveIcebergStorageHandler.java | 22 ++++ .../apache/iceberg/mr/hive/IcebergTableUtil.java | 20 ++++ .../iceberg/mr/hive/HiveIcebergTestUtils.java | 22 ++++ .../iceberg/mr/hive/TestHiveIcebergRollback.java | 133 +++++++++++++++++++++ .../iceberg/mr/hive/TestHiveIcebergTimeTravel.java | 63 +++------- .../org/apache/iceberg/mr/hive/TestTables.java | 30 +++++ .../test/queries/positive/rollback_iceberg_table.q | 9 ++ .../results/positive/rollback_iceberg_table.q.out | 30 +++++ .../hadoop/hive/ql/parse/AlterClauseParser.g | 8 ++ .../org/apache/hadoop/hive/ql/parse/HiveParser.g | 1 + .../ddl/table/AbstractBaseAlterTableAnalyzer.java | 6 +- .../hadoop/hive/ql/ddl/table/AlterTableType.java | 3 +- .../table/execute/AlterTableExecuteAnalyzer.java | 85 +++++++++++++ .../ddl/table/execute/AlterTableExecuteDesc.java | 58 +++++++++ .../table/execute/AlterTableExecuteOperation.java | 40 +++++++ .../org/apache/hadoop/hive/ql/metadata/Hive.java | 11 +- .../hive/ql/metadata/HiveStorageHandler.java | 8 ++ .../hive/ql/parse/AlterTableExecuteSpec.java | 94 +++++++++++++++ .../apache/hadoop/hive/ql/plan/HiveOperation.java | 1 + .../authorization/plugin/HiveOperationType.java | 1 + .../plugin/sqlstd/Operation2Privilege.java | 2 + .../results/clientnegative/alter_non_native.q.out | 2 +- 24 files changed, 595 insertions(+), 58 deletions(-) diff --git a/hbase-handler/src/test/results/negative/hbase_ddl.q.out b/hbase-handler/src/test/results/negative/hbase_ddl.q.out index e4c146b8a7..60d7279814 100644 --- a/hbase-handler/src/test/results/negative/hbase_ddl.q.out +++ b/hbase-handler/src/test/results/negative/hbase_ddl.q.out @@ -26,4 +26,4 @@ key int It is a column key value string It is the column string value #### A masked pattern was here #### -FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADDCOLS] to a non-native table hbase_table_1 +FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADDCOLS, EXECUTE] to a non-native table hbase_table_1 diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 13142ce144..c8f652d577 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -108,7 +108,7 @@ public class HiveIcebergMetaHook implements HiveMetaHook { static final EnumSet<AlterTableType> SUPPORTED_ALTER_OPS = EnumSet.of( AlterTableType.ADDCOLS, AlterTableType.REPLACE_COLUMNS, AlterTableType.RENAME_COLUMN, AlterTableType.ADDPROPS, AlterTableType.DROPPROPS, AlterTableType.SETPARTITIONSPEC, - AlterTableType.UPDATE_COLUMNS); + AlterTableType.UPDATE_COLUMNS, AlterTableType.SETPARTITIONSPEC, AlterTableType.EXECUTE); private static final List<String> MIGRATION_ALLOWED_SOURCE_FORMATS = ImmutableList.of( FileFormat.PARQUET.name().toLowerCase(), FileFormat.ORC.name().toLowerCase(), diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index a6b5c6deba..c693c941e4 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; +import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec; import org.apache.hadoop.hive.ql.parse.PartitionTransformSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; @@ -451,6 +452,27 @@ public class HiveIcebergStorageHandler implements HiveStoragePredicateHandler, H return true; } + @Override + public void executeOperation(org.apache.hadoop.hive.ql.metadata.Table hmsTable, AlterTableExecuteSpec executeSpec) { + switch (executeSpec.getOperationType()) { + case ROLLBACK: + TableDesc tableDesc = Utilities.getTableDesc(hmsTable); + Table icebergTable = IcebergTableUtil.getTable(conf, tableDesc.getProperties()); + LOG.info("Executing rollback operation on iceberg table. If you would like to revert rollback you could " + + "try altering the metadata location to the current metadata location by executing the following query:" + + "ALTER TABLE {}.{} SET TBLPROPERTIES('metadata_location'='{}'). This operation is supported for Hive " + + "Catalog tables.", hmsTable.getDbName(), hmsTable.getTableName(), + ((BaseTable) icebergTable).operations().current().metadataFileLocation()); + AlterTableExecuteSpec.RollbackSpec rollbackSpec = + (AlterTableExecuteSpec.RollbackSpec) executeSpec.getOperationParams(); + IcebergTableUtil.rollback(icebergTable, rollbackSpec.getRollbackType(), rollbackSpec.getParam()); + break; + default: + throw new UnsupportedOperationException( + String.format("Operation type %s is not supported", executeSpec.getOperationType().name())); + } + } + @Override public boolean isValidMetadataTable(String metaTableName) { return IcebergMetadataTables.isValidMetaTable(metaTableName); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 6e2c01a72a..6e471f7be3 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -24,8 +24,10 @@ import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec; import org.apache.hadoop.hive.ql.parse.PartitionTransformSpec; import org.apache.hadoop.hive.ql.session.SessionStateUtil; +import org.apache.iceberg.ManageSnapshots; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; @@ -184,4 +186,22 @@ public class IcebergTableUtil { public static boolean isBucketed(Table table) { return table.spec().fields().stream().anyMatch(f -> f.transform().toString().startsWith("bucket[")); } + + /** + * Roll an iceberg table's data back to a specific snapshot identified either by id or before a given timestamp. + * @param table the iceberg table + * @param type the type of the rollback, can be either time based or version based + * @param value parameter of the rollback, that can be a timestamp in millis or a snapshot id + */ + public static void rollback(Table table, AlterTableExecuteSpec.RollbackSpec.RollbackType type, Long value) { + ManageSnapshots manageSnapshots = table.manageSnapshots(); + if (type == AlterTableExecuteSpec.RollbackSpec.RollbackType.TIME) { + LOG.debug("Trying to rollback iceberg table to snapshot before timestamp {}", value); + manageSnapshots.rollbackToTime(value); + } else { + LOG.debug("Trying to rollback iceberg table to snapshot ID {}", value); + manageSnapshots.rollbackTo(value); + } + manageSnapshots.commit(); + } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergTestUtils.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergTestUtils.java index 79e3dfee9e..5e217acc82 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergTestUtils.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergTestUtils.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Timestamp; +import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -35,6 +36,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.util.Arrays; import java.util.Comparator; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.UUID; @@ -59,6 +61,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobID; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.HistoryEntry; import org.apache.iceberg.PartitionKey; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; @@ -379,4 +382,23 @@ public class HiveIcebergTestUtils { return posWriter.toDeleteFile(); } + /** + * Get the timestamp string which we can use in the queries. The timestamp will be after the given snapshot + * and before the next one + * @param table The table which we want to query + * @param snapshotPosition The position of the last snapshot we want to see in the query results + * @return The timestamp which we can use in the queries + */ + public static String timestampAfterSnapshot(Table table, int snapshotPosition) { + List<HistoryEntry> history = table.history(); + long snapshotTime = history.get(snapshotPosition).timestampMillis(); + long time = snapshotTime + 100; + if (history.size() > snapshotPosition + 1) { + time = snapshotTime + ((history.get(snapshotPosition + 1).timestampMillis() - snapshotTime) / 2); + } + + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS000000"); + return simpleDateFormat.format(new Date(time)); + } + } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergRollback.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergRollback.java new file mode 100644 index 0000000000..52a3cccd72 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergRollback.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive; + + +import java.io.IOException; +import org.apache.iceberg.AssertHelpers; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.TableIdentifier; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + + +/** + * Tests covering the rollback feature + */ +public class TestHiveIcebergRollback extends HiveIcebergStorageHandlerWithEngineBase { + + @Test + public void testRollbackToTimestamp() throws IOException, InterruptedException { + TableIdentifier identifier = TableIdentifier.of("default", "source"); + Table table = testTables.createTableWithVersions(shell, identifier.name(), + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, fileFormat, + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 3); + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK('" + + HiveIcebergTestUtils.timestampAfterSnapshot(table, 2) + "')"); + Assert.assertEquals(5, shell.executeStatement("SELECT * FROM " + identifier.name()).size()); + Assert.assertEquals(3, table.history().size()); + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK('" + + HiveIcebergTestUtils.timestampAfterSnapshot(table, 1) + "')"); + Assert.assertEquals(4, shell.executeStatement("SELECT * FROM " + identifier.name()).size()); + table.refresh(); + Assert.assertEquals(4, table.history().size()); + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK('" + + HiveIcebergTestUtils.timestampAfterSnapshot(table, 0) + "')"); + Assert.assertEquals(3, shell.executeStatement("SELECT * FROM " + identifier.name()).size()); + table.refresh(); + Assert.assertEquals(5, table.history().size()); + } + + @Test + public void testRollbackToVersion() throws IOException, InterruptedException { + TableIdentifier identifier = TableIdentifier.of("default", "source"); + Table table = testTables.createTableWithVersions(shell, identifier.name(), + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, fileFormat, + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 3); + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK(" + + table.history().get(2).snapshotId() + ")"); + Assert.assertEquals(5, shell.executeStatement("SELECT * FROM " + identifier.name()).size()); + table.refresh(); + Assert.assertEquals(3, table.history().size()); + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK(" + + table.history().get(1).snapshotId() + ")"); + Assert.assertEquals(4, shell.executeStatement("SELECT * FROM " + identifier.name()).size()); + table.refresh(); + Assert.assertEquals(4, table.history().size()); + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK(" + + table.history().get(0).snapshotId() + ")"); + Assert.assertEquals(3, shell.executeStatement("SELECT * FROM " + identifier.name()).size()); + table.refresh(); + Assert.assertEquals(5, table.history().size()); + } + + @Test + public void testRevertRollback() throws IOException, InterruptedException { + Assume.assumeTrue("Rollback revert is only supported for tables from Hive Catalog", + testTableType.equals(TestTables.TestTableType.HIVE_CATALOG)); + TableIdentifier identifier = TableIdentifier.of("default", "source"); + Table table = testTables.createTableWithVersions(shell, identifier.name(), + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, fileFormat, + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); + String metadataLocationBeforeRollback = ((BaseTable) table).operations().current().metadataFileLocation(); + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK(" + + table.history().get(0).snapshotId() + ")"); + Assert.assertEquals(3, shell.executeStatement("SELECT * FROM " + identifier.name()).size()); + table.refresh(); + Assert.assertEquals(3, table.history().size()); + shell.executeStatement("ALTER TABLE " + identifier.name() + " SET TBLPROPERTIES('metadata_location'='" + + metadataLocationBeforeRollback + "')"); + Assert.assertEquals(4, shell.executeStatement("SELECT * FROM " + identifier.name()).size()); + table.refresh(); + Assert.assertEquals(2, table.history().size()); + } + + @Test + public void testInvalidRollbackToTimestamp() throws IOException, InterruptedException { + TableIdentifier identifier = TableIdentifier.of("default", "source"); + testTables.createTableWithVersions(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); + AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class, + "Cannot roll back, no valid snapshot older than", () -> { + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK('1970-01-01 00:00:00')"); + }); + } + + @Test + public void testInvalidRollbackToVersion() throws IOException, InterruptedException { + TableIdentifier identifier = TableIdentifier.of("default", "source"); + Table table = testTables.createTableWithVersions(shell, identifier.name(), + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, fileFormat, + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); + AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class, + "Cannot roll back to unknown snapshot id", () -> { + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK(1111)"); + }); + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK(" + + table.history().get(0).snapshotId() + ")"); + AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class, + "Cannot roll back to snapshot, not an ancestor of the current state", () -> { + shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE ROLLBACK(" + + table.history().get(1).snapshotId() + ")"); + }); + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTimeTravel.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTimeTravel.java index e2aac33d82..233f87a857 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTimeTravel.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTimeTravel.java @@ -20,8 +20,6 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; import java.util.List; import org.apache.iceberg.AssertHelpers; import org.apache.iceberg.HistoryEntry; @@ -29,6 +27,8 @@ import org.apache.iceberg.Table; import org.junit.Assert; import org.junit.Test; +import static org.apache.iceberg.mr.hive.HiveIcebergTestUtils.timestampAfterSnapshot; + /** * Tests covering the time travel feature, aka reading from a table as of a certain snapshot. */ @@ -36,7 +36,9 @@ public class TestHiveIcebergTimeTravel extends HiveIcebergStorageHandlerWithEngi @Test public void testSelectAsOfTimestamp() throws IOException, InterruptedException { - Table table = prepareTableWithVersions(2); + Table table = testTables.createTableWithVersions(shell, "customers", + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); List<Object[]> rows = shell.executeStatement( "SELECT * FROM customers FOR SYSTEM_TIME AS OF '" + timestampAfterSnapshot(table, 0) + "'"); @@ -56,7 +58,9 @@ public class TestHiveIcebergTimeTravel extends HiveIcebergStorageHandlerWithEngi @Test public void testSelectAsOfVersion() throws IOException, InterruptedException { - Table table = prepareTableWithVersions(2); + Table table = testTables.createTableWithVersions(shell, "customers", + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); HistoryEntry first = table.history().get(0); List<Object[]> rows = @@ -77,7 +81,9 @@ public class TestHiveIcebergTimeTravel extends HiveIcebergStorageHandlerWithEngi @Test public void testCTASAsOfVersionAndTimestamp() throws IOException, InterruptedException { - Table table = prepareTableWithVersions(3); + Table table = testTables.createTableWithVersions(shell, "customers", + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, fileFormat, + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 3); shell.executeStatement("CREATE TABLE customers2 AS SELECT * FROM customers FOR SYSTEM_VERSION AS OF " + table.history().get(0).snapshotId()); @@ -106,7 +112,9 @@ public class TestHiveIcebergTimeTravel extends HiveIcebergStorageHandlerWithEngi @Test public void testAsOfWithJoins() throws IOException, InterruptedException { - Table table = prepareTableWithVersions(4); + Table table = testTables.createTableWithVersions(shell, "customers", + HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, fileFormat, + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 4); List<Object[]> rows = shell.executeStatement("SELECT * FROM " + "customers FOR SYSTEM_TIME AS OF '" + timestampAfterSnapshot(table, 0) + "' fv, " + @@ -136,47 +144,4 @@ public class TestHiveIcebergTimeTravel extends HiveIcebergStorageHandlerWithEngi Assert.assertEquals(8, rows.size()); } - - /** - * Creates the 'customers' table with the default records and creates extra snapshots by inserting one more line - * into the table. - * @param versions The number of history elements we want to create - * @return The table created - * @throws IOException When there is a problem during table creation - * @throws InterruptedException When there is a problem during adding new data to the table - */ - private Table prepareTableWithVersions(int versions) throws IOException, InterruptedException { - Table table = testTables.createTable(shell, "customers", HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, - fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS); - - for (int i = 0; i < versions - 1; ++i) { - // Just wait a little so we definitely will not have the same timestamp for the snapshots - Thread.sleep(100); - shell.executeStatement("INSERT INTO customers values(" + - (i + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS.size()) + ",'Alice','Green_" + i + "')"); - } - - table.refresh(); - - return table; - } - - /** - * Get the timestamp string which we can use in the queries. The timestamp will be after the given snapshot - * and before the next one - * @param table The table which we want to query - * @param snapshotPosition The position of the last snapshot we want to see in the query results - * @return The timestamp which we can use in the queries - */ - private String timestampAfterSnapshot(Table table, int snapshotPosition) { - List<HistoryEntry> history = table.history(); - long snapshotTime = history.get(snapshotPosition).timestampMillis(); - long time = snapshotTime + 100; - if (history.size() > snapshotPosition + 1) { - time = snapshotTime + ((history.get(snapshotPosition + 1).timestampMillis() - snapshotTime) / 2); - } - - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS000000"); - return simpleDateFormat.format(new Date(time)); - } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestTables.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestTables.java index 91c70dde62..f2b58d776c 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestTables.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestTables.java @@ -169,6 +169,36 @@ abstract class TestTables { return createTable(shell, tableName, schema, fileFormat, records, 1); } + + /** + * Creates a non partitioned Hive test table. Creates the Iceberg table/data and creates the corresponding Hive + * table as well when needed. The table will be in the 'default' database. The table will be populated with the + * provided List of {@link Record}s and will create extra snapshots by inserting one more line into the table. + * @param shell The HiveShell used for Hive table creation + * @param tableName The name of the test table + * @param schema The schema used for the table creation + * @param fileFormat The file format used for writing the data + * @param records The records with which the table is populated + * @param versions The number of history elements we want to create + * @return The created table + * @throws IOException If there is an error writing data + */ + public Table createTableWithVersions(TestHiveShell shell, String tableName, Schema schema, FileFormat fileFormat, + List<Record> records, int versions) throws IOException, InterruptedException { + Table table = createTable(shell, tableName, schema, fileFormat, records); + + for (int i = 0; i < versions - 1; ++i) { + // Just wait a little so we definitely will not have the same timestamp for the snapshots + Thread.sleep(100); + shell.executeStatement("INSERT INTO " + tableName + " values(" + + (i + records.size()) + ",'Alice','Green_" + i + "')"); + } + + table.refresh(); + + return table; + } + /** * Creates a non partitioned Hive test table. Creates the Iceberg table/data and creates the corresponding Hive * table as well when needed. The table will be in the 'default' database. The table will be populated with the diff --git a/iceberg/iceberg-handler/src/test/queries/positive/rollback_iceberg_table.q b/iceberg/iceberg-handler/src/test/queries/positive/rollback_iceberg_table.q new file mode 100644 index 0000000000..0db54dd38b --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/rollback_iceberg_table.q @@ -0,0 +1,9 @@ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ +-- Mask random uuid +--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ + +drop table if exists tbl_ice; +create external table tbl_ice(a int, b string, c int) stored by iceberg stored as parquet; +explain alter table tbl_ice execute rollback(11111); +explain alter table tbl_ice execute rollback('2022-05-12 00:00:00'); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/positive/rollback_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/rollback_iceberg_table.q.out new file mode 100644 index 0000000000..394859ffe6 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/rollback_iceberg_table.q.out @@ -0,0 +1,30 @@ +PREHOOK: query: drop table if exists tbl_ice +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists tbl_ice +POSTHOOK: type: DROPTABLE +PREHOOK: query: create external table tbl_ice(a int, b string, c int) stored by iceberg stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tbl_ice +POSTHOOK: query: create external table tbl_ice(a int, b string, c int) stored by iceberg stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tbl_ice +PREHOOK: query: explain alter table tbl_ice execute rollback(11111) +PREHOOK: type: ALTERTABLE_EXECUTE +PREHOOK: Input: default@tbl_ice +POSTHOOK: query: explain alter table tbl_ice execute rollback(11111) +POSTHOOK: type: ALTERTABLE_EXECUTE +POSTHOOK: Input: default@tbl_ice +Stage-0 + Execute operation{"table name:":"default.tbl_ice","spec:":"AlterTableExecuteSpec{operationType=ROLLBACK, operationParams=RollbackSpec{rollbackType=VERSION, param=11111}}"} + +PREHOOK: query: explain alter table tbl_ice execute rollback('2022-05-12 00:00:00') +PREHOOK: type: ALTERTABLE_EXECUTE +PREHOOK: Input: default@tbl_ice +POSTHOOK: query: explain alter table tbl_ice execute rollback('2022-05-12 00:00:00') +POSTHOOK: type: ALTERTABLE_EXECUTE +POSTHOOK: Input: default@tbl_ice +Stage-0 + Execute operation{"table name:":"default.tbl_ice","spec:":"AlterTableExecuteSpec{operationType=ROLLBACK, operationParams=RollbackSpec{rollbackType=TIME, param=1652338800000}}"} + diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g index 321eb1af81..a89055d760 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g @@ -73,6 +73,7 @@ alterTableStatementSuffix | partitionSpec alterTblPartitionStatementSuffix[true] -> alterTblPartitionStatementSuffix partitionSpec | alterStatementSuffixSetOwner | alterStatementSuffixSetPartSpec + | alterStatementSuffixExecute ; alterTblPartitionStatementSuffix[boolean partition] @@ -452,6 +453,13 @@ alterStatementSuffixSetPartSpec -> ^(TOK_ALTERTABLE_SETPARTSPEC $spec) ; +alterStatementSuffixExecute +@init { gParent.pushMsg("alter table execute", state); } +@after { gParent.popMsg(state); } + : KW_EXECUTE KW_ROLLBACK LPAREN (rollbackParam=(StringLiteral | Number)) RPAREN + -> ^(TOK_ALTERTABLE_EXECUTE KW_ROLLBACK $rollbackParam) + ; + fileFormat @init { gParent.pushMsg("file format specification", state); } @after { gParent.popMsg(state); } diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 9a2c485360..25bd5a259f 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -215,6 +215,7 @@ TOK_ALTERTABLE_ADDCONSTRAINT; TOK_ALTERTABLE_UPDATECOLUMNS; TOK_ALTERTABLE_OWNER; TOK_ALTERTABLE_SETPARTSPEC; +TOK_ALTERTABLE_EXECUTE; TOK_MSCK; TOK_SHOWDATABASES; TOK_SHOWDATACONNECTORS; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractBaseAlterTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractBaseAlterTableAnalyzer.java index 03aa02293b..17f9fec4d1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractBaseAlterTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractBaseAlterTableAnalyzer.java @@ -159,12 +159,10 @@ public abstract class AbstractBaseAlterTableAnalyzer extends BaseSemanticAnalyze throw new SemanticException(ErrorMsg.ALTER_COMMAND_FOR_TABLES.getMsg()); } } - if (table.isNonNative() && table.getStorageHandler() != null) { - if (!table.getStorageHandler().isAllowedAlterOperation(op) || - (op == AlterTableType.SETPARTITIONSPEC && !table.getStorageHandler().supportsPartitionTransform())) { + if (table.isNonNative() && table.getStorageHandler() != null && + !table.getStorageHandler().isAllowedAlterOperation(op)) { throw new SemanticException(ErrorMsg.ALTER_TABLE_NON_NATIVE.format( AlterTableType.NON_NATIVE_TABLE_ALLOWED.toString(), table.getTableName())); - } } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java index 9b1de08956..df5ba186b6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java @@ -39,6 +39,7 @@ public enum AlterTableType { RENAMEPARTITION("rename partition"), // Note: used in RenamePartitionDesc, not here. ALTERPARTITION("alter partition"), // Note: this is never used in AlterTableDesc. SETPARTITIONSPEC("set partition spec"), + EXECUTE("execute"), // constraint ADD_CONSTRAINT("add constraint"), DROP_CONSTRAINT("drop constraint"), @@ -78,7 +79,7 @@ public enum AlterTableType { } public static final List<AlterTableType> NON_NATIVE_TABLE_ALLOWED = - ImmutableList.of(ADDPROPS, DROPPROPS, ADDCOLS); + ImmutableList.of(ADDPROPS, DROPPROPS, ADDCOLS, EXECUTE); public static final Set<AlterTableType> SUPPORT_PARTIAL_PARTITION_SPEC = ImmutableSet.of(ADDCOLS, REPLACE_COLUMNS, RENAME_COLUMN, ADDPROPS, DROPPROPS, SET_SERDE, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/execute/AlterTableExecuteAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/execute/AlterTableExecuteAnalyzer.java new file mode 100644 index 0000000000..6c4471dc60 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/execute/AlterTableExecuteAnalyzer.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.execute; + +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.common.type.TimestampTZ; +import org.apache.hadoop.hive.common.type.TimestampTZUtil; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType; +import org.apache.hadoop.hive.ql.ddl.DDLWork; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.session.SessionState; + +import java.time.ZoneId; +import java.util.Map; + +import static org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec.ExecuteOperationType.ROLLBACK; +import static org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec.RollbackSpec.RollbackType.TIME; +import static org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec.RollbackSpec.RollbackType.VERSION; + +/** + * Analyzer for ALTER TABLE ... EXECUTE commands. + */ +@DDLType(types = HiveParser.TOK_ALTERTABLE_EXECUTE) +public class AlterTableExecuteAnalyzer extends AbstractAlterTableAnalyzer { + + public AlterTableExecuteAnalyzer(QueryState queryState) throws SemanticException { + super(queryState); + } + + @Override + protected void analyzeCommand(TableName tableName, Map<String, String> partitionSpec, ASTNode command) + throws SemanticException { + Table table = getTable(tableName); + // the first child must be the execute operation type + ASTNode executeCommandType = (ASTNode) command.getChild(0); + validateAlterTableType(table, AlterTableType.EXECUTE, false); + inputs.add(new ReadEntity(table)); + AlterTableExecuteDesc desc = null; + if (HiveParser.KW_ROLLBACK == executeCommandType.getType()) { + AlterTableExecuteSpec<AlterTableExecuteSpec.RollbackSpec> spec; + // the second child must be the rollback parameter + ASTNode child = (ASTNode) command.getChild(1); + + if (child.getType() == HiveParser.StringLiteral) { + ZoneId timeZone = SessionState.get() == null ? new HiveConf().getLocalTimeZone() : SessionState.get().getConf() + .getLocalTimeZone(); + TimestampTZ time = TimestampTZUtil.parse(PlanUtils.stripQuotes(child.getText()), timeZone); + spec = new AlterTableExecuteSpec(ROLLBACK, new AlterTableExecuteSpec.RollbackSpec(TIME, time.toEpochMilli())); + } else { + spec = new AlterTableExecuteSpec(ROLLBACK, new AlterTableExecuteSpec.RollbackSpec(VERSION, + Long.valueOf(child.getText()))); + } + desc = new AlterTableExecuteDesc(tableName, partitionSpec, spec); + } + + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/execute/AlterTableExecuteDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/execute/AlterTableExecuteDesc.java new file mode 100644 index 0000000000..ba5ddf55d9 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/execute/AlterTableExecuteDesc.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.ddl.table.execute; + +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; +import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +import java.util.Map; + +/** + * DDL task task description for ALTER TABLE ... EXECUTE commands. + */ +@Explain(displayName = "Execute operation", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableExecuteDesc extends AbstractAlterTableDesc { + private static final long serialVersionUID = 1L; + + private final AlterTableExecuteSpec executeSpec; + + public AlterTableExecuteDesc(TableName tableName, Map<String, String> partitionSpec, AlterTableExecuteSpec executeSpec) + throws SemanticException { + super(AlterTableType.EXECUTE, tableName, partitionSpec, null, false, false, null); + this.executeSpec = executeSpec; + } + + public AlterTableExecuteSpec getExecuteSpec() { + return executeSpec; + } + + @Explain(displayName = "spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getExplainOutput() { + return executeSpec.toString(); + } + + @Override + public boolean mayNeedWriteId() { + return false; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/execute/AlterTableExecuteOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/execute/AlterTableExecuteOperation.java new file mode 100644 index 0000000000..e36234c7ef --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/execute/AlterTableExecuteOperation.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.execute; + +import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.metadata.Table; + +/** + * Operation process of ALTER TABLE ... EXECUTE command + */ +public class AlterTableExecuteOperation extends DDLOperation<AlterTableExecuteDesc> { + + public AlterTableExecuteOperation(DDLOperationContext context, AlterTableExecuteDesc desc) { + super(context, desc); + } + + @Override + public int execute() throws Exception { + Table table = context.getDb().getTable(desc.getFullTableName()); + context.getDb().alterTableExecuteOperation(table, desc.getExecuteSpec()); + return 0; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index d2617ca006..719c3edf24 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -211,6 +211,7 @@ import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; @@ -223,7 +224,6 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; @@ -6505,4 +6505,13 @@ private void constructOneLBLocationMap(FileStatus fSta, throw new HiveException(e); } } + + public void alterTableExecuteOperation(Table table, AlterTableExecuteSpec executeSpec) throws HiveException { + try { + HiveStorageHandler storageHandler = createStorageHandler(table.getTTable()); + storageHandler.executeOperation(table, executeSpec); + } catch (MetaException e) { + throw new HiveException(e); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 5dac8c0661..b4b3cfca9d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.Context.Operation; import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec; import org.apache.hadoop.hive.ql.parse.PartitionTransformSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; @@ -462,4 +463,11 @@ public interface HiveStorageHandler extends Configurable { */ default void validateSinkDesc(FileSinkDesc sinkDesc) throws SemanticException { } + + /** + * Execute an operation on storage handler level + * @param executeSpec operation specification + */ + default void executeOperation(org.apache.hadoop.hive.ql.metadata.Table table, AlterTableExecuteSpec executeSpec) { + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableExecuteSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableExecuteSpec.java new file mode 100644 index 0000000000..5480b090c3 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableExecuteSpec.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import com.google.common.base.MoreObjects; + +/** + * Execute operation specification. It stores the type of the operation and its parameters. + * The following operation are supported + * <ul> + * <li>Rollback</li> + * </ul> + * @param <T> Value object class to store the operation specific parameters + */ +public class AlterTableExecuteSpec<T> { + + public enum ExecuteOperationType { + ROLLBACK + } + + private final ExecuteOperationType operationType; + private final T operationParams; + + public AlterTableExecuteSpec(ExecuteOperationType type, T value) { + this.operationType = type; + this.operationParams = value; + } + + public ExecuteOperationType getOperationType() { + return operationType; + } + + public T getOperationParams() { + return operationParams; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("operationType", operationType.name()) + .add("operationParams", operationParams).toString(); + } + + /** + * Value object class, that stores the rollback operation specific parameters + * <ul> + * <li>Rollback type: it can be either version based or time based</li> + * <li>Rollback value: it should either a snapshot id or a timestamp in milliseconds</li> + * </ul> + */ + public static class RollbackSpec { + + public enum RollbackType { + VERSION, TIME + } + + private final RollbackType rollbackType; + private final Long param; + + public RollbackSpec(RollbackType rollbackType, Long param) { + this.rollbackType = rollbackType; + this.param = param; + } + + public RollbackType getRollbackType() { + return rollbackType; + } + + public Long getParam() { + return param; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("rollbackType", rollbackType.name()) + .add("param", param).toString(); + } + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java index 0c72ed7d04..03c40e68b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java @@ -76,6 +76,7 @@ public enum HiveOperation { new Privilege[]{Privilege.ALTER_METADATA}, null), ALTERTABLE_OWNER("ALTERTABLE_OWNER", HiveParser.TOK_ALTERTABLE_OWNER, null, null), ALTERTABLE_SETPARTSPEC("ALTERTABLE_SETPARTSPEC", HiveParser.TOK_ALTERTABLE_SETPARTSPEC, null, null), + ALTERTABLE_EXECUTE("ALTERTABLE_EXECUTE", HiveParser.TOK_ALTERTABLE_EXECUTE, null, null), ALTERTABLE_SERIALIZER("ALTERTABLE_SERIALIZER", HiveParser.TOK_ALTERTABLE_SERIALIZER, new Privilege[]{Privilege.ALTER_METADATA}, null), ALTERPARTITION_SERIALIZER("ALTERPARTITION_SERIALIZER", HiveParser.TOK_ALTERPARTITION_SERIALIZER, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java index d903b9a551..52023affd9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java @@ -59,6 +59,7 @@ public enum HiveOperationType { ALTERTABLE_PROPERTIES, ALTERTABLE_OWNER, ALTERTABLE_SETPARTSPEC, + ALTERTABLE_EXECUTE, ALTERTABLE_SERIALIZER, ALTERTABLE_PARTCOLTYPE, ALTERTABLE_DROPCONSTRAINT, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java index 354f4f0e04..dcd9d9aac7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java @@ -190,6 +190,8 @@ public class Operation2Privilege { PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); op2Priv.put(HiveOperationType.ALTERTABLE_SETPARTSPEC, PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); + op2Priv.put(HiveOperationType.ALTERTABLE_EXECUTE, + PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); op2Priv.put(HiveOperationType.ALTERTABLE_SERIALIZER, PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); op2Priv.put(HiveOperationType.ALTERTABLE_PARTCOLTYPE, diff --git a/ql/src/test/results/clientnegative/alter_non_native.q.out b/ql/src/test/results/clientnegative/alter_non_native.q.out index bd8fb4fbf5..922d2147f4 100644 --- a/ql/src/test/results/clientnegative/alter_non_native.q.out +++ b/ql/src/test/results/clientnegative/alter_non_native.q.out @@ -8,4 +8,4 @@ STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler' POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@non_native1 -FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADDCOLS] to a non-native table non_native1 +FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADDCOLS, EXECUTE] to a non-native table non_native1