date:20231106

(hive) branch master updated: HIVE-27789: Iceberg: Add a way to expire snapshots with retain last. (#4835). (Ayush Saxena, reviewed by Denys Kuzmenko, zhangbutao)

2023-11-06 Thread ayushsaxena

This is an automated email from the ASF dual-hosted git repository.

ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
 new 3b2e8a9f4a1 HIVE-27789: Iceberg: Add a way to expire snapshots with 
retain last. (#4835). (Ayush Saxena, reviewed by Denys Kuzmenko, zhangbutao)
3b2e8a9f4a1 is described below

commit 3b2e8a9f4a120e269abccdb93a649cf3132ae5de
Author: Ayush Saxena 
AuthorDate: Tue Nov 7 07:34:06 2023 +0530

HIVE-27789: Iceberg: Add a way to expire snapshots with retain last. 
(#4835). (Ayush Saxena, reviewed by Denys Kuzmenko, zhangbutao)
---
 .../iceberg/mr/hive/HiveIcebergStorageHandler.java  | 11 +++
 .../mr/hive/TestHiveIcebergExpireSnapshots.java | 21 +
 .../apache/hadoop/hive/ql/parse/AlterClauseParser.g |  2 ++
 .../table/execute/AlterTableExecuteAnalyzer.java|  8 +++-
 .../hadoop/hive/ql/parse/AlterTableExecuteSpec.java | 16 
 5 files changed, 57 insertions(+), 1 deletion(-)

diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index 0afdd79ba55..538c7316779 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -867,6 +867,8 @@ public class HiveIcebergStorageHandler implements 
HiveStoragePredicateHandler, H
 expireSnapshotsSpec.getTimestampMillis(), deleteExecutorService);
   } else if (expireSnapshotsSpec.isExpireByIds()) {
 expireSnapshotByIds(icebergTable, 
expireSnapshotsSpec.getIdsToExpire(), deleteExecutorService);
+  } else if (expireSnapshotsSpec.isExpireByRetainLast()) {
+expireSnapshotRetainLast(icebergTable, 
expireSnapshotsSpec.getNumRetainLast(), deleteExecutorService);
   } else {
 expireSnapshotOlderThanTimestamp(icebergTable, 
expireSnapshotsSpec.getTimestampMillis(), deleteExecutorService);
   }
@@ -877,6 +879,15 @@ public class HiveIcebergStorageHandler implements 
HiveStoragePredicateHandler, H
 }
   }
 
+  private void expireSnapshotRetainLast(Table icebergTable, int numRetainLast, 
ExecutorService deleteExecutorService) {
+ExpireSnapshots expireSnapshots = icebergTable.expireSnapshots();
+expireSnapshots.retainLast(numRetainLast);
+if (deleteExecutorService != null) {
+  expireSnapshots.executeDeleteWith(deleteExecutorService);
+}
+expireSnapshots.commit();
+  }
+
   private void expireSnapshotByTimestampRange(Table icebergTable, Long 
fromTimestamp, Long toTimestamp,
   ExecutorService deleteExecutorService) {
 ExpireSnapshots expireSnapshots = icebergTable.expireSnapshots();
diff --git 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergExpireSnapshots.java
 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergExpireSnapshots.java
index a851578ee6c..8fd808f4fed 100644
--- 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergExpireSnapshots.java
+++ 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergExpireSnapshots.java
@@ -28,6 +28,8 @@ import org.apache.iceberg.catalog.TableIdentifier;
 import org.junit.Assert;
 import org.junit.Test;
 
+import static org.apache.iceberg.TableProperties.MAX_SNAPSHOT_AGE_MS;
+
 /**
  * Tests covering the rollback feature
  */
@@ -82,4 +84,23 @@ public class TestHiveIcebergExpireSnapshots extends 
HiveIcebergStorageHandlerWit
 table.refresh();
 Assert.assertEquals(6, IterableUtils.size(table.snapshots()));
   }
+
+  @Test
+  public void testExpireSnapshotsWithRetainLast() throws IOException, 
InterruptedException {
+TableIdentifier identifier = TableIdentifier.of("default", "source");
+Table table = testTables.createTableWithVersions(shell, identifier.name(),
+HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, fileFormat,
+HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 10);
+// No snapshot should expire, since the max snapshot age to expire is by 
default 5 days
+shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE 
EXPIRE_SNAPSHOTS RETAIN LAST 5");
+table.refresh();
+Assert.assertEquals(10, IterableUtils.size(table.snapshots()));
+
+// Change max snapshot age to expire to 1 ms & re-execute, this time it 
should retain only 5
+shell.executeStatement(
+"ALTER TABLE " + identifier.name() + " SET TBLPROPERTIES('" + 
MAX_SNAPSHOT_AGE_MS + "'='1')");
+shell.executeStatement("ALTER TABLE " + identifier.name() + " EXECUTE 
EXPIRE_SNAPSHOTS RETAIN LAST 5");
+table.refresh();
+Assert.assertEquals(5, IterableUtils.size(table.snapshots()));
+  }

(hive) branch master updated: HIVE-27846: HIVE-27849: Tests under hive-unit module are not running (Stamatis Zampetakis reviewed by Ayush Saxena)

2023-11-06 Thread zabetak

This is an automated email from the ASF dual-hosted git repository.

zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
 new 698194183a2 HIVE-27846: HIVE-27849: Tests under hive-unit module are 
not running (Stamatis Zampetakis reviewed by Ayush Saxena)
698194183a2 is described below

commit 698194183a2cb4ee6bc528ee71462cbae2c2c7ad
Author: Stamatis Zampetakis 
AuthorDate: Thu Nov 2 12:52:41 2023 +0100

HIVE-27846: HIVE-27849: Tests under hive-unit module are not running 
(Stamatis Zampetakis reviewed by Ayush Saxena)

1. Drop jupiter-engine dependency from hive-unit module preventing
tests from running.
2. Remove redundant jupiter annotations from TestStorageSchemaReader to
avoid dependencies to jupiter API.
3. Revert ivy upgrade cause since it leads to failures in replication
tests:
* TestReplAcrossInstancesWithJsonMessageFormat;
* TestReplicationScenariosAcrossInstances tests.
4. Disable 
TestCrudCompactorOnTez#secondCompactionShouldBeRefusedBeforeEnqueueing
test till HIVE-27848 is resolved.

Close apache/hive#4850
---
 itests/hive-unit/pom.xml| 5 -
 .../org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java   | 6 ++
 .../apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java | 2 ++
 pom.xml | 2 +-
 4 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml
index 09d4f5712f9..29ece216722 100644
--- a/itests/hive-unit/pom.xml
+++ b/itests/hive-unit/pom.xml
@@ -208,11 +208,6 @@
   junit
   test
 
-
-  org.junit.jupiter
-  junit-jupiter-engine
-  test
-
 
   org.apache.hadoop
   hadoop-hdfs
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java
index c6e1f1a4b0d..fadd793bc23 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java
@@ -36,8 +36,6 @@ import org.apache.thrift.TException;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
 
 import java.util.HashMap;
 import java.util.List;
@@ -59,7 +57,7 @@ public class TestStorageSchemaReader {
   Map jdbcTableParams = new HashMap<>();
   Map jdbcSerdeParams = new HashMap<>();
 
-  @Before @BeforeEach public void setUp() throws Exception {
+  @Before public void setUp() throws Exception {
 dbName = "sampleDb";
 hiveConf = new HiveConf(this.getClass());
 new DatabaseBuilder().setName(dbName).create(new 
HiveMetaStoreClient(hiveConf), hiveConf);
@@ -83,7 +81,7 @@ public class TestStorageSchemaReader {
 jdbcTableParams.put("storage_handler", 
"org.apache.hive.storage.jdbc.JdbcStorageHandler");
   }
 
-  @After @AfterEach public void tearDown() throws Exception {
+  @After public void tearDown() throws Exception {
 new HiveMetaStoreClient(hiveConf).dropDatabase(dbName, true, true, true);
   }
 
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
index e72811e103b..295bbdd09f7 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
@@ -77,6 +77,7 @@ import org.apache.orc.StripeInformation;
 import org.apache.orc.TypeDescription;
 import org.apache.orc.impl.RecordReaderImpl;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
@@ -601,6 +602,7 @@ public class TestCrudCompactorOnTez extends 
CompactorOnTezTest {
   }
 
   @Test
+  @Ignore("HIVE-27848")
   public void secondCompactionShouldBeRefusedBeforeEnqueueing() throws 
Exception {
 conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true);
 
diff --git a/pom.xml b/pom.xml
index bf8a5d3e5fd..c163ce1d3a2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -151,7 +151,7 @@
 
 4.5.13
 4.4.13
-2.5.2
+2.5.1
 2.13.5
 2.3.4
 2.4.1

(hive) branch master updated: HIVE-27789: Iceberg: Add a way to expire snapshots with retain last. (#4835). (Ayush Saxena, reviewed by Denys Kuzmenko, zhangbutao)

(hive) branch master updated: HIVE-27846: HIVE-27849: Tests under hive-unit module are not running (Stamatis Zampetakis reviewed by Ayush Saxena)

2 matches

Site Navigation

Mail list logo

Footer information