This is an automated email from the ASF dual-hosted git repository.

simhadrig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 87613cf2d61 HIVE-28373: Iceberg: Refactor the code of 
HadoopTableOptions (#5349)(Yongzhi Shao, reviewed by Denys Kuzmenko, Ayush 
Saxena , Simhadri Govindappa) )
87613cf2d61 is described below

commit 87613cf2d619dc2cb71235eddac88f74568f91f0
Author: PLASH SPEED <[email protected]>
AuthorDate: Wed Sep 11 18:37:15 2024 +0800

    HIVE-28373: Iceberg: Refactor the code of HadoopTableOptions 
(#5349)(Yongzhi Shao, reviewed by Denys Kuzmenko, Ayush Saxena , Simhadri 
Govindappa) )
---
 .../iceberg/hadoop/HiveHadoopTableTestBase.java    |  68 +++
 .../iceberg/hadoop/TestHiveHadoopCommits.java      | 640 +++++++++++++++++++++
 iceberg/patched-iceberg-core/pom.xml               |   1 +
 .../iceberg/hadoop/HadoopTableOperations.java      | 609 ++++++++++++++++++++
 4 files changed, 1318 insertions(+)

diff --git 
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hadoop/HiveHadoopTableTestBase.java
 
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hadoop/HiveHadoopTableTestBase.java
new file mode 100644
index 00000000000..ebfed820f4b
--- /dev/null
+++ 
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hadoop/HiveHadoopTableTestBase.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.hadoop;
+
+import java.io.File;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+
+public class HiveHadoopTableTestBase {
+  static final Schema SCHEMA =
+        new Schema(
+              required(3, "id", Types.IntegerType.get(), "unique ID"),
+              required(4, "data", Types.StringType.get()));
+
+  static final PartitionSpec SPEC = 
PartitionSpec.builderFor(SCHEMA).bucket("data", 16).build();
+
+  static final HadoopTables TABLES = new HadoopTables(new Configuration());
+
+  static final DataFile FILE_A =
+        DataFiles.builder(SPEC)
+              .withPath("/path/to/data-a.parquet")
+              .withFileSizeInBytes(0)
+              .withPartitionPath("data_bucket=0") // easy way to set partition 
data for now
+              .withRecordCount(2) // needs at least one record or else metrics 
will filter it out
+              .build();
+
+  @TempDir
+  File tableDir;
+
+  String tableLocation = null;
+  File metadataDir = null;
+  File versionHintFile = null;
+  Table table = null;
+
+  @BeforeEach
+  public void setupTable() {
+    this.tableLocation = tableDir.toURI().toString();
+    this.metadataDir = new File(tableDir, "metadata");
+    this.versionHintFile = new File(metadataDir, "version-hint.text");
+    this.table = TABLES.create(SCHEMA, SPEC, tableLocation);
+  }
+}
diff --git 
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hadoop/TestHiveHadoopCommits.java
 
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hadoop/TestHiveHadoopCommits.java
new file mode 100644
index 00000000000..c304b7c205e
--- /dev/null
+++ 
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hadoop/TestHiveHadoopCommits.java
@@ -0,0 +1,640 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.hadoop;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hadoop.fs.Path;
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+
+class TestHiveHadoopCommits extends HiveHadoopTableTestBase {
+
+  @Test
+  void testCommitFailedBeforeChangeVersionHint() {
+    table.newFastAppend().appendFile(FILE_A).commit();
+    BaseTable baseTable = (BaseTable) table;
+    HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+
+    HadoopTableOperations spyOps2 = (HadoopTableOperations) 
spy(tableOperations);
+    doReturn(10000).when(spyOps2).findVersionWithOutVersionHint(any());
+    TableMetadata metadataV1 = spyOps2.current();
+    SortOrder dataSort = 
SortOrder.builderFor(baseTable.schema()).asc("data").build();
+    TableMetadata metadataV2 = metadataV1.replaceSortOrder(dataSort);
+    assertThatThrownBy(() -> spyOps2.commit(metadataV1, metadataV2))
+          .isInstanceOf(CommitFailedException.class)
+          .hasMessageContaining("Are there other clients running in parallel 
with the current task?");
+
+    HadoopTableOperations spyOps3 = (HadoopTableOperations) 
spy(tableOperations);
+    doReturn(false).when(spyOps3).nextVersionIsLatest(anyInt(), anyInt());
+    assertCommitNotChangeVersion(
+          baseTable,
+          spyOps3,
+          CommitFailedException.class,
+          "Are there other clients running in parallel with the current 
task?");
+
+    HadoopTableOperations spyOps4 = (HadoopTableOperations) 
spy(tableOperations);
+    doThrow(new RuntimeException("FileSystem crash!"))
+          .when(spyOps4)
+          .renameMetaDataFileAndCheck(any(), any(), any(), anyBoolean());
+    assertCommitNotChangeVersion(
+          baseTable, spyOps4, CommitFailedException.class, "FileSystem 
crash!");
+  }
+
+  @Test
+  void testCommitFailedAndCheckFailed() throws IOException {
+    table.newFastAppend().appendFile(FILE_A).commit();
+    BaseTable baseTable = (BaseTable) table;
+    HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+    HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+    doThrow(new IOException("FileSystem crash!"))
+          .when(spyOps)
+          .renameMetaDataFile(any(), any(), any());
+    doThrow(new IOException("Can not check new Metadata!"))
+          .when(spyOps)
+          .checkMetaDataFileRenameSuccess(any(), any(), any(), anyBoolean());
+    assertCommitNotChangeVersion(
+          baseTable, spyOps, CommitStateUnknownException.class, "FileSystem 
crash!");
+
+    HadoopTableOperations spyOps2 = (HadoopTableOperations) 
spy(tableOperations);
+    doThrow(new OutOfMemoryError("Java heap space"))
+          .when(spyOps2)
+          .renameMetaDataFile(any(), any(), any());
+    assertCommitFail(baseTable, spyOps2, OutOfMemoryError.class, "Java heap 
space");
+
+    HadoopTableOperations spyOps3 = (HadoopTableOperations) 
spy(tableOperations);
+    doThrow(new RuntimeException("UNKNOWN ERROR"))
+          .when(spyOps3)
+          .renameMetaDataFile(any(), any(), any());
+    assertCommitNotChangeVersion(
+          baseTable, spyOps3, CommitStateUnknownException.class, "UNKNOWN 
ERROR");
+  }
+
+  @Test
+  void testCommitFailedAndRenameNotSuccess() throws IOException {
+    table.newFastAppend().appendFile(FILE_A).commit();
+    BaseTable baseTable = (BaseTable) table;
+    HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+    HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+    doThrow(new IOException("FileSystem crash!"))
+          .when(spyOps)
+          .renameMetaDataFile(any(), any(), any());
+    assertCommitNotChangeVersion(
+          baseTable,
+          spyOps,
+          CommitFailedException.class,
+          "Are there other clients running in parallel with the current 
task?");
+  }
+
+  @Test
+  void testCommitFailedButActualSuccess() throws IOException {
+    table.newFastAppend().appendFile(FILE_A).commit();
+    BaseTable baseTable = (BaseTable) table;
+    HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+    HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+    doThrow(new IOException("FileSystem crash!"))
+          .when(spyOps)
+          .renameMetaDataFile(any(), any(), any());
+    doReturn(true).when(spyOps).checkMetaDataFileRenameSuccess(any(), any(), 
any(), anyBoolean());
+    int versionBefore = spyOps.findVersion();
+    TableMetadata metadataV1 = spyOps.current();
+    SortOrder dataSort = 
SortOrder.builderFor(baseTable.schema()).asc("data").build();
+    TableMetadata metadataV2 = metadataV1.replaceSortOrder(dataSort);
+    spyOps.commit(metadataV1, metadataV2);
+    int versionAfter = spyOps.findVersion();
+    assertThat(versionAfter).isEqualTo(versionBefore + 1);
+  }
+
+  private void assertCommitNotChangeVersion(
+        BaseTable baseTable,
+        HadoopTableOperations spyOps,
+        Class<? extends Throwable> exceptionClass,
+        String msg) {
+    int versionBefore = spyOps.findVersion();
+    assertCommitFail(baseTable, spyOps, exceptionClass, msg);
+    int versionAfter = spyOps.findVersion();
+    assertThat(versionBefore).isEqualTo(versionAfter);
+  }
+
+  private void assertCommitFail(
+        BaseTable baseTable,
+        HadoopTableOperations spyOps,
+        Class<? extends Throwable> exceptionClass,
+        String msg) {
+    TableMetadata metadataV1 = spyOps.current();
+    SortOrder dataSort = 
SortOrder.builderFor(baseTable.schema()).asc("data").build();
+    TableMetadata metadataV2 = metadataV1.replaceSortOrder(dataSort);
+    assertThatThrownBy(() -> spyOps.commit(metadataV1, metadataV2))
+          .isInstanceOf(exceptionClass)
+          .hasMessageContaining(msg);
+  }
+
+  @Test
+  void testCommitFailedAfterChangeVersionHintRepeatCommit() {
+    // Submit a new file to test the functionality.
+    table.newFastAppend().appendFile(FILE_A).commit();
+    BaseTable baseTable = (BaseTable) table;
+    HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+    HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+    doThrow(new RuntimeException("FileSystem crash!"))
+          .when(spyOps)
+          .deleteRemovedMetadataFiles(any(), any());
+    int versionBefore = spyOps.findVersion();
+    TableMetadata metadataV1 = spyOps.current();
+
+    // first commit
+    SortOrder dataSort = 
SortOrder.builderFor(baseTable.schema()).asc("data").build();
+    TableMetadata metadataV2 = metadataV1.replaceSortOrder(dataSort);
+
+    spyOps.commit(metadataV1, metadataV2);
+
+    // The commit should actually success
+    int versionAfterSecond = spyOps.findVersion();
+    assertThat(versionAfterSecond).isEqualTo(versionBefore + 1);
+
+    // second commit
+    SortOrder idSort = 
SortOrder.builderFor(baseTable.schema()).desc("id").build();
+    TableMetadata currentMeta = spyOps.current();
+    TableMetadata metadataV3 = metadataV2.replaceSortOrder(idSort);
+    spyOps.commit(currentMeta, metadataV3);
+
+    // The commit should actually success
+    int versionAfterThird = spyOps.findVersion();
+    assertThat(versionAfterThird).isEqualTo(versionBefore + 2);
+  }
+
+  @Test
+  void testTwoClientCommitSameVersion() throws InterruptedException {
+    // In the linux environment, the JDK FileSystem interface implementation 
class is
+    // java.io.UnixFileSystem.
+    // Its behavior follows the posix protocol, which causes rename operations 
to overwrite the
+    // target file (because linux is compatible with some of the unix 
interfaces).
+    // However, linux also supports renaming without overwriting the target 
file. In addition, some
+    // other file systems such as Windows, HDFS, etc. also support renaming 
without overwriting the
+    // target file.
+    // We use the `mv -n` command to simulate the behavior of such filesystems.
+    table.newFastAppend().appendFile(FILE_A).commit();
+    ExecutorService executorService = Executors.newFixedThreadPool(8);
+    AtomicReference<Throwable> unexpectedException = new 
AtomicReference<>(null);
+    AtomicReference<Throwable> expectedException = new AtomicReference<>(null);
+    CountDownLatch countDownLatch = new CountDownLatch(2);
+    BaseTable baseTable = (BaseTable) table;
+    assertThat(((HadoopTableOperations) 
baseTable.operations()).findVersion()).isEqualTo(2);
+    executorService.execute(() -> {
+      try {
+        HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+        HadoopTableOperations spyOps = spy(tableOperations);
+        doNothing().when(spyOps).tryLock(any(), any());
+        doAnswer(x -> {
+          countDownLatch.countDown();
+          countDownLatch.await();
+          return x.callRealMethod();
+        }).when(spyOps).renameMetaDataFileAndCheck(any(), any(), any(), 
anyBoolean());
+        doAnswer(x -> {
+          Path srcPath = x.getArgument(1);
+          Path dstPath = x.getArgument(2);
+          File src = new File(srcPath.toUri());
+          File dst = new File(dstPath.toUri());
+          String srcPathStr = src.getAbsolutePath();
+          String dstPathStr = dst.getAbsolutePath();
+          String cmd = String.format("mv -n %s  %s", srcPathStr, dstPathStr);
+          Process process = Runtime.getRuntime().exec(cmd);
+          assertThat(process.waitFor()).isZero();
+          return dst.exists() && !src.exists();
+        }).when(spyOps).renameMetaDataFile(any(), any(), any());
+        TableMetadata metadataV1 = spyOps.current();
+        SortOrder dataSort = 
SortOrder.builderFor(baseTable.schema()).asc("data").build();
+        TableMetadata metadataV2 = metadataV1.replaceSortOrder(dataSort);
+        spyOps.commit(metadataV1, metadataV2);
+      } catch (CommitFailedException e) {
+        expectedException.set(e);
+      } catch (Throwable e) {
+        unexpectedException.set(e);
+      }
+    });
+
+    executorService.execute(() -> {
+      try {
+        HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+        HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+        doNothing().when(spyOps).tryLock(any(), any());
+        doAnswer(x -> {
+          countDownLatch.countDown();
+          countDownLatch.await();
+          return x.callRealMethod();
+        }).when(spyOps).renameMetaDataFileAndCheck(any(), any(), any(), 
anyBoolean());
+        doAnswer(x -> {
+          Path srcPath = x.getArgument(1);
+          Path dstPath = x.getArgument(2);
+          File src = new File(srcPath.toUri());
+          File dst = new File(dstPath.toUri());
+          String srcPathStr = src.getAbsolutePath();
+          String dstPathStr = dst.getAbsolutePath();
+          String cmd = String.format("mv -n %s  %s", srcPathStr, dstPathStr);
+          Process process = Runtime.getRuntime().exec(cmd);
+          assertThat(process.waitFor()).isZero();
+          return dst.exists() && !src.exists();
+        }).when(spyOps).renameMetaDataFile(any(), any(), any());
+        TableMetadata metadataV1 = spyOps.current();
+        SortOrder dataSort = 
SortOrder.builderFor(baseTable.schema()).asc("data").build();
+        TableMetadata metadataV2 = metadataV1.replaceSortOrder(dataSort);
+        spyOps.commit(metadataV1, metadataV2);
+      } catch (CommitFailedException e) {
+        expectedException.set(e);
+      } catch (Throwable e) {
+        unexpectedException.set(e);
+      }
+    });
+    executorService.shutdown();
+    if (!executorService.awaitTermination(610, TimeUnit.SECONDS)) {
+      executorService.shutdownNow();
+    }
+    assertThat(expectedException.get()).isNotNull();
+    assertThat(unexpectedException.get()).isNull();
+    assertThatThrownBy(() -> {
+      throw expectedException.get();
+    }).isInstanceOf(CommitFailedException.class)
+          .hasMessageContaining(
+                "Can not commit newMetaData because version [3] has already 
been committed.");
+    assertThat(((HadoopTableOperations) 
baseTable.operations()).findVersion()).isEqualTo(3);
+  }
+
+  @Test
+  void testConcurrentCommitAndRejectCommitAlreadyExistsVersion()
+        throws InterruptedException {
+    table.newFastAppend().appendFile(FILE_A).commit();
+    ExecutorService executorService = Executors.newFixedThreadPool(8);
+    AtomicReference<Throwable> unexpectedException = new 
AtomicReference<>(null);
+    AtomicInteger commitTimes = new AtomicInteger(0);
+    int maxCommitTimes = 20;
+    CountDownLatch countDownLatch = new CountDownLatch(5);
+    CountDownLatch countDownLatch2 = new CountDownLatch(1);
+    executorService.execute(() -> {
+      try {
+        BaseTable baseTable = (BaseTable) table;
+        HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+        HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+        doNothing().when(spyOps).tryLock(any(), any());
+        doAnswer(x -> {
+          countDownLatch2.countDown();
+          countDownLatch.await();
+          return x.callRealMethod();
+        }).when(spyOps).commitNewVersion(any(), any(), any(), any(), 
anyBoolean());
+        assertCommitFail(
+              baseTable, spyOps, CommitFailedException.class, "Version 3 
already exists");
+      } catch (Throwable e) {
+        unexpectedException.set(e);
+        throw new RuntimeException(e);
+      }
+    });
+
+    executorService.execute(() -> {
+      try {
+        countDownLatch2.await();
+        for (; commitTimes.get() < maxCommitTimes; 
commitTimes.incrementAndGet()) {
+          table.newFastAppend().appendFile(FILE_A).commit();
+          countDownLatch.countDown();
+        }
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    });
+    executorService.shutdown();
+    if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) {
+      executorService.shutdownNow();
+    }
+    assertThat(unexpectedException.get()).isNull();
+    assertThat(commitTimes.get()).isEqualTo(maxCommitTimes);
+  }
+
+  @Test
+  void testRejectCommitAlreadyExistsVersionWithUsingObjectStore()
+        throws InterruptedException {
+    // Since java.io.UnixFileSystem.rename overwrites existing files and we 
currently share the same
+    // memory locks. So we can use the local file system to simulate the use 
of object storage.
+    table.updateProperties().set(TableProperties.OBJECT_STORE_ENABLED, 
"true").commit();
+    table.newFastAppend().appendFile(FILE_A).commit();
+    ExecutorService executorService = Executors.newFixedThreadPool(8);
+    AtomicReference<Throwable> unexpectedException = new 
AtomicReference<>(null);
+    AtomicInteger commitTimes = new AtomicInteger(0);
+    int maxCommitTimes = 20;
+    CountDownLatch countDownLatch = new CountDownLatch(5);
+    CountDownLatch countDownLatch2 = new CountDownLatch(1);
+    executorService.execute(() -> {
+      try {
+        BaseTable baseTable = (BaseTable) table;
+        HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+        HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+        doAnswer(x -> {
+          countDownLatch2.countDown();
+          countDownLatch.await();
+          return x.callRealMethod();
+        }).when(spyOps).tryLock(any(), any());
+        assertCommitFail(
+              baseTable, spyOps, CommitFailedException.class, "Version 4 
already exists");
+      } catch (Throwable e) {
+        unexpectedException.set(e);
+        throw new RuntimeException(e);
+      }
+    });
+
+    executorService.execute(() -> {
+      try {
+        countDownLatch2.await();
+        for (; commitTimes.get() < maxCommitTimes; 
commitTimes.incrementAndGet()) {
+          table.newFastAppend().appendFile(FILE_A).commit();
+          countDownLatch.countDown();
+        }
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    });
+    executorService.shutdown();
+    if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
+      executorService.shutdownNow();
+    }
+    assertThat(unexpectedException.get()).isNull();
+    assertThat(commitTimes.get()).isEqualTo(maxCommitTimes);
+  }
+
+  @Test
+  void testConcurrentCommitAndRejectTooOldCommit() throws InterruptedException 
{
+    // Too-old-commit: commitVersion < currentMaxVersion - 
METADATA_PREVIOUS_VERSIONS_MAX
+    table.newFastAppend().appendFile(FILE_A).commit();
+    
table.updateProperties().set(TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, 
"2").commit();
+    
table.updateProperties().set(TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED,
 "true")
+          .commit();
+    ExecutorService executorService = Executors.newFixedThreadPool(8);
+    AtomicReference<Throwable> unexpectedException = new 
AtomicReference<>(null);
+    AtomicInteger commitTimes = new AtomicInteger(0);
+    int maxCommitTimes = 20;
+    CountDownLatch countDownLatch = new CountDownLatch(5);
+    CountDownLatch countDownLatch2 = new CountDownLatch(1);
+    CountDownLatch countDownLatch3 = new CountDownLatch(1);
+    executorService.execute(() -> {
+      try {
+        BaseTable baseTable = (BaseTable) table;
+        HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+        HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+        doNothing().when(spyOps).tryLock(any(), any());
+        doAnswer(x -> {
+          countDownLatch2.countDown();
+          countDownLatch.await();
+          return x.callRealMethod();
+        }).when(spyOps).commitNewVersion(any(), any(), any(), any(), 
anyBoolean());
+        assertCommitFail(
+              baseTable,
+              spyOps,
+              CommitFailedException.class,
+              "Cannot commit version [5] because it is smaller or " +
+                    "much larger than the current latest version [9].");
+        countDownLatch3.countDown();
+      } catch (Throwable e) {
+        unexpectedException.set(e);
+        throw new RuntimeException(e);
+      }
+    });
+
+    executorService.execute(() -> {
+      try {
+        countDownLatch2.await();
+        for (; commitTimes.get() < maxCommitTimes; 
commitTimes.incrementAndGet()) {
+          table.newFastAppend().appendFile(FILE_A).commit();
+          countDownLatch.countDown();
+          if (countDownLatch.getCount() == 0) {
+            countDownLatch3.await();
+          }
+        }
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    });
+    executorService.shutdown();
+    if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) {
+      executorService.shutdownNow();
+    }
+    assertThat(unexpectedException.get()).isNull();
+    assertThat(commitTimes.get()).isEqualTo(maxCommitTimes);
+  }
+
+  @Test
+  void testRejectTooOldCommitWithUsingObjectStore() throws 
InterruptedException {
+    // Since java.io.UnixFileSystem.rename overwrites existing files and we 
currently share the same
+    // memory locks. So we can use the local file system to simulate the use 
of object storage.
+    table.updateProperties().set(TableProperties.OBJECT_STORE_ENABLED, 
"true").commit();
+    table.newFastAppend().appendFile(FILE_A).commit();
+    
table.updateProperties().set(TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, 
"2").commit();
+    table.updateProperties()
+          .set(TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, "true")
+          .commit();
+    ExecutorService executorService = Executors.newFixedThreadPool(8);
+    AtomicReference<Throwable> unexpectedException = new 
AtomicReference<>(null);
+    AtomicInteger commitTimes = new AtomicInteger(0);
+    int maxCommitTimes = 20;
+    CountDownLatch countDownLatch = new CountDownLatch(5);
+    CountDownLatch countDownLatch2 = new CountDownLatch(1);
+    executorService.execute(() -> {
+      try {
+        BaseTable baseTable = (BaseTable) table;
+        HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+        HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+        doNothing().when(spyOps).tryLock(any(), any());
+        doAnswer(x -> {
+          countDownLatch2.countDown();
+          countDownLatch.await();
+          return x.callRealMethod();
+        }).when(spyOps).tryLock(any(), any());
+        assertCommitFail(
+              baseTable,
+              spyOps,
+              CommitFailedException.class,
+              "Cannot commit version [6] because it is smaller or much larger 
than the current latest version [10].");
+      } catch (Throwable e) {
+        unexpectedException.set(e);
+        throw new RuntimeException(e);
+      }
+    });
+
+    executorService.execute(() -> {
+      try {
+        countDownLatch2.await();
+        for (; commitTimes.get() < maxCommitTimes; 
commitTimes.incrementAndGet()) {
+          table.newFastAppend().appendFile(FILE_A).commit();
+          countDownLatch.countDown();
+        }
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    });
+    executorService.shutdown();
+    if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) {
+      executorService.shutdownNow();
+    }
+    assertThat(unexpectedException.get()).isNull();
+    assertThat(commitTimes.get()).isEqualTo(maxCommitTimes);
+  }
+
+  @Test
+  void testConcurrentCommitAndRejectDirtyCommit() throws InterruptedException {
+    // At the end of the commit, if we realize that it is a dirty commit, we 
should fail the commit.
+    table.newFastAppend().appendFile(FILE_A).commit();
+    
table.updateProperties().set(TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, 
"2").commit();
+    table.updateProperties()
+          .set(TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, "true")
+          .commit();
+    ExecutorService executorService = Executors.newFixedThreadPool(8);
+    AtomicReference<Throwable> unexpectedException = new 
AtomicReference<>(null);
+    AtomicInteger commitTimes = new AtomicInteger(0);
+    CountDownLatch countDownLatch = new CountDownLatch(5);
+    CountDownLatch countDownLatch2 = new CountDownLatch(1);
+    int maxCommitTimes = 20;
+    executorService.execute(() -> {
+      try {
+        BaseTable baseTable = (BaseTable) table;
+        HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+        HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+        TableMetadata metadataV1 = spyOps.current();
+        SortOrder dataSort = 
SortOrder.builderFor(baseTable.schema()).asc("data").build();
+        TableMetadata metadataV2 = metadataV1.replaceSortOrder(dataSort);
+        doNothing().when(spyOps).tryLock(any(), any());
+        doAnswer(invocation -> {
+          countDownLatch.await();
+          return invocation.callRealMethod();
+        }).when(spyOps).renameMetaDataFileAndCheck(any(), any(), any(), 
anyBoolean());
+        countDownLatch2.countDown();
+        assertThatThrownBy(() -> spyOps.commit(metadataV1, metadataV2))
+              .isInstanceOf(CommitStateUnknownException.class)
+              .hasMessageContaining("Commit rejected by server!" +
+                    "The current commit version [5] is much smaller than the 
latest version [9]");
+      } catch (Throwable e) {
+        unexpectedException.set(e);
+        throw new RuntimeException(e);
+      }
+    });
+
+    executorService.execute(() -> {
+      try {
+        countDownLatch2.await();
+        for (; commitTimes.get() < maxCommitTimes; 
commitTimes.incrementAndGet()) {
+          table.newFastAppend().appendFile(FILE_A).commit();
+          countDownLatch.countDown();
+        }
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    });
+
+    executorService.shutdown();
+    if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
+      executorService.shutdownNow();
+    }
+    assertThat(commitTimes.get()).isEqualTo(maxCommitTimes);
+    assertThat(unexpectedException.get()).isNull();
+  }
+
+  @Test
+  void testCleanTooOldDirtyCommit() throws InterruptedException {
+    // If there are dirty commits in the metadata for some reason, we need to 
clean them up.
+    table.newFastAppend().appendFile(FILE_A).commit();
+    
table.updateProperties().set(TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, 
"2").commit();
+    table.updateProperties()
+          .set(TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, "true")
+          .commit();
+    ExecutorService executorService = Executors.newFixedThreadPool(8);
+    AtomicReference<Exception> unexpectedException = new 
AtomicReference<>(null);
+    AtomicInteger commitTimes = new AtomicInteger(0);
+    int maxCommitTimes = 20;
+    BaseTable baseTable = (BaseTable) table;
+    HadoopTableOperations tableOperations = (HadoopTableOperations) 
baseTable.operations();
+    HadoopTableOperations spyOps = (HadoopTableOperations) 
spy(tableOperations);
+    CountDownLatch countDownLatch = new CountDownLatch(5);
+    CountDownLatch countDownLatch2 = new CountDownLatch(1);
+    AtomicReference<File> dirtyCommitFile = new AtomicReference<>(null);
+    executorService.execute(() -> {
+      try {
+        TableMetadata metadataV1 = spyOps.current();
+        int currentVersion = spyOps.findVersion();
+        int nextVersion = currentVersion + 1;
+        SortOrder dataSort = 
SortOrder.builderFor(baseTable.schema()).asc("data").build();
+        TableMetadata metadataV2 = metadataV1.replaceSortOrder(dataSort);
+        doNothing().when(spyOps).tryLock(any(), any());
+        doAnswer(invocation -> {
+          countDownLatch2.countDown();
+          countDownLatch.await();
+          return invocation.callRealMethod();
+        }).when(spyOps).renameMetaDataFileAndCheck(any(), any(), any(), 
anyBoolean());
+        doNothing().when(spyOps).fastFailIfDirtyCommit(anyInt(), anyInt(), 
any(), any());
+        spyOps.commit(metadataV1, metadataV2);
+        Path metadataFile = spyOps.getMetadataFile(nextVersion);
+        dirtyCommitFile.set(new File(metadataFile.toUri()));
+      } catch (Exception e) {
+        unexpectedException.set(e);
+        throw new RuntimeException(e);
+      }
+    });
+
+    executorService.execute(() -> {
+      try {
+        countDownLatch2.await();
+        for (; commitTimes.get() < maxCommitTimes; 
commitTimes.incrementAndGet()) {
+          table.newFastAppend().appendFile(FILE_A).commit();
+          countDownLatch.countDown();
+        }
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    });
+
+    executorService.shutdown();
+    if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
+      executorService.shutdownNow();
+    }
+
+    assertThat(unexpectedException.get()).isNull();
+    assertThat(dirtyCommitFile.get()).isNotNull();
+    assertThat(dirtyCommitFile.get()).exists();
+    long ttl = 30L * 3600 * 24 * 1000;
+    
assertThat(dirtyCommitFile.get().setLastModified(System.currentTimeMillis() - 
ttl)).isTrue();
+    table.newFastAppend().appendFile(FILE_A).commit();
+    assertThat(dirtyCommitFile.get()).doesNotExist();
+  }
+}
diff --git a/iceberg/patched-iceberg-core/pom.xml 
b/iceberg/patched-iceberg-core/pom.xml
index 171eadabbeb..e1f7ecd1b99 100644
--- a/iceberg/patched-iceberg-core/pom.xml
+++ b/iceberg/patched-iceberg-core/pom.xml
@@ -96,6 +96,7 @@
                   <excludes>
                       **/HadoopInputFile.class
                       **/SerializableTable.class
+                      **/HadoopTableOperations.class
                   </excludes>
                 </artifactItem>
               </artifactItems>
diff --git 
a/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/hadoop/HadoopTableOperations.java
 
b/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/hadoop/HadoopTableOperations.java
new file mode 100644
index 00000000000..8d71f9c648f
--- /dev/null
+++ 
b/iceberg/patched-iceberg-core/src/main/java/org/apache/iceberg/hadoop/HadoopTableOperations.java
@@ -0,0 +1,609 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.hadoop;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.iceberg.LocationProviders;
+import org.apache.iceberg.LockManager;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableMetadataParser;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.LocationProvider;
+import 
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.Tasks;
+import org.apache.iceberg.util.ThreadPools;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TableOperations implementation for file systems.
+ *
+ * <p>For normal file-system(support atomic non-overwriting rename). Users can 
use it directly
+ * without configuring additional lockManager.
+ *
+ * <p>For object storage(not support atomic non-overwriting rename), user 
should choose a suitable
+ * lockManager implementation.
+ *
+ * <p>This maintains metadata in a "metadata" folder under the table location.
+ */
+public class HadoopTableOperations implements TableOperations {
+
+  private static final Logger LOG = 
LoggerFactory.getLogger(HadoopTableOperations.class);
+  private static final Pattern VERSION_PATTERN = 
Pattern.compile("v([^\\.]*)\\..*");
+  private static final TableMetadataParser.Codec[] 
TABLE_METADATA_PARSER_CODEC_VALUES =
+        TableMetadataParser.Codec.values();
+  private final Configuration conf;
+  private final Path location;
+  private final FileIO fileIO;
+  private final LockManager lockManager;
+  private TableMetadata currentMetadata = null;
+  private volatile Integer version = null;
+  private volatile boolean shouldRefresh = true;
+
+  protected HadoopTableOperations(
+        Path location, FileIO fileIO, Configuration conf, LockManager 
lockManager) {
+    this.conf = conf;
+    this.location = location;
+    this.fileIO = fileIO;
+    this.lockManager = lockManager;
+  }
+
+  @Override
+  public TableMetadata current() {
+    if (shouldRefresh) {
+      return refresh();
+    }
+    return currentMetadata;
+  }
+
+  private synchronized Pair<Integer, TableMetadata> versionAndMetadata() {
+    return Pair.of(version, currentMetadata);
+  }
+
+  private synchronized void updateVersionAndMetadata(int newVersion, String 
metadataFile) {
+    // update if the current version is out of date
+    if (version == null || version != newVersion) {
+      this.version = newVersion;
+      this.currentMetadata =
+            checkUUID(currentMetadata, TableMetadataParser.read(io(), 
metadataFile));
+    }
+  }
+
+  @Override
+  public TableMetadata refresh() {
+    int ver = version != null ? version : findVersion();
+    try {
+      Path metadataFile = getMetadataFile(ver);
+      if (version == null && metadataFile == null && ver == 0) {
+        // no v0 metadata means the table doesn't exist yet
+        return null;
+      } else if (metadataFile == null) {
+        throw new ValidationException("Metadata file for version %d is 
missing", ver);
+      }
+
+      Path nextMetadataFile = getMetadataFile(ver + 1);
+      while (nextMetadataFile != null) {
+        ver += 1;
+        metadataFile = nextMetadataFile;
+        nextMetadataFile = getMetadataFile(ver + 1);
+      }
+
+      updateVersionAndMetadata(ver, metadataFile.toString());
+
+      this.shouldRefresh = false;
+      return currentMetadata;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e, "Failed to refresh the table");
+    }
+  }
+
+  @Override
+  public void commit(TableMetadata base, TableMetadata metadata) {
+    Pair<Integer, TableMetadata> current = versionAndMetadata();
+    if (base != current.second()) {
+      throw new CommitFailedException("Cannot commit changes based on stale 
table metadata");
+    }
+
+    if (base == metadata) {
+      LOG.info("Nothing to commit.");
+      return;
+    }
+
+    Preconditions.checkArgument(
+          base == null || base.location().equals(metadata.location()),
+          "Hadoop path-based tables cannot be relocated");
+    Preconditions.checkArgument(
+          
!metadata.properties().containsKey(TableProperties.WRITE_METADATA_LOCATION),
+          "Hadoop path-based tables cannot relocate metadata");
+
+    String codecName =
+          metadata.property(
+                TableProperties.METADATA_COMPRESSION, 
TableProperties.METADATA_COMPRESSION_DEFAULT);
+    TableMetadataParser.Codec codec = 
TableMetadataParser.Codec.fromName(codecName);
+    String fileExtension = TableMetadataParser.getFileExtension(codec);
+    Path tempMetadataFile = metadataPath(UUID.randomUUID() + fileExtension);
+    TableMetadataParser.write(metadata, 
io().newOutputFile(tempMetadataFile.toString()));
+
+    int nextVersion = (current.first() != null ? current.first() : 0) + 1;
+    Path finalMetadataFile = metadataFilePath(nextVersion, codec);
+    FileSystem fs = getFileSystem(tempMetadataFile, conf);
+    boolean versionCommitSuccess = false;
+    boolean useObjectStore =
+          metadata.propertyAsBoolean(
+                TableProperties.OBJECT_STORE_ENABLED, 
TableProperties.OBJECT_STORE_ENABLED_DEFAULT);
+    int previousVersionsMax =
+          metadata.propertyAsInt(
+                TableProperties.METADATA_PREVIOUS_VERSIONS_MAX,
+                TableProperties.METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT);
+    boolean supportGlobalLocking = useObjectStore;
+    try {
+      tryLock(tempMetadataFile, metadataRoot());
+      versionCommitSuccess =
+            commitNewVersion(
+                  fs, tempMetadataFile, finalMetadataFile, nextVersion, 
supportGlobalLocking);
+      if (!versionCommitSuccess) {
+        throw new CommitFailedException(
+              "Can not commit newMetaData because version [%s] has already 
been committed. " +
+                    "tempMetaData=[%s],finalMetaData=[%s].Are there other 
clients running " +
+                    "in parallel with the current task?",
+              nextVersion, tempMetadataFile, finalMetadataFile);
+      }
+      validate(supportGlobalLocking, previousVersionsMax, nextVersion, fs, 
finalMetadataFile);
+      this.shouldRefresh = true;
+      LOG.info("Committed a new metadata file {}", finalMetadataFile);
+      // update the best-effort version pointer
+      writeVersionHint(fs, nextVersion);
+      deleteRemovedMetadataFiles(base, metadata);
+    } catch (CommitStateUnknownException e) {
+      this.shouldRefresh = true;
+      throw e;
+    } catch (Exception e) {
+      this.shouldRefresh = versionCommitSuccess;
+      if (!versionCommitSuccess) {
+        tryDelete(tempMetadataFile);
+        throw new CommitFailedException(e);
+      }
+    } finally {
+      unlock(tempMetadataFile, metadataRoot());
+    }
+  }
+
+  private void tryDelete(Path path) {
+    try {
+      io().deleteFile(path.toString());
+    } catch (Exception ignored) {
+      // do nothing
+    }
+  }
+
+  @VisibleForTesting
+  void tryLock(Path src, Path dst) {
+    if (!lockManager.acquire(dst.toString(), src.toString())) {
+      throw new CommitFailedException(
+            "Failed to acquire lock on file: %s with owner: %s", dst, src);
+    }
+  }
+
+  void unlock(Path src, Path dst) {
+    try {
+      if (!lockManager.release(dst.toString(), src.toString())) {
+        LOG.warn("Failed to release lock on file: {} with owner: {}", dst, 
src);
+      }
+    } catch (Exception ignored) {
+      // do nothing.
+    }
+  }
+
+  private void validate(
+        boolean supportGlobalLocking,
+        int previousVersionsMax,
+        int nextVersion,
+        FileSystem fs,
+        Path finalMetadataFile)
+        throws IOException {
+    if (!supportGlobalLocking) {
+      fastFailIfDirtyCommit(previousVersionsMax, nextVersion, fs, 
finalMetadataFile);
+      cleanAllTooOldDirtyCommit(fs, previousVersionsMax);
+    }
+  }
+
+  @VisibleForTesting
+  void fastFailIfDirtyCommit(
+        int previousVersionsMax, int nextVersion, FileSystem fs, Path 
finalMetadataFile)
+        throws IOException {
+    int currentMaxVersion = findVersionWithOutVersionHint(fs);
+    if ((currentMaxVersion - nextVersion) > previousVersionsMax && 
fs.exists(finalMetadataFile)) {
+      tryDelete(finalMetadataFile);
+      throw new CommitStateUnknownException(
+            new RejectedExecutionException(
+                  String.format(
+                        "Commit rejected by server!The current commit version 
[%d] is much smaller " +
+                              "than the latest version [%d].Are there other 
clients running in " +
+                              "parallel with the current task?",
+                        nextVersion, currentMaxVersion)));
+    }
+  }
+
+  void cleanAllTooOldDirtyCommit(FileSystem fs, int previousVersionsMax) 
throws IOException {
+    FileStatus[] files =
+          fs.listStatus(metadataRoot(), name -> 
VERSION_PATTERN.matcher(name.getName()).matches());
+    List<Path> dirtyCommits = Lists.newArrayList();
+    int currentMaxVersion = findVersionWithOutVersionHint(fs);
+    long now = System.currentTimeMillis();
+    // We only clean up dirty commits that are some time old. This ensures 
that other clients can
+    // find out as soon as possible if their current commit is dirty.
+    long ttl = 3600L * 24 * 1000 * 7;
+    for (FileStatus file : files) {
+      long modificationTime = file.getModificationTime();
+      Path path = file.getPath();
+      if ((currentMaxVersion - version(path.getName()) > previousVersionsMax) 
&&
+            (now - modificationTime) > ttl) {
+        dirtyCommits.add(path);
+      }
+    }
+    for (Path dirtyCommit : dirtyCommits) {
+      io().deleteFile(dirtyCommit.toString());
+    }
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  @Override
+  public LocationProvider locationProvider() {
+    return LocationProviders.locationsFor(current().location(), 
current().properties());
+  }
+
+  @Override
+  public String metadataFileLocation(String fileName) {
+    return metadataPath(fileName).toString();
+  }
+
+  @Override
+  public TableOperations temp(TableMetadata uncommittedMetadata) {
+    return new TableOperations() {
+      @Override
+      public TableMetadata current() {
+        return uncommittedMetadata;
+      }
+
+      @Override
+      public TableMetadata refresh() {
+        throw new UnsupportedOperationException(
+              "Cannot call refresh on temporary table operations");
+      }
+
+      @Override
+      public void commit(TableMetadata base, TableMetadata metadata) {
+        throw new UnsupportedOperationException("Cannot call commit on 
temporary table operations");
+      }
+
+      @Override
+      public String metadataFileLocation(String fileName) {
+        return HadoopTableOperations.this.metadataFileLocation(fileName);
+      }
+
+      @Override
+      public LocationProvider locationProvider() {
+        return LocationProviders.locationsFor(
+              uncommittedMetadata.location(), 
uncommittedMetadata.properties());
+      }
+
+      @Override
+      public FileIO io() {
+        return HadoopTableOperations.this.io();
+      }
+
+      @Override
+      public EncryptionManager encryption() {
+        return HadoopTableOperations.this.encryption();
+      }
+
+      @Override
+      public long newSnapshotId() {
+        return HadoopTableOperations.this.newSnapshotId();
+      }
+    };
+  }
+
+  @VisibleForTesting
+  Path getMetadataFile(int metadataVersion) throws IOException {
+    for (TableMetadataParser.Codec codec : TABLE_METADATA_PARSER_CODEC_VALUES) 
{
+      Path metadataFile = metadataFilePath(metadataVersion, codec);
+      FileSystem fs = getFileSystem(metadataFile, conf);
+      if (fs.exists(metadataFile)) {
+        return metadataFile;
+      }
+
+      if (codec.equals(TableMetadataParser.Codec.GZIP)) {
+        // we have to be backward-compatible with .metadata.json.gz files
+        metadataFile = oldMetadataFilePath(metadataVersion, codec);
+        fs = getFileSystem(metadataFile, conf);
+        if (fs.exists(metadataFile)) {
+          return metadataFile;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  private Path metadataFilePath(int metadataVersion, TableMetadataParser.Codec 
codec) {
+    return metadataPath("v" + metadataVersion + 
TableMetadataParser.getFileExtension(codec));
+  }
+
+  private Path oldMetadataFilePath(int metadataVersion, 
TableMetadataParser.Codec codec) {
+    return metadataPath("v" + metadataVersion + 
TableMetadataParser.getOldFileExtension(codec));
+  }
+
+  private Path metadataPath(String filename) {
+    return new Path(metadataRoot(), filename);
+  }
+
+  private Path metadataRoot() {
+    return new Path(location, "metadata");
+  }
+
+  private int version(String fileName) {
+    Matcher matcher = VERSION_PATTERN.matcher(fileName);
+    if (!matcher.matches()) {
+      return -1;
+    }
+    String versionNumber = matcher.group(1);
+    try {
+      return Integer.parseInt(versionNumber);
+    } catch (NumberFormatException ne) {
+      return -1;
+    }
+  }
+
+  @VisibleForTesting
+  Path versionHintFile() {
+    return metadataPath(Util.VERSION_HINT_FILENAME);
+  }
+
+  @VisibleForTesting
+  void writeVersionHint(FileSystem fs, Integer versionToWrite) throws 
IOException {
+    Path versionHintFile = versionHintFile();
+    Path tempVersionHintFile = metadataPath(UUID.randomUUID() + 
"-version-hint.temp");
+    try {
+      writeVersionToPath(fs, tempVersionHintFile, versionToWrite);
+      fs.rename(tempVersionHintFile, versionHintFile);
+    } catch (IOException e) {
+      // Cleaning up temporary files.
+      if (fs.exists(tempVersionHintFile)) {
+        io().deleteFile(tempVersionHintFile.toString());
+      }
+      throw e;
+    }
+  }
+
+  @VisibleForTesting
+  boolean nextVersionIsLatest(int nextVersion, int currentMaxVersion) {
+    return nextVersion == (currentMaxVersion + 1);
+  }
+
+  private void writeVersionToPath(FileSystem fs, Path path, int 
versionToWrite) {
+    try (FSDataOutputStream out = fs.create(path, false /* overwrite */)) {
+      
out.write(String.valueOf(versionToWrite).getBytes(StandardCharsets.UTF_8));
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @VisibleForTesting
+  int findVersionByUsingVersionHint(FileSystem fs, Path versionHintFile) 
throws IOException {
+    try (InputStreamReader fsr =
+               new InputStreamReader(fs.open(versionHintFile), 
StandardCharsets.UTF_8);
+         BufferedReader in = new BufferedReader(fsr)) {
+      return Integer.parseInt(in.readLine().replace("\n", ""));
+    }
+  }
+
+  @VisibleForTesting
+  int findVersionWithOutVersionHint(FileSystem fs) {
+    try {
+      if (!fs.exists(metadataRoot())) {
+        // Either the table has just been created, or it has been corrupted, 
but either way, we have
+        // to start at version 0.
+        LOG.warn("Metadata for table not found in directory [{}]", 
metadataRoot());
+        return 0;
+      }
+      // List the metadata directory to find the version files, and try to 
recover the max
+      // available version
+      FileStatus[] files =
+            fs.listStatus(metadataRoot(), name -> 
VERSION_PATTERN.matcher(name.getName()).matches());
+      int maxVersion = 0;
+      for (FileStatus file : files) {
+        int currentVersion = version(file.getPath().getName());
+        if (currentVersion > maxVersion && getMetadataFile(currentVersion) != 
null) {
+          maxVersion = currentVersion;
+        }
+      }
+      return maxVersion;
+    } catch (Exception e) {
+      throw new RuntimeIOException(new IOException(e));
+    }
+  }
+
+  @VisibleForTesting
+  int findVersion() {
+    Path versionHintFile = versionHintFile();
+    FileSystem fs = getFileSystem(versionHintFile, conf);
+    try {
+      return fs.exists(versionHintFile) ?
+            findVersionByUsingVersionHint(fs, versionHintFile) :
+            findVersionWithOutVersionHint(fs);
+    } catch (Exception e) {
+      // try one last time
+      return findVersionWithOutVersionHint(fs);
+    }
+  }
+
+  /**
+   * Renames the source file to destination, using the provided file system.
+   *
+   * @param fs the filesystem used for the rename
+   * @param src the source file
+   * @param dst the destination file
+   * @return If it returns true, then the commit was successful.
+   */
+  @VisibleForTesting
+  boolean commitNewVersion(
+        FileSystem fs, Path src, Path dst, Integer nextVersion, boolean 
supportGlobalLocking)
+        throws IOException {
+    if (fs.exists(dst)) {
+      throw new CommitFailedException("Version %d already exists: %s", 
nextVersion, dst);
+    }
+    int maxVersion = supportGlobalLocking ? findVersion() : 
findVersionWithOutVersionHint(fs);
+    if (!nextVersionIsLatest(nextVersion, maxVersion)) {
+      if (!supportGlobalLocking) {
+        io().deleteFile(versionHintFile().toString());
+      }
+      throw new CommitFailedException(
+            "Cannot commit version [%d] because it is smaller or much larger 
than the current latest version [%d]." +
+                  "Are there other clients running in parallel with the 
current task?",
+            nextVersion, maxVersion);
+    }
+    io().deleteFile(versionHintFile().toString());
+    return renameMetaDataFileAndCheck(fs, src, dst, supportGlobalLocking);
+  }
+
+  protected FileSystem getFileSystem(Path path, Configuration hadoopConf) {
+    return Util.getFs(path, hadoopConf);
+  }
+
+  @VisibleForTesting
+  boolean checkMetaDataFileRenameSuccess(
+        FileSystem fs, Path tempMetaDataFile, Path finalMetaDataFile, boolean 
supportGlobalLocking)
+        throws IOException {
+    if (!supportGlobalLocking) {
+      return fs.exists(finalMetaDataFile) && !fs.exists(tempMetaDataFile);
+    } else {
+      return fs.exists(finalMetaDataFile);
+    }
+  }
+
+  @VisibleForTesting
+  boolean renameMetaDataFile(FileSystem fs, Path tempMetaDataFile, Path 
finalMetaDataFile)
+        throws IOException {
+    return fs.rename(tempMetaDataFile, finalMetaDataFile);
+  }
+
+  private boolean renameCheck(
+        FileSystem fs,
+        Path tempMetaDataFile,
+        Path finalMetaDataFile,
+        Throwable rootError,
+        boolean supportGlobalLocking) {
+    try {
+      return checkMetaDataFileRenameSuccess(
+            fs, tempMetaDataFile, finalMetaDataFile, supportGlobalLocking);
+    } catch (Exception e) {
+      throw new CommitStateUnknownException(rootError != null ? rootError : e);
+    }
+  }
+
+  @VisibleForTesting
+  boolean renameMetaDataFileAndCheck(
+        FileSystem fs, Path tempMetaDataFile, Path finalMetaDataFile, boolean 
supportGlobalLocking) {
+    try {
+      return renameMetaDataFile(fs, tempMetaDataFile, finalMetaDataFile);
+    } catch (IOException e) {
+      // Server-side error, we need to try to recheck it again
+      return renameCheck(fs, tempMetaDataFile, finalMetaDataFile, e, 
supportGlobalLocking);
+    } catch (Exception e) {
+      // Maybe Client-side error,Since the rename command may have already 
been committed and has
+      // not yet been executed.There is no point in performing a check 
operation at this point.throw
+      // CommitStateUnknownException and stop everything.
+      throw new CommitStateUnknownException(e);
+    }
+  }
+  /**
+   * Deletes the oldest metadata files if {@link
+   * TableProperties#METADATA_DELETE_AFTER_COMMIT_ENABLED} is true.
+   *
+   * @param base table metadata on which previous versions were based
+   * @param metadata new table metadata with updated previous versions
+   */
+  @VisibleForTesting
+  void deleteRemovedMetadataFiles(TableMetadata base, TableMetadata metadata) {
+    if (base == null) {
+      return;
+    }
+
+    boolean deleteAfterCommit =
+          metadata.propertyAsBoolean(
+                TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED,
+                TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT);
+
+    if (deleteAfterCommit) {
+      Set<TableMetadata.MetadataLogEntry> removedPreviousMetadataFiles =
+            Sets.newHashSet(base.previousFiles());
+      removedPreviousMetadataFiles.removeAll(metadata.previousFiles());
+      Tasks.foreach(removedPreviousMetadataFiles)
+            .executeWith(ThreadPools.getWorkerPool())
+            .noRetry()
+            .suppressFailureWhenFinished()
+            .onFailure((previousMetadataFile, exc) ->
+                  LOG.warn("Delete failed for previous metadata file: {}", 
previousMetadataFile, exc))
+            .run(previousMetadataFile -> 
io().deleteFile(previousMetadataFile.file()));
+    }
+  }
+
+  private static TableMetadata checkUUID(TableMetadata currentMetadata, 
TableMetadata newMetadata) {
+    String newUUID = newMetadata.uuid();
+    if (currentMetadata != null && currentMetadata.uuid() != null && newUUID 
!= null) {
+      Preconditions.checkState(
+            newUUID.equals(currentMetadata.uuid()),
+            "Table UUID does not match: current=%s != refreshed=%s",
+            currentMetadata.uuid(),
+            newUUID);
+    }
+    return newMetadata;
+  }
+}

Reply via email to