This is an automated email from the ASF dual-hosted git repository. xyao pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new c183bd8 HDDS-1527. HDDS Datanode start fails due to datanode.id file read error. Contributed by Siddharth Wagle. c183bd8 is described below commit c183bd8e2009c41ca9bdde964ec7e428dacc0c03 Author: Siddharth <swa...@hortonworks.com> AuthorDate: Thu May 16 15:13:10 2019 -0700 HDDS-1527. HDDS Datanode start fails due to datanode.id file read error. Contributed by Siddharth Wagle. This closes #822. --- .../container/common/helpers/ContainerUtils.java | 19 ++++++-- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 54 +++++++++++++--------- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java index 770435e..ff6dec8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Res import static org.apache.hadoop.ozone.container.common.impl.ContainerData.CHARSET_ENCODING; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Paths; import java.security.MessageDigest; @@ -35,6 +36,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.impl.ContainerData; @@ -51,6 +53,9 @@ import com.google.common.base.Preconditions; */ public final class ContainerUtils { + private static final Logger LOG = + LoggerFactory.getLogger(ContainerUtils.class); + private ContainerUtils() { //never constructed. } @@ -198,7 +203,7 @@ public final class ContainerUtils { throw new IOException("Unable to overwrite the datanode ID file."); } } else { - if(!path.getParentFile().exists() && + if (!path.getParentFile().exists() && !path.getParentFile().mkdirs()) { throw new IOException("Unable to create datanode ID directories."); } @@ -221,8 +226,16 @@ public final class ContainerUtils { try { return DatanodeIdYaml.readDatanodeIdFile(path); } catch (IOException e) { - throw new IOException("Failed to parse DatanodeDetails from " - + path.getAbsolutePath(), e); + LOG.warn("Error loading DatanodeDetails yaml from " + + path.getAbsolutePath(), e); + // Try to load as protobuf before giving up + try (FileInputStream in = new FileInputStream(path)) { + return DatanodeDetails.getFromProtoBuf( + HddsProtos.DatanodeDetailsProto.parseFrom(in)); + } catch (IOException io) { + throw new IOException("Failed to parse DatanodeDetails from " + + path.getAbsolutePath(), io); + } } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java index e8db976..f3a5d2c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java @@ -18,24 +18,38 @@ package org.apache.hadoop.ozone; +import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.XceiverClientGrpc; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; -import org.apache.hadoop.ozone.container.common.statemachine - .DatanodeStateMachine; -import org.apache.hadoop.ozone.container.common.statemachine - .EndpointStateMachine; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; +import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.XceiverClientGrpc; -import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.test.TestGenericTestUtils; import org.junit.AfterClass; @@ -44,19 +58,6 @@ import org.junit.BeforeClass; import org.junit.Test; import org.yaml.snakeyaml.Yaml; -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; - -import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT; -import static org.junit.Assert.*; - /** * Test cases for mini ozone cluster. */ @@ -167,6 +168,17 @@ public class TestMiniOzoneCluster { } catch (Exception e) { assertTrue(e instanceof IOException); } + + // Test upgrade scenario - protobuf file instead of yaml + File protoFile = new File(WRITE_TMP, "valid-proto.id"); + try (FileOutputStream out = new FileOutputStream(protoFile)) { + HddsProtos.DatanodeDetailsProto proto = id1.getProtoBufMessage(); + proto.writeTo(out); + } + validId = ContainerUtils.readDatanodeDetailsFrom(protoFile); + assertEquals(validId.getCertSerialId(), certSerialId); + assertEquals(id1, validId); + assertEquals(id1.getProtoBufMessage(), validId.getProtoBufMessage()); } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org