[50/50] [abbrv] hadoop git commit: HDFS-13421. [PROVIDED Phase 2] Implement DNA_BACKUP command in Datanode. Contributed by Ewan Higgs.
HDFS-13421. [PROVIDED Phase 2] Implement DNA_BACKUP command in Datanode. Contributed by Ewan Higgs. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06477abc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06477abc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06477abc Branch: refs/heads/HDFS-12090 Commit: 06477abcd93eb988b4afd0a2dff549e67e0dbd85 Parents: 8cdd033 Author: Virajith Jalaparti Authored: Wed Aug 1 12:13:31 2018 -0700 Committer: Ewan Higgs Committed: Mon Sep 3 14:42:33 2018 +0200 -- .../apache/hadoop/hdfs/BlockInputStream.java| 52 .../hdfs/server/datanode/BPOfferService.java| 6 + .../hadoop/hdfs/server/datanode/DataNode.java | 18 +++ .../SyncServiceSatisfierDatanodeWorker.java | 97 +++ .../SyncTaskExecutionFeedbackCollector.java | 54 .../executor/BlockSyncOperationExecutor.java| 122 +++ .../executor/BlockSyncReaderFactory.java| 92 ++ .../executor/BlockSyncTaskRunner.java | 69 +++ .../hadoop/hdfs/TestBlockInputStream.java | 84 + .../TestBlockSyncOperationExecutor.java | 94 ++ 10 files changed, 688 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/06477abc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockInputStream.java new file mode 100644 index 000..152f83e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockInputStream.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Facade around BlockReader that indeed implements the InputStream interface. + */ +public class BlockInputStream extends InputStream { + private final BlockReader blockReader; + + public BlockInputStream(BlockReader blockReader) { +this.blockReader = blockReader; + } + + @Override + public int read() throws IOException { +byte[] b = new byte[1]; +int c = blockReader.read(b, 0, b.length); +if (c > 0) { + return b[0]; +} else { + return -1; +} + } + + @Override + public int read(byte b[], int off, int len) throws IOException { +return blockReader.read(b, off, len); + } + + @Override + public long skip(long n) throws IOException { +return blockReader.skip(n); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/06477abc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index a25f6a9..b8eef5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -795,6 +795,12 @@ class BPOfferService { ((BlockECReconstructionCommand) cmd).getECTasks(); dn.getErasureCodingWorker().processErasureCodingTasks(ecTasks); break; +case DatanodeProtocol.DNA_BACKUP: + LOG.info("DatanodeCommand action: DNA_BACKUP"); + Collection backupTasks = + ((SyncCommand) cmd).getSyncTasks(); + dn.getSyncServiceSatisfierDatanodeWorker().processSyncTasks(backupTasks); + break; default: LOG.warn("Unknown DatanodeCommand action: " + cmd.getAction()); }
[50/50] [abbrv] hadoop git commit: HDFS-13421. [PROVIDED Phase 2] Implement DNA_BACKUP command in Datanode. Contributed by Ewan Higgs.
HDFS-13421. [PROVIDED Phase 2] Implement DNA_BACKUP command in Datanode. Contributed by Ewan Higgs. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/959f49b4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/959f49b4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/959f49b4 Branch: refs/heads/HDFS-12090 Commit: 959f49b4803bae374da086b1891655f90e5502e5 Parents: 2c52ff5 Author: Virajith Jalaparti Authored: Wed Aug 1 12:13:31 2018 -0700 Committer: Ewan Higgs Committed: Fri Aug 10 13:34:28 2018 +0200 -- .../apache/hadoop/hdfs/BlockInputStream.java| 52 .../hdfs/server/datanode/BPOfferService.java| 6 + .../hadoop/hdfs/server/datanode/DataNode.java | 20 ++- .../SyncServiceSatisfierDatanodeWorker.java | 97 +++ .../SyncTaskExecutionFeedbackCollector.java | 54 .../executor/BlockSyncOperationExecutor.java| 122 +++ .../executor/BlockSyncReaderFactory.java| 92 ++ .../executor/BlockSyncTaskRunner.java | 69 +++ .../hadoop/hdfs/TestBlockInputStream.java | 84 + .../TestBlockSyncOperationExecutor.java | 94 ++ 10 files changed, 689 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/959f49b4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockInputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockInputStream.java new file mode 100644 index 000..152f83e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockInputStream.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Facade around BlockReader that indeed implements the InputStream interface. + */ +public class BlockInputStream extends InputStream { + private final BlockReader blockReader; + + public BlockInputStream(BlockReader blockReader) { +this.blockReader = blockReader; + } + + @Override + public int read() throws IOException { +byte[] b = new byte[1]; +int c = blockReader.read(b, 0, b.length); +if (c > 0) { + return b[0]; +} else { + return -1; +} + } + + @Override + public int read(byte b[], int off, int len) throws IOException { +return blockReader.read(b, off, len); + } + + @Override + public long skip(long n) throws IOException { +return blockReader.skip(n); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/959f49b4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index a25f6a9..b8eef5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -795,6 +795,12 @@ class BPOfferService { ((BlockECReconstructionCommand) cmd).getECTasks(); dn.getErasureCodingWorker().processErasureCodingTasks(ecTasks); break; +case DatanodeProtocol.DNA_BACKUP: + LOG.info("DatanodeCommand action: DNA_BACKUP"); + Collection backupTasks = + ((SyncCommand) cmd).getSyncTasks(); + dn.getSyncServiceSatisfierDatanodeWorker().processSyncTasks(backupTasks); + break; default: LOG.warn("Unknown DatanodeCommand action: " + cmd.getAction()); }