This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 8acc0413a95 HDDS-14812. Support datanode topology in MiniOzoneCluster
(#10598)
add 147b009a6db HDDS-11622. Support domain socket creation (#7397)
add 46b858007e6 HDDS-11627. Support getBlock operation on short-circuit
channel. (#7456)
add fb9398a3131 HDDS-11626. Document short-circuit read feature (#7529)
add 13a905468a2 Merge remote-tracking branch 'origin/master' into
HDDS-10685
add cd3e1c7ca04 HDDS-12883. Close file descriptor of block file in
datanode (#8319)
add 1395fae0640 Merge remote-tracking branch 'origin/master' into
HDDS-10685
add 625a6ad85cd HDDS-11651. Support short-circuit read channel for freon
rk command (#8316)
add 5676ac4b338 HDDS-15318. Download libhadoop library during maven build
(#10305)
add 879707874bc Merge remote-tracking branch 'origin/master' into
HDDS-10685
add 2f90c1731a3 HDDS-15404. Enable short-circuit unit test in CI (#10377)
add 6c9676481b9 HDDS-15442. Enhance short-circuit robot test (#10400)
add c2118e70585 Merge branch 'master' of github.com:apache/ozone into
HDDS-10685
add 277737474cd HDDS-15559. Add libhadoop to DYLD_LIBRARY_PATH or
LD_LIBRARY_PATH (#10513)
add 1ea7e9e7985 Merge remote-tracking branch 'origin/master' into
HDDS-10685
add b01e0c715fa HDDS-15992. Convert short-circuit-read design PDF to
markdown (#10877)
add 6db4545f1da HDDS-16013. Remove HDDSLayoutFeature.SHORT_CIRCUIT_READS
layout version (#10893)
new ec381e5d3b0 Merge remote-tracking branch 'origin/HDDS-10685' into
master
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 2 +-
hadoop-hdds/client/pom.xml | 4 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 52 ++
.../hadoop/hdds/scm/XceiverClientCreator.java | 37 +-
.../hadoop/hdds/scm/XceiverClientFactory.java | 19 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 8 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 104 ++-
.../hadoop/hdds/scm/XceiverClientShortCircuit.java | 623 ++++++++++++++++
.../hadoop/hdds/scm/storage/BlockInputStream.java | 149 +++-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +-
.../apache/hadoop/hdds/scm/storage/DomainPeer.java | 111 +++
.../hdds/scm/storage/DomainSocketFactory.java | 276 +++++++
.../hdds/scm/storage/LocalChunkInputStream.java | 116 +++
.../org/apache/hadoop/hdds/DatanodeVersion.java | 2 +
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 14 +
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 1 +
.../ContainerCommandResponseBuilders.java | 20 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 30 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 14 +
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 24 +-
.../common/src/main/resources/ozone-default.xml | 11 +-
.../hadoop/hdds/utils/TestProtobufUtils.java | 36 +
.../ozone/container/ContainerTestHelper.java | 2 +-
.../container/common/helpers/ContainerMetrics.java | 49 +-
.../ozone/container/common/interfaces/Handler.java | 23 +-
.../common/transport/server/Receiver.java | 353 +++++++++
.../server/XceiverServerDomainSocket.java | 324 +++++++++
.../common/transport/server/XceiverServerGrpc.java | 53 +-
.../common/transport/server/XceiverServerSpi.java | 2 +
.../transport/server/ratis/XceiverServerRatis.java | 5 +
.../ozone/container/keyvalue/KeyValueHandler.java | 53 +-
.../container/keyvalue/helpers/BlockUtils.java | 9 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 7 +
.../keyvalue/impl/FilePerBlockStrategy.java | 22 +-
.../keyvalue/interfaces/ChunkManager.java | 17 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 77 +-
.../container/keyvalue/TestKeyValueHandler.java | 2 +-
.../design/short-circuit-read-current-flow.png | Bin 0 -> 33740 bytes
.../design/short-circuit-read-getblock-flow.png | Bin 0 -> 66181 bytes
.../design/short-circuit-read-hbase-benchmark.png | Bin 0 -> 34029 bytes
.../short-circuit-read-mmap-cache-benchmark.png | Bin 0 -> 28688 bytes
.../design/short-circuit-read-target-flow.png | Bin 0 -> 39362 bytes
.../short-circuit-read-unix-domain-socket.png | Bin 0 -> 82958 bytes
.../docs/content/design/short-circuit-read.md | 175 +++++
.../docs/content/feature/Short-Circuit-Read.md | 90 +++
.../docs/content/feature/Short-Circuit-Read.zh.md | 88 +++
.../src/main/proto/DatanodeClientProtocol.proto | 12 +
hadoop-hdds/rocks-native/pom.xml | 6 +-
.../ozone/client/MockXceiverClientFactory.java | 9 +-
.../client/checksum/FileChecksumBenchmark.java | 12 +
.../dist/dev-support/bin/dist-layout-stitching | 18 +
.../diskbalancer.yaml => ozone/short-circuit.yaml} | 11 +-
.../{test-failures1.sh => test-short-circuit.sh} | 17 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +
.../src/main/smoketest/freon/read-write-key.robot | 7 +
.../smoketest/short-circuit/short-circuit.robot | 55 ++
.../dist/src/shell/ozone/ozone-functions.sh | 42 ++
.../hadoop/ozone/freon/RandomKeyGenerator.java | 34 +
hadoop-ozone/integration-test/pom.xml | 11 +
.../hadoop/hdds/TestXceiverServerDomainSocket.java | 793 +++++++++++++++++++++
.../hadoop/hdds/scm/TestXceiverClientManager.java | 6 +-
.../hdds/scm/TestXceiverClientManagerSC.java | 109 +++
.../TestDNDataDistributionFinalization.java | 11 +-
.../TestScmDataDistributionFinalization.java | 8 +-
.../ozone/client/rpc/read/InputStreamTests.java | 22 +-
.../client/rpc/read/TestChunkInputStream.java | 16 +-
.../client/rpc/read/TestDomainSocketFactory.java | 81 +++
.../client/rpc/read/TestLocalChunkInputStream.java | 163 +++++
.../ozone/om/TestOmContainerLocationCache.java | 4 +
.../ozone/om/service/TestBlockDeletionService.java | 7 +-
pom.xml | 183 +++++
72 files changed, 4493 insertions(+), 158 deletions(-)
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientShortCircuit.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/DomainPeer.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/DomainSocketFactory.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/LocalChunkInputStream.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/Receiver.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerDomainSocket.java
create mode 100644
hadoop-hdds/docs/content/design/short-circuit-read-current-flow.png
create mode 100644
hadoop-hdds/docs/content/design/short-circuit-read-getblock-flow.png
create mode 100644
hadoop-hdds/docs/content/design/short-circuit-read-hbase-benchmark.png
create mode 100644
hadoop-hdds/docs/content/design/short-circuit-read-mmap-cache-benchmark.png
create mode 100644
hadoop-hdds/docs/content/design/short-circuit-read-target-flow.png
create mode 100644
hadoop-hdds/docs/content/design/short-circuit-read-unix-domain-socket.png
create mode 100644 hadoop-hdds/docs/content/design/short-circuit-read.md
create mode 100644 hadoop-hdds/docs/content/feature/Short-Circuit-Read.md
create mode 100644 hadoop-hdds/docs/content/feature/Short-Circuit-Read.zh.md
copy hadoop-ozone/dist/src/main/compose/{ozonesecure/diskbalancer.yaml =>
ozone/short-circuit.yaml} (80%)
copy hadoop-ozone/dist/src/main/compose/ozone/{test-failures1.sh =>
test-short-circuit.sh} (79%)
create mode 100644
hadoop-ozone/dist/src/main/smoketest/short-circuit/short-circuit.robot
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/TestXceiverServerDomainSocket.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestDomainSocketFactory.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
