This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a change to branch branch-0.10
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
from 477bc3037 [MINOR] chore: Fix the issue of license loss (#2248)
new 6cafe465f [#2241] improvement(server): Introduce storage flush
operation timeout cancel to avoid disk hang (#2247)
new 6951e367e [#2219] feat: Introduce ShuffleBlockIdManagerFactory and
PartitionedShuffleBlockIdManager (#2227)
new 7fd296a66 [MINOR] feat(spark-client): Support set accessId by another
config dynamically (#2250)
new da96ff300 [#2251] fix(server): fix wrong disk used space while config
multiply dirs in one mount point (#2252)
new cce120068 [#2241][FOLLOWUP] improvement(server): Mark the underlying
thread name (#2256)
new d2f8ea01f [MINOR] feat(server,dashboard,coordinator): Report
configured metrics of server to coordinator and display to dashboard (#2239)
new 7ae4613ef [MINOR] Bump version to 0.11.0-SNAPSHOT (#2249)
new 2f349976c [MINOR] fix(client): Fix error log for
CoordinatorGrpcRetryableClient#accesCluster (#2258)
new ff534ea86 [#2259] fix(server) dataLength of
ShuffleBufferWithLinkedList not clear after toFlushEvent (#2260)
new d65389794 [#436] feat(client,server): Introduce multi-part
LocalStorageManager (#2253)
new 199a81b6b [MINOR] improvement(spark-client): put sparkConf as extra
properties while client request accessCluster (#2254)
new 850cd8ab3 [MINOR] improvement(dashboard): Support sort application
info by partition num
new 747c02d21 [MINOR] community: Add security.md (#2268)
new c4da70235 [#436][FOLLOWUP] docs: Add documents for
localStorageManagerClass and MultiPartLocalStorageManager (#2266)
new 623a4bf90 [MINOR] Correct the NOTICE and LICENSE (#2271)
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
LICENSE-binary | 3 +-
NOTICE | 2 +-
cli/pom.xml | 2 +-
client-mr/core/pom.xml | 4 +-
client-mr/hadoop2.8/pom.xml | 2 +-
client-mr/hadoop3.2/pom.xml | 2 +-
client-spark/common/pom.xml | 4 +-
.../org/apache/spark/shuffle/RssSparkConfig.java | 16 ++
.../shuffle/manager/RssShuffleManagerBase.java | 17 +-
client-spark/spark2-shaded/pom.xml | 2 +-
client-spark/spark2/pom.xml | 2 +-
.../spark/shuffle/DelegationRssShuffleManager.java | 17 +-
client-spark/spark3-shaded/pom.xml | 2 +-
client-spark/spark3/pom.xml | 2 +-
.../spark/shuffle/DelegationRssShuffleManager.java | 17 ++
.../shuffle/DelegationRssShuffleManagerTest.java | 12 ++
client-tez/pom.xml | 4 +-
client/pom.xml | 4 +-
common/pom.xml | 2 +-
.../java/io/netty/util/CompositeFileRegion.java | 142 +++++++++++++
.../apache/uniffle/common/ShuffleDataSegment.java | 10 +-
.../apache/uniffle/common/ShuffleIndexResult.java | 25 ++-
.../uniffle/common/config/RssClientConf.java | 15 ++
.../common/future/CompletableFutureUtils.java | 72 +++++++
.../uniffle/common/metrics/MetricsManager.java | 8 +
.../uniffle/common/netty/MessageEncoder.java | 3 +-
...fer.java => MultiFileSegmentManagedBuffer.java} | 40 ++--
.../netty/protocol/GetLocalShuffleDataRequest.java | 30 +++
...uest.java => GetLocalShuffleDataV2Request.java} | 92 +++------
.../protocol/GetLocalShuffleIndexResponse.java | 5 +
...se.java => GetLocalShuffleIndexV2Response.java} | 60 ++++--
.../uniffle/common/netty/protocol/Message.java | 13 +-
...tSplitter.java => AbstractSegmentSplitter.java} | 78 ++++---
.../common/segment/FixedSizeSegmentSplitter.java | 90 +-------
.../common/segment/LocalOrderSegmentSplitter.java | 120 +----------
.../apache/uniffle/common/storage/StorageInfo.java | 4 +
.../org/apache/uniffle/common/util/Constants.java | 1 +
.../common/future/CompletableFutureUtilsTest.java | 78 +++++++
.../segment/FixedSizeSegmentSplitterTest.java | 51 +++++
.../segment/LocalOrderSegmentSplitterTest.java | 57 ++++++
coordinator/pom.xml | 2 +-
.../coordinator/CoordinatorGrpcService.java | 3 +-
.../org/apache/uniffle/coordinator/ServerNode.java | 12 +-
dashboard/pom.xml | 2 +-
.../src/main/webapp/src/pages/ApplicationPage.vue | 2 +-
.../webapp/src/pages/serverstatus/NodeListPage.vue | 7 +
deploy/kubernetes/pom.xml | 2 +-
docs/client_guide/client_guide.md | 1 +
docs/client_guide/spark_client_guide.md | 2 +
docs/server_guide.md | 16 +-
integration-test/common/pom.xml | 4 +-
...ShuffleWithRssClientTestWhenShuffleFlushed.java | 10 +-
.../apache/uniffle/test/ShuffleServerGrpcTest.java | 9 +-
.../uniffle/test/ShuffleServerWithLocalTest.java | 12 +-
integration-test/mr/pom.xml | 4 +-
integration-test/spark-common/pom.xml | 4 +-
...thLocalForMultiPartLocalStorageManagerTest.java | 226 +++++++++++++++++++++
integration-test/spark2/pom.xml | 2 +-
integration-test/spark3/pom.xml | 2 +-
integration-test/tez/pom.xml | 4 +-
internal-client/pom.xml | 4 +-
.../client/impl/grpc/CoordinatorGrpcClient.java | 71 ++-----
.../impl/grpc/CoordinatorGrpcRetryableClient.java | 2 +-
.../client/impl/grpc/ShuffleServerGrpcClient.java | 6 +-
.../impl/grpc/ShuffleServerGrpcNettyClient.java | 37 +++-
.../client/request/RssGetShuffleDataRequest.java | 23 ++-
.../client/request/RssSendHeartBeatRequest.java | 10 +-
.../response/RssGetShuffleIndexResponse.java | 10 +-
pom.xml | 2 +-
proto/pom.xml | 2 +-
proto/src/main/proto/Rss.proto | 3 +
security.md | 30 +++
server-common/pom.xml | 2 +-
server/pom.xml | 2 +-
.../java/org/apache/uniffle/server/Checker.java | 5 +-
.../uniffle/server/DefaultFlushEventHandler.java | 26 ++-
.../apache/uniffle/server/RegisterHeartBeat.java | 80 +++-----
.../uniffle/server/ShuffleDataReadEvent.java | 11 +
.../org/apache/uniffle/server/ShuffleServer.java | 47 +++--
.../apache/uniffle/server/ShuffleServerConf.java | 29 +++
.../uniffle/server/ShuffleServerGrpcService.java | 16 +-
.../uniffle/server/ShuffleServerMetrics.java | 9 +
.../org/apache/uniffle/server/ShuffleTaskInfo.java | 34 +++-
.../apache/uniffle/server/ShuffleTaskManager.java | 194 ++++++------------
.../server/block/DefaultShuffleBlockIdManager.java | 220 ++++++++++++++++++++
.../block/PartitionedShuffleBlockIdManager.java | 159 +++++++++++++++
.../server/block/ShuffleBlockIdManager.java | 56 +++++
.../server/block/ShuffleBlockIdManagerFactory.java | 33 +--
.../server/buffer/ShuffleBufferWithLinkedList.java | 1 +
.../server/netty/ShuffleServerNettyHandler.java | 30 ++-
.../server/storage/HybridStorageManager.java | 7 +-
.../server/storage/LocalStorageManager.java | 12 +-
.../server/storage/LocalStorageManagerFactory.java | 30 +--
.../storage/MultiPartLocalStorageManager.java | 118 +++++++++++
.../uniffle/server/storage/StorageManager.java | 4 +
.../server/storage/StorageManagerFactory.java | 2 +-
.../apache/uniffle/server/HealthyMockChecker.java | 2 +-
.../uniffle/server/ShuffleTaskManagerTest.java | 11 +-
.../uniffle/server/UnHealthyMockChecker.java | 2 +-
.../buffer/ShuffleBufferWithLinkedListTest.java | 5 +
.../server/storage/LocalStorageManagerTest.java | 30 ++-
storage/pom.xml | 2 +-
.../common/CompositeReadingViewStorage.java | 91 +++++++++
.../uniffle/storage/common/LocalStorage.java | 15 +-
.../storage/handler/api/ServerReadHandler.java | 2 +
.../impl/CompositeLocalFileServerReadHandler.java | 78 +++++++
.../handler/impl/LocalFileClientReadHandler.java | 1 +
.../handler/impl/LocalFileServerReadHandler.java | 22 +-
108 files changed, 2279 insertions(+), 745 deletions(-)
create mode 100644 common/src/main/java/io/netty/util/CompositeFileRegion.java
create mode 100644
common/src/main/java/org/apache/uniffle/common/future/CompletableFutureUtils.java
copy
common/src/main/java/org/apache/uniffle/common/netty/buffer/{NettyManagedBuffer.java
=> MultiFileSegmentManagedBuffer.java} (50%)
copy
common/src/main/java/org/apache/uniffle/common/netty/protocol/{GetLocalShuffleDataRequest.java
=> GetLocalShuffleDataV2Request.java} (52%)
copy
common/src/main/java/org/apache/uniffle/common/netty/protocol/{GetLocalShuffleIndexResponse.java
=> GetLocalShuffleIndexV2Response.java} (59%)
copy
common/src/main/java/org/apache/uniffle/common/segment/{FixedSizeSegmentSplitter.java
=> AbstractSegmentSplitter.java} (57%)
create mode 100644
common/src/test/java/org/apache/uniffle/common/future/CompletableFutureUtilsTest.java
create mode 100644
integration-test/spark-common/src/test/java/org/apache/uniffle/test/SparkClientWithLocalForMultiPartLocalStorageManagerTest.java
create mode 100644 security.md
create mode 100644
server/src/main/java/org/apache/uniffle/server/block/DefaultShuffleBlockIdManager.java
create mode 100644
server/src/main/java/org/apache/uniffle/server/block/PartitionedShuffleBlockIdManager.java
create mode 100644
server/src/main/java/org/apache/uniffle/server/block/ShuffleBlockIdManager.java
copy
storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileWriterFactory.java
=>
server/src/main/java/org/apache/uniffle/server/block/ShuffleBlockIdManagerFactory.java
(52%)
copy
storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileWriterFactory.java
=>
server/src/main/java/org/apache/uniffle/server/storage/LocalStorageManagerFactory.java
(57%)
create mode 100644
server/src/main/java/org/apache/uniffle/server/storage/MultiPartLocalStorageManager.java
create mode 100644
storage/src/main/java/org/apache/uniffle/storage/common/CompositeReadingViewStorage.java
create mode 100644
storage/src/main/java/org/apache/uniffle/storage/handler/impl/CompositeLocalFileServerReadHandler.java
