This is an automated email from the ASF dual-hosted git repository.
zuston pushed a change to branch branch-0.10
in repository https://gitbox.apache.org/repos/asf/uniffle.git
discard 3cab5a8a0 [#2307] license: Add left jars with Apache license (#2310)
discard 401488705 [MINOR] docs: Fix the license format of
remote_merge_guide.md (#2314)
discard 9aaead8c3 [#2306] improvement: Remove repeated notice information
(#2311)
discard 23ff0ea10 [#2304] improvement: Update the year of NOTICE (#2313)
discard b10a4bc82 [#2298] fix(server): followup to correct the metric of
grpc_get_local_shuffle_index_process_latency (#2302)
discard 77fbd8582 [#2188] fix(client): Fixup timeoutMs for executeTasks in
ShuffleWriteClientImpl#sendAppHeartbeat (#2189)
discard 7d07a9d1f [#2298] fix(server): Correct the metric of
grpc_get_local_shuffle_data_process_latency (#2299)
discard d2faa9566 [#2292] fix(server): Potential hang when
HadoopShuffleWriteHandler initialization failure in
PooledHadoopShuffleWriteHandler (#2293)
discard 562fa7304 [#2290] docker: Fix the build of Centos image (#2291)
discard 7ca24bb38 [#2282] fix(spark3): Invoke with object self in
DelegationRssShuffleManager (#2283)
discard 334a3fb71 Fix ConcurrentModifiedException while use TestAppender
omit 9a9bc7c26 Revert "[MINOR] Bump version to 0.11.0-SNAPSHOT (#2249)"
omit 28dc8a2e0 [MINOR] Add left licenses (#2272)
omit 623a4bf90 [MINOR] Correct the NOTICE and LICENSE (#2271)
omit c4da70235 [#436][FOLLOWUP] docs: Add documents for
localStorageManagerClass and MultiPartLocalStorageManager (#2266)
omit 747c02d21 [MINOR] community: Add security.md (#2268)
omit 850cd8ab3 [MINOR] improvement(dashboard): Support sort application
info by partition num
omit 199a81b6b [MINOR] improvement(spark-client): put sparkConf as extra
properties while client request accessCluster (#2254)
omit d65389794 [#436] feat(client,server): Introduce multi-part
LocalStorageManager (#2253)
omit ff534ea86 [#2259] fix(server) dataLength of
ShuffleBufferWithLinkedList not clear after toFlushEvent (#2260)
omit 2f349976c [MINOR] fix(client): Fix error log for
CoordinatorGrpcRetryableClient#accesCluster (#2258)
omit 7ae4613ef [MINOR] Bump version to 0.11.0-SNAPSHOT (#2249)
omit d2f8ea01f [MINOR] feat(server,dashboard,coordinator): Report
configured metrics of server to coordinator and display to dashboard (#2239)
omit cce120068 [#2241][FOLLOWUP] improvement(server): Mark the underlying
thread name (#2256)
omit da96ff300 [#2251] fix(server): fix wrong disk used space while config
multiply dirs in one mount point (#2252)
omit 7fd296a66 [MINOR] feat(spark-client): Support set accessId by another
config dynamically (#2250)
omit 6951e367e [#2219] feat: Introduce ShuffleBlockIdManagerFactory and
PartitionedShuffleBlockIdManager (#2227)
omit 6cafe465f [#2241] improvement(server): Introduce storage flush
operation timeout cancel to avoid disk hang (#2247)
add 4eebbd461 [#2241] improvement(server): Introduce storage flush
operation timeout cancel to avoid disk hang (#2247)
add 3cf82d70f [#2219] feat: Introduce ShuffleBlockIdManagerFactory and
PartitionedShuffleBlockIdManager (#2227)
add b00b67c38 [MINOR] feat(spark-client): Support set accessId by another
config dynamically (#2250)
add 99eaa9d27 [#2251] fix(server): fix wrong disk used space while config
multiply dirs in one mount point (#2252)
add c1bfa043d [#2241][FOLLOWUP] improvement(server): Mark the underlying
thread name (#2256)
add 910823d81 [MINOR] feat(server,dashboard,coordinator): Report
configured metrics of server to coordinator and display to dashboard (#2239)
add 7787519b9 [MINOR] Bump version to 0.11.0-SNAPSHOT (#2249)
add ab1d3aeba [MINOR] fix(client): Fix error log for
CoordinatorGrpcRetryableClient#accesCluster (#2258)
add 54611f3f2 [#2259] fix(server) dataLength of
ShuffleBufferWithLinkedList not clear after toFlushEvent (#2260)
add b7d391c80 [#436] feat(client,server): Introduce multi-part
LocalStorageManager (#2253)
add 204e4e4ff [MINOR] improvement(spark-client): put sparkConf as extra
properties while client request accessCluster (#2254)
add bd7c2cca8 [MINOR] improvement(dashboard): Support sort application
info by partition num
add ac89c19b9 [MINOR] community: Add security.md (#2268)
add db611f442 [#436][FOLLOWUP] docs: Add documents for
localStorageManagerClass and MultiPartLocalStorageManager (#2266)
add 28475fced [MINOR] Correct the NOTICE and LICENSE (#2271)
add 4ce1aa825 [MINOR] Add left licenses (#2272)
add 0552d3b0a [#1844] fix(spark): Reassign shuffle servers when retrying
stage (#1845)
add 44bfd0da8 [MINOR] test: Fix ConcurrentModifiedException while use
TestAppender (#2285)
add b7c9ca59b [#2282] fix(spark3): Invoke with object self in
DelegationRssShuffleManager (#2283)
add 8270bfd7a [#2290] docker: Fix the build of Centos image (#2291)
add a1d325287 [#2292] fix(server): Potential hang when
HadoopShuffleWriteHandler initialization failure in
PooledHadoopShuffleWriteHandler (#2293)
add 258db12f5 [#2288] feat(dashboard): Show nodeInfo in excludeNode page
(#2289)
add 8c550309d [#2298] fix(server): Correct the metric of
grpc_get_local_shuffle_data_process_latency (#2299)
add 0481f210e [#2188] fix(client): Fixup timeoutMs for executeTasks in
ShuffleWriteClientImpl#sendAppHeartbeat (#2189)
add e7e191b3f [#2279] improvement(spark): Trigger the upstream rewrite
when the read stage fails (#2281)
add 1c56e74c9 [#2298] fix(server): followup to correct the metric of
grpc_get_local_shuffle_index_process_latency (#2302)
add a413556e4 [#2304] improvement: Update the year of NOTICE (#2313)
add 288f5ca51 [#2305] improvement: Add copyright for PingCAP (#2312)
add e9e59046f [#2306] improvement: Remove repeated notice information
(#2311)
add 580b4f2a4 [MINOR] docs: Fix the license format of
remote_merge_guide.md (#2314)
add 9396abce7 [#2307] license: Add left jars with Apache license (#2310)
add c90d463db [#2308] improvement(server): More detailed and clear logs on
reading failure for FileSegmentManagedBuffer (#2309)
add d7aad66ac [#2315] feat(test): Introduce a client simulator tool to
simulate send shuffle data request for testing server performance (#2316)
add fd27a0834 [#2254][FOLLOWUP] improvement(spark):Add include key filter
before report extraProperties (#2265)
add 8aa361683 [#1782] fix: Fix flaky test ShuffleBufferManagerTest (#2100)
add 3b0521abe [#2269] refactor: Fix duplicated blockIds issue caused by
duplicated reportShuffleResult (#2270)
add 66c3f3c5e [MINOR] test: fix NPE for test
PartitionBalanceAssignmentStrategyTest (#2322)
add 74366d580 [MINOR] fix(test): Disable System exit for HadoopTestBase
(#2327)
add a669c2b5f [MINOR] fix(test): Fix flaky test for CoordinatorGrpcTest
(#2324)
add 995de6d00 [MINOR] improvement(server): LocalFileNioWriter support
writeIndex api (#2320)
add 284d139ca [MINOR] feat(server): Introduce LocalFileBlackHoleWriter for
test flush ceiling performance (#2321)
add eed5ebaf6 [MINOR] fix(server): Server heartbeat to coordinator while
unregister shuffle (#2132)
add b2d060ac2 [MINOR] fix(spark-client) Close coordinator client in time
(#2328)
add c14bff7ea [MINOR] fix(spark-client): Fix fallback to sort fail issue
(#2329)
add 1ac46dbd9 [MINOR] fix(test): Fix flaky test for
DynamicClientConfServiceHadoopTest (#2326)
add a15751b59 [MINOR] fix(test): Fix flaky test for
CoordinatorGrpcServerTest#testGrpcConnectionSize (#2323)
add 3ea5c2c78 [#1872] improvement: Update the project directory name
(#2319)
add 5b7d18c30 [MINOR] fix(test): Fix flaky test CoordinatorServerTest#test
(#2332)
add 43879ca13 [MINOR] improvement(spark-client): Refactor
RssShuffleManager for spark v2 and v3 to reduce redundant code (#2330)
add 51b009ae3 [MINOR] feat(server): Introduce LocalFileNullDeviceWriter to
flush data into null device (#2337)
add 4288efee4 [#2335] Improvement: Fix the warning of that parameter
systemProperties is deprecated: Use systemPropertyVariables instead (#2336)
add 3bce707c0 [#2343] Improvement: Fix the warning Discovered
module-info.class. Shading will break its strong encapsulation. (#2345)
add 08e739708 [#2346] Improvement: Fix the warning that delete(Path) in
FileSystem has been deprecated (#2347)
add deb5de309 [#2333] Improvement: extend the timeout value for
QuorumTest#case6 (#2348)
add 7e63bdeb7 [#2350] improvement(coordinator): Add metrics of active/lost
server number (#2351)
add 7cad484a9 [#2349] Improvement: Fix the warning: unchecked method
invocation: method put in interface Map is applied to given types (#2352)
add 3590940eb [#1658] improvement: FileSegmentManagedBuffer#nioByteBuffer
read result cacheable (#1659)
add e6e070236 [#2354] feat(client): Explicitly setting grpc netty based
event loop threads to avoid too much threads (#2355)
add 89dd9283a fix(CI): Upgrade to actions/upload-artifact@v4 (#2360)
add 9e5cd616a improvement(spark): Involve shuffle result report time into
shuffle write time metrics (#2361)
add e5cfc4a02 improvement(spark): Record the partition choosing sort time
for analysis (#2364)
add 47e5d17dd [#2362] feat(client): Add support of zstd parallel
compression (#2363)
add 81e00bec9 [#291] feat(client): Introduce PrefetchableClientReadHandler
to support async read (#2365)
add 4cb8c6982 [#291] feat(client): Record background fetch time for
prefetch (#2366)
add 608471a12 [#2362] feat(client): Log more compression infos (#2367)
add 124a90158 [#2356] Improvement: Fix the warning: unchecked call to
put(E) as a member of the raw type RMRecordsReader.Queue (#2357)
add e2523ec0b improvement(client): Simplify skipping processed blockIds
check to speed up (#2368)
add 12d611b5c [#2353] Improvement: Fix the warning: unchecked method
invocation: method `sendCachedBuffer` in class
`RMRecordsReader.RecordsCombiner` is applied to given types (#2358)
add af342e6e8 [#2083] improvement: Quickly delete local or HDFS data at
the shuffleId level. (#2084)
add ea3ec58cc chore: Remove the rust impl (#2373)
add 413c852ad [#2370] Improvement: Fix the warning: [unchecked] unchecked
call to add(E) as a member of the raw type List (#2371)
add ba7a26790 [Minor] Improve return boolean expression (#2375)
add 8ad0f8d2e test(coordinator): The interval of
SimpleClusterManager.nodesCheck should be bigger than that of
org.awaitility.Awaitility.await (#2377)
add 01f6e2557 [#2378] test(client): Fix flaky test QuorumTest#case10
(#2379)
add 69ea53f00 [#2372] Improvement: Fix the warning: unchecked method
invocation: method deserializeStream in class SerializerInstance is applied to
given types (#2374)
add 41155a7f9 [#2382] improvement(test): Fix ServletTest,
ShuffleServerFaultToleranceTest,ShuffleServerWithKerberizedHadoopTest to use
random port (#2384)
add 86da646b2 [#2383] Improvement: Fix the warning that unchecked call to
`RecordsReader(RssConf,SerInputStream,Class<K>,Class<V>,boolean,boolean)` as a
member of the raw type `RecordsReader` (#2386)
add f65d09e42 [#2389] fix:(remote merge): Fixed the issue of losing data
when calling hasNext multiple times. (#2390)
add 87362fb70 [#2385] improvement(test): Make all tests in
`integration-common-test` use random port (#2387)
add 4300f93f9 [#1745] feat(remote merge): Introduce kryo serializer for
spark. (#2395)
add 75ccb7256 fix(test): Prevent shuffle rpc client leak in all
integration test (#2403)
add 31b443e37 [#2385] fix(server): Concurrent FileSegmentManagedBuffer
release (#2406)
add b7969b47a introduce a new way to support jetty use random port (#2402)
add 9ce55e32b [#2398]Improvement: Fix the warning unchecked generic array
creation for varargs parameter of type Class<? extends Throwable>[] (#2400)
add 4def3cc7e [#2093][followup] feat: Add support of partition split for
grpc (#2396)
add 3874f154c [#2388] improvement(test): mr, tez, spark integration test
use random port (#2397)
add dac1065fe [#2411] fix(spark): Spill memory corresponding to
successfully sent blocks (#2415)
add 47ce1355c fix(test): Fix test about nvme mount for
LocalStorageManagerTest (#2416)
add cd8c8bae8 [#2417] improvement: Remove "incubating" in scripts and docs
(#2418)
add cad584a2f [#1651] improvement(netty): Set Netty as the default server
type (#1919)
add 76a4dcf76 feat(client): Introduce `LOAD_BALANCE` mode for partition
split (#2408)
add 475022b33 [#1651] fix(netty): Set Netty as the default server type
(#2421)
add c826cf5a1 improvement(test): Clean temp dir and allocate real ports
for MiniDFSCluster (#2425)
add 6085a7dfe [MINOR] chore(ci): update dorny/paths-filter@v2 to
dorny/[email protected] (#2429)
add 650d1c7d8 [#2425][FOLLOWUP] fix: Fix spotless errors
add 94e211c4d [MINOR] Eliminate the usage of the shaded Guava Sets class
(#2432)
add 0c6ec3a4d [MINOR] improvement(client): Detailed error message in
ShuffleWriteClientImpl (#2433)
add fe447f243 [MINOR] improvement: Fix logger classname in RssMergeManager
(#2435)
add 32187f8db [#2426] improvement: Skip the application which is expired
when flushing buffers (#2427)
add 6555280b4 [#2082] fix: spark executor task error when reading shuffle
data when using java open jdk11 (#2436)
add 9007bed27 [#1651] fix(netty): Set Netty as the default client type
(#2422)
add 890758bb9 fix(test): Fix test about default storage type for
DefaultStorageMediaProviderTest (#2419)
add 9d43c1e6d [MINOR] fix(docs): Correct storage type in MR / TEZ
documentation. (#2438)
add 07eb8d55d [#2439] fix(test): remove shuffleServer from coordinators
when it stops (#2440)
add 6843c06b1 fix(spark3): Reserve previous writer initialization way for
gluten + uniffle (#2437)
add f732fc227 [#493][FOLLOWUP] improvement: replace putIfAbsent to avoid
performance loss (#2444)
add eedbc5989 [#2428] fix(test): Fix flaky test for
LocalStorageManagerTest#testGetLocalStorageInfo (#2430)
add b43872f47 [#2441] improvement(client): remove synchronized in
ShuffleServerClientFactory#getShuffleServerClient (#2442)
add da71d06b6 fix(CI): Update CI ubuntu version to latest (#2448)
add 6c00c79e2 [MINOR] fix(docker): Fix hadoop version (#2447)
add f27570e0f Fix: Use correct Field object for InputDescriptor className
(#2445)
add 9c01bff79 [MINOR] fix(docs): Fixing formatting error (#2452)
add 7a9e7cf13 [#2457] feat(spark): Introducing shuffle-server data push
statistics (#2458)
add 21e175333 [MINOR] fix(docs): Fixing rss-env.sh directory (#2462)
add 4064da22d [Improvement] Clear the alarm log when starting the relevant
service (#2453)
add 4e3dfbc64 [#2460] feat(spark)(part-1): Introducing spark uniffle ui
for better observability (#2459)
add 3e681e394 [#2460] feat(spark)(part-2): Add support of history server
plugin (#2464)
add aa85e3f5f improvement(docker): Add dashboard to docker (#2463)
add 4d9202038 [#2466] fix(server): Ignore appIdToInfos when serialize
ServerNode (#2467)
add 3beec2b1a [#2460] feat(spark)(part-3): Add support of read metrics
report (#2465)
add 95dccbda6 [#2454] improvement(server): Remove lock in
ShuffleTaskManager#registerShuffle (#2456)
add 057dcd227 [#2460] feat(spark)(part-4): Hybrid storage reading
statistics (#2468)
add 917e45e17 [#2369] fix(spark): Potential race condition on reading
prefetch (#2475)
add 293d01b24 [#1745][FOLLOWUP] fix(client): exclude kryo when package
spark shaded client (#2478)
add cc440dfe0 [#2469] feat(spark): Introducing a dedicated shuffleId of
uniffle for stage retry (#2471)
add 855cec120 [MINOR] improvement(remote merge): print log when merge
failed (#2479)
add 07cc9ae82 [#2460] feat(spark3): Tracking failure of pushing data for
spark UI (#2481)
add abb6bd52e [#2483] fix(spark): Avoid `NoSuchElementException` error in
data pusher (#2484)
add bbdf71e67 [#2485] fix(Client): NPE in
`CoordinatorGrpcRetryableClient.getShuffleAssignments` (#2486)
add b7faef885 [#2487] fix(spark3): Fix NPE in UniffleListener (#2488)
add b45e986d4 [#2490] feat(spark): Display memory usage of all consumers
under low-memory conditions (#2491)
add b3855e286 [#2495] feat(server): Add an option to disable flush buffer
when caching data (#2497)
add c83a1b5eb [#1727] feat(server): Introduce local allocation buffer to
store blocks in memory (#2492)
add 6c636ac85 [#2460] feat(spark3): Add fine-grained shuffle write times
with different parts (#2498)
add 6356e4290 feat(client): Log show the bytes length for each data RPC
add 575eee05c [#2460] feat(spark3): Add client/uniffle different observed
shuffle speed (#2500)
add 22be6287a [#2460] feat(spark3): Add reassign info into UI tab (#2501)
add 3407a0a35 [#2460] fix(spark3): Record require buffer failure number
every time (#2505)
add a69930dee [#2506] feat(spark3): Introduce option to enable reorder
multi servers for reader (#2507)
add 907e6252c [#2512] feat(spark3): Attach stacktrace on any exception
when UI tab rendering (#2513)
add 786ed4843 [#2514] fix(spark3): Invalid decimal value conversion on
spark UI rendering (#2515)
add 5a45daaf3 [#2502] feat(spark3): Fast fail for those stale assign
blocks for partition reassign (#2516)
add ed2684086 [#2508] feat(spark3): Record failed tasks on any shuffle
write/failure into event logs (#2509)
add f3eaf0627 [#2518] fix(client): Disable fast fail on any pushing
failure if partition reassign is activated (#2519)
add 69b2d5178 [#2503] fix(spark): Skip reassignment number check on
partition split (#2504)
add 7781bf626 [#2520] fix(spark3): Shuffle write total duration is
incorrectly accumulated in event log (#2521)
add 887d04284 [#2496] improvement(server): Improve the performance of
flushing single buffer (#2523)
add 80aa1705c [#2494] feat(client): Introduce codec statistics to measure
de/compression performance (#2528)
add f194cc57f [#2529] fix(spark3): Incorrect clientInfo without nettyPort
if netty is enabled (#2530)
add ad7babb7d [#2496] improvement(server): Use tryLock instead of
synchronized on buffer flushing (#2524)
add f2d7d868f [#2526] improvement: Reset the chunkID if it overflow (#2531)
add 77e6ab1f5 [#2494] feat(spark): Overlapping compression to avoid block
shuffle writing (#2511)
add ece59eee1 [#2537] feat(spark): Introduce option to activate small
cache in grpc server (#2538)
add 262519e5b [#2540] feat(spark): Show compression ratio into spark UI
tab (#2542)
add f1323ab95 [#2536] feat(spark): Controllable concurrency for
overlapping compression (#2541)
add f57eabe13 [#2545] feat: Align with spark executor cores on overlapping
compression (#2548)
add 10df07dc8 [#2533] fix(client): NPE on rendering spark UI tab (#2552)
add 4629fc6ee [#2544] fix(client): NPE about StatisticsCodec (#2547)
add 03f42c542 [#2525] improvement(server): add some metrics for LAB (#2553)
add ccc534be5 [#2539] fix(spark): NPE in DataPusher when sendShuffleData
fails (#2556)
add 4b256e9c5 [#2492][FOLLOWUP] improvement: change the default value of
chunkPoolCapacityRatio (#2554)
add 850db71f1 [MINOR] improvement(client): Simplify logging of heartbeat
failures (#2559)
add a5086b305 [#2555] feat(server): support dynamically modifying the tags
of shuffle server (#2557)
add 8fa51bb30 [MINOR] improvement(client): Shorten log for multi replica
client reader (#2561)
add be3a1ffc0 [#2560] improvement(client): Fast fail on hadoop reader
initialization failure (#2551)
add 1713c1f70 [#2549] fix(spark): Invalid remote storage configuration was
propagated during application registration (#2550)
add 066e71e6c [#2563] improvement(spark): Add more logs of shuffle write
on reassignment failure (#2564)
add 99d5c3a11 [#2525][FOLLOWUP] fix(server): remove metric
`buffer_block_size` (#2567)
add 41d0fc53f [#2558] improvement(server): Limit the max flush event count
for a single buffer (#2562)
add 7414ed5f5 [MINOR] chore(CI): bump dorny/paths-filter from v3.0.2 to
de90cc6fb38fc0963ad72b210f1f284cd68cea36 (#2570)
add 4eb83eea2 [#2571] fix(client): Race condition when adding shuffle
servers (#2574)
add a1974f6e3 [#2576] fix: Warm up java version var to eliminate lock on
creating concurrent hashmap (#2577)
add e6f0941ad [#2581] fix(spark): Use `SparkContext.getActive` instead of
`getOrCreate` to align with method semantics (#2582)
add 7f1586e9c [#2579] fix(spark): Correct partition length for overlapping
compression (#2580)
add 9b611cf14 [#2527] docs: Add some docs for LAB (#2585)
add 0facb7be1 [#2568] feat(spark): Use space-efficient protobuf for
`MutableShuffleHandleInfo` to reduce RPC memory overhead (#2578)
add e787d87c9 [#2583] fix(spark): Enable taskIds filter only on AQE and
multi replicas for reader (#2584)
add fe0ff7e60 [#2586] fix(spark): Support writer switching servers on
partition split with LOAD_BALANCE mode without reassign (#2587)
add f3bc84fbe [#2494] feat(spark): Enable overlapping compression by
default (#2588)
add 32f4ac6c5 [#2575] fix(spark): Fix java.lang.IndexOutOfBoundsException:
len is negative (#2589)
add d5e689c32 [#2592] fix(spark): Ignore failure when reporting shuffle
read metrics to driver (#2593)
add 2a32171b9 [#2569] feat(spark): Add statistic of shuffle read times
(#2598)
add 1e48bc673 [#2601] feat(spark): Introduce overlapping decompression for
shuffle read (#2602)
add 96bf76cbc [#2591] feat(client): Introduce the mechanism to report
localfile read plan (#2603)
add 14a50985a [#2591] fix(client): Incorrect header length for
getLocalShuffleDataV3 (#2604)
add 9fdde02b5 [#2591] fix(client): Missing task_id propagation in
getLocalShuffleDataV3 (#2605)
add 7015613ad [#2609] feat(spark): Expose `checkDataIfAnyFailure` method
so that Gluten can invoke it to trigger reassign ASSP (#2610)
add 04964f30e [#2606] feat(mr): Add safety switch for map-stage combiner
(#2607)
add 1a46e2d0c [#2599] fix(spark): Fix bug the incorrect shuffle read
metric for spark (#2600)
add 10aa39dcc Remove protected branch (#2615)
add e0a49b934 Just a minor
new 37004c9a4 Revert "[MINOR] Bump version to 0.11.0-SNAPSHOT (#2249)"
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (3cab5a8a0)
\
N -- N -- N refs/heads/branch-0.10 (37004c9a4)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 8 +-
.github/workflows/build.yml | 2 +-
.github/workflows/changes.yml | 4 +-
.github/workflows/docker.yml | 4 +-
.github/workflows/parallel.yml | 6 +-
.github/workflows/rust.yml | 70 -
.github/workflows/sequential.yml | 4 +-
.github/workflows/single.yml | 4 +-
DISCLAIMER | 2 +-
LICENSE | 3 +
NOTICE | 2 +-
NOTICE-binary | 2 +-
README.md | 2 +-
bin/start-coordinator.sh | 1 +
.../apache/uniffle/client/HttpClientFactory.java | 11 +-
client-mr/core/pom.xml | 1 +
.../hadoop/mapred/RssMapOutputCollector.java | 31 +-
.../org/apache/hadoop/mapreduce/RssMRConfig.java | 4 +
.../hadoop/mapreduce/task/reduce/RMRssShuffle.java | 12 +-
.../hadoop/mapreduce/task/reduce/RssFetcher.java | 2 +-
.../hadoop/mapreduce/v2/app/RssMRAppMaster.java | 1 -
.../hadoop/mapred/SortWriteBufferManagerTest.java | 4 -
.../hadoop/mapreduce/task/reduce/FetcherTest.java | 10 +-
.../shuffle/DelegationRssShuffleManagerUtils.java} | 34 +-
.../org/apache/spark/shuffle/RssSparkConfig.java | 33 +
.../apache/spark/shuffle/RssSparkShuffleUtils.java | 51 +-
.../spark/shuffle/RssStageResubmitManager.java | 23 +-
.../shuffle/events/ShuffleAssignmentInfoEvent.java | 25 +-
.../apache/spark/shuffle/events/ShuffleMetric.java | 19 +-
.../spark/shuffle/events/ShuffleReadMetric.java | 74 +
.../spark/shuffle/events/ShuffleWriteMetric.java | 59 +
.../spark/shuffle/events/ShuffleWriteTimes.java | 84 +
.../shuffle/events/TaskReassignInfoEvent.java | 45 +
.../shuffle/events/TaskShuffleReadInfoEvent.java | 77 +
.../shuffle/events/TaskShuffleWriteInfoEvent.java | 82 +
.../apache/spark/shuffle/events/UniffleEvent.java | 10 +-
.../shuffle/handle/MutableShuffleHandleInfo.java | 201 +-
.../spark/shuffle/handle/ShuffleHandleInfo.java | 12 +-
.../shuffle/handle/SimpleShuffleHandleInfo.java | 15 +-
.../handle/StageAttemptShuffleHandleInfo.java | 11 +-
.../shuffle/handle/split/PartitionSplitInfo.java | 40 +-
.../shuffle/reader/RssShuffleDataIterator.java | 68 +-
.../apache/spark/shuffle/writer/AddBlockEvent.java | 13 +-
.../apache/spark/shuffle/writer/DataPusher.java | 74 +-
.../writer/OverlappingCompressionDataPusher.java | 85 +
.../shuffle/writer/TaskAttemptAssignment.java | 65 +-
.../spark/shuffle/writer/WriteBufferManager.java | 177 +-
.../org/apache/uniffle/shuffle/BlockIdManager.java | 3 +-
.../BlockIdSelfManagedShuffleWriteClient.java | 25 +
.../uniffle/shuffle/ShuffleIdMappingManager.java | 128 +
.../shuffle/manager/RssShuffleManagerBase.java | 772 +++-
.../manager/RssShuffleManagerInterface.java | 10 +-
.../shuffle/manager/ShuffleManagerGrpcService.java | 378 +-
.../handle/MutableShuffleHandleInfoTest.java | 72 +
.../shuffle/reader/RssShuffleDataIteratorTest.java | 3 +-
.../spark/shuffle/writer/DataPusherTest.java | 39 +-
.../OverlappingCompressionDataPusherTest.java | 94 +
.../shuffle/writer/WriteBufferManagerTest.java | 26 +-
.../shuffle/manager/DummyRssShuffleManager.java | 8 +-
client-spark/{spark3 => extension}/pom.xml | 83 +-
.../org.apache.spark.status.AppHistoryServerPlugin | 1 +
.../apache/spark/UniffleHistoryServerPlugin.scala | 24 +-
.../scala/org/apache/spark/UniffleListener.scala | 170 +
.../scala/org/apache/spark/UnifflePlugin.scala | 65 +
.../org/apache/spark/UniffleStatusStore.scala | 187 +
.../scala/org/apache/spark/ui/ShufflePage.scala | 569 +++
.../scala/org/apache/spark/ui/ShuffleTab.scala | 20 +-
client-spark/spark2-shaded/pom.xml | 5 +
.../spark/shuffle/DelegationRssShuffleManager.java | 39 +-
.../apache/spark/shuffle/RssShuffleManager.java | 452 +-
.../spark/shuffle/writer/RssShuffleWriter.java | 58 +-
.../shuffle/DelegationRssShuffleManagerTest.java | 10 +-
.../spark/shuffle/RssSpark2ShuffleUtilsTest.java | 23 +
.../spark/shuffle/writer/RssShuffleWriterTest.java | 3 +-
client-spark/spark3-shaded/pom.xml | 5 +
client-spark/spark3/pom.xml | 5 +
.../spark/shuffle/DelegationRssShuffleManager.java | 41 +-
.../apache/spark/shuffle/RssShuffleManager.java | 560 +--
.../spark/shuffle/reader/RssShuffleReader.java | 157 +-
.../shuffle/writer/PartitionLengthStatistic.java | 56 +
.../spark/shuffle/writer/RssShuffleWriter.java | 259 +-
.../shuffle/DelegationRssShuffleManagerTest.java | 86 +
.../spark/shuffle/RssShuffleManagerTest.java | 21 +
.../spark/shuffle/RssShuffleManagerTestBase.java | 5 +-
.../spark/shuffle/RssSpark3ShuffleUtilsTest.java | 23 +
.../writer/PartitionLengthStatisticTest.java | 67 +
.../spark/shuffle/writer/RssShuffleWriterTest.java | 3 +-
client-tez/pom.xml | 1 +
.../org/apache/tez/dag/app/RssDAGAppMaster.java | 6 +-
.../tez/dag/app/TezRemoteShuffleManager.java | 1 -
.../common/shuffle/impl/RssShuffleManager.java | 2 +-
.../library/common/shuffle/impl/RssTezFetcher.java | 2 +-
.../shuffle/orderedgrouped/RssMergeManager.java | 2 +-
.../orderedgrouped/RssShuffleScheduler.java | 2 +-
.../orderedgrouped/RssTezShuffleDataFetcher.java | 2 +-
.../common/shuffle/impl/RssTezFetcherTest.java | 6 +
.../RssTezShuffleDataFetcherTest.java | 6 +
.../common/sort/buffer/WriteBufferManagerTest.java | 4 -
.../uniffle/client/api/ShuffleReadClient.java | 7 +-
.../uniffle/client/api/ShuffleWriteClient.java | 20 +-
.../client/factory/ShuffleClientFactory.java | 55 +
.../uniffle/client/impl/DecompressionWorker.java | 103 +
.../client/impl/FailedBlockSendTracker.java | 7 +
.../uniffle/client/impl/ShuffleReadClientImpl.java | 76 +-
.../client/impl/ShuffleWriteClientImpl.java | 141 +-
.../client/record/reader/KeyValueReader.java | 8 +-
.../client/record/reader/RMRecordsReader.java | 60 +-
.../client/response/CompressedShuffleBlock.java | 5 +-
...fleBlock.java => DecompressedShuffleBlock.java} | 23 +-
.../client/response/SendShuffleDataResult.java | 14 +
.../uniffle/client/response/ShuffleBlock.java | 10 +-
.../uniffle/client/util/RssClientConfig.java | 4 +-
.../org/apache/uniffle/client/ClientUtilsTest.java | 5 +-
.../java/org/apache/uniffle/client/TestUtils.java | 4 +-
.../client/impl/DecompressionWorkerTest.java | 89 +
.../client/impl/ShuffleReadClientImplTest.java | 20 +-
.../record/reader/MockedShuffleServerClient.java | 3 +-
.../record/reader/MockedShuffleWriteClient.java | 4 -
.../client/record/reader/RMRecordsReaderTest.java | 38 +-
.../client/record/writer/RecordCollectionTest.java | 4 +
common/pom.xml | 4 +
.../uniffle/common/DeferredCompressedBlock.java | 106 +
.../{Arguments.java => PartitionSplitMode.java} | 18 +-
.../{ShuffleDataSegment.java => ReadSegment.java} | 32 +-
.../uniffle/common/ReconfigurableRegistry.java | 3 +-
.../apache/uniffle/common/ShuffleBlockInfo.java | 75 +-
.../uniffle/common/ShufflePartitionedBlock.java | 24 +-
.../apache/uniffle/common/ShuffleReadTimes.java | 97 +
.../apache/uniffle/common/ShuffleServerInfo.java | 27 +-
.../apache/uniffle/common/compression/Codec.java | 12 +-
.../common/compression/StatisticsCodec.java | 119 +
.../uniffle/common/compression/ZstdCodec.java | 22 +-
.../apache/uniffle/common/config/ConfigUtils.java | 3 +
.../apache/uniffle/common/config/RssBaseConf.java | 39 +-
.../uniffle/common/config/RssClientConf.java | 75 +-
.../netty/buffer/FileSegmentManagedBuffer.java | 30 +-
...uest.java => GetLocalShuffleDataV3Request.java} | 48 +-
.../uniffle/common/netty/protocol/Message.java | 1 +
.../org/apache/uniffle/common/rpc/GrpcServer.java | 9 +-
.../serializer/kryo/KryoDeserializationStream.java | 109 +
.../serializer/kryo/KryoSerializationStream.java | 86 +
.../KryoSerializer.java} | 14 +-
.../serializer/kryo/KryoSerializerInstance.java | 118 +
.../common/serializer/kryo/PoolWrapper.java | 68 +
.../apache/uniffle/common/util/ByteBufUtils.java | 7 +
.../org/apache/uniffle/common/util/JavaUtils.java | 11 +-
.../org/apache/uniffle/common/util/RssUtils.java | 45 +
.../org/apache/uniffle/common/web/JettyServer.java | 6 +-
.../common/web/resource/MetricResource.java | 1 -
...yTest.java => DeferredCompressedBlockTest.java} | 46 +-
.../apache/uniffle/common/KerberizedHadoop.java | 84 +-
.../uniffle/common/ShuffleBlockInfoTest.java | 28 +
.../common/ShufflePartitionedBlockTest.java | 11 -
.../uniffle/common/ShuffleServerInfoTest.java | 18 +
.../apache/uniffle/common/merger/MergerTest.java | 20 +-
.../netty/buffer/FileSegmentManagedBufferTest.java | 58 +
.../apache/uniffle/common/port/PortRegistry.java | 178 -
.../common/records/RecordsReaderWriterTest.java | 50 +-
.../common/serializer/KryoSerializerTest.java | 152 +
.../common/serializer/SerializerFactoryTest.java | 12 +-
.../common/serializer/WritableSerializerTest.java | 16 +-
.../apache/uniffle/common/util/RetryUtilsTest.java | 14 +-
.../apache/uniffle/common/web/JettyServerTest.java | 36 +-
coordinator/pom.xml | 1 +
.../uniffle/coordinator/ApplicationManager.java | 2 +-
.../uniffle/coordinator/CoordinatorConf.java | 10 +
.../coordinator/CoordinatorGrpcService.java | 14 +-
.../uniffle/coordinator/CoordinatorServer.java | 7 +-
.../org/apache/uniffle/coordinator/ServerNode.java | 3 +-
.../uniffle/coordinator/SimpleClusterManager.java | 116 +-
.../coordinator/metric/CoordinatorMetrics.java | 8 +-
.../coordinator/web/resource/ServerResource.java | 11 +-
.../uniffle/coordinator/CoordinatorConfTest.java | 2 +-
.../uniffle/coordinator/CoordinatorServerTest.java | 81 +-
.../coordinator/SimpleClusterManagerTest.java | 54 +-
.../assignment/BasicAssignmentStrategyTest.java | 13 +
.../PartitionBalanceAssignmentStrategyTest.java | 13 +
coordinator/src/test/resources/coordinator.conf | 2 +-
.../apache/uniffle/dashboard/web/Dashboard.java | 2 +
.../main/webapp/src/pages/ShuffleServerPage.vue | 12 +-
.../src/pages/serverstatus/ExcludeNodeList.vue | 264 --
.../webapp/src/pages/serverstatus/NodeListPage.vue | 174 +-
dashboard/src/main/webapp/src/router/index.js | 3 +-
deploy/docker/build.sh | 4 +
deploy/docker/docker-compose.yml | 12 +
deploy/docker/uniffle/Dockerfile | 1 +
deploy/docker/uniffle/start.sh | 7 +
deploy/kubernetes/docker/build.sh | 1 +
dev/scripts/checkshade.sh | 2 +-
docs/client_guide/client_guide.md | 33 +-
docs/client_guide/mr_client_guide.md | 2 +-
docs/client_guide/spark_client_guide.md | 24 +
docs/client_guide/tez_client_guide.md | 2 +-
docs/dashboard_guide.md | 28 +-
docs/server_guide.md | 30 +-
integration-test/common/pom.xml | 4 +-
.../org/apache/uniffle/test/AccessClusterTest.java | 54 +-
.../uniffle/test/AssignmentWithTagsTest.java | 112 +-
.../uniffle/test/CoordinatorAdminServiceTest.java | 12 +-
.../uniffle/test/CoordinatorAssignmentTest.java | 33 +-
.../uniffle/test/CoordinatorGrpcServerTest.java | 46 +-
.../apache/uniffle/test/CoordinatorGrpcTest.java | 63 +-
.../test/CoordinatorReconfigureNodeMaxTest.java | 29 +-
.../apache/uniffle/test/CoordinatorTestBase.java | 12 +-
.../uniffle/test/DiskErrorToleranceTest.java | 40 +-
.../test/DynamicClientConfServiceHadoopTest.java | 111 +-
.../apache/uniffle/test/FetchClientConfTest.java | 52 +-
.../test/HealthCheckCoordinatorGrpcTest.java | 48 +-
.../org/apache/uniffle/test/HealthCheckTest.java | 2 +-
.../test/HybridStorageFaultToleranceBase.java | 21 +-
.../test/HybridStorageHadoopFallbackTest.java | 45 +-
.../test/HybridStorageLocalFileFallbackTest.java | 35 +-
.../apache/uniffle/test/IntegrationTestBase.java | 158 +-
.../test/PartitionBalanceCoordinatorGrpcTest.java | 51 +-
.../java/org/apache/uniffle/test/QuorumTest.java | 148 +-
.../test/RemoteMergeShuffleWithRssClientTest.java | 101 +-
...ShuffleWithRssClientTestWhenShuffleFlushed.java | 102 +-
.../apache/uniffle/test/RpcClientRetryTest.java | 51 +-
.../java/org/apache/uniffle/test/ServletTest.java | 155 +-
.../apache/uniffle/test/ShuffleReadWriteBase.java | 7 -
.../ShuffleServerConcurrentWriteOfHadoopTest.java | 34 +-
.../test/ShuffleServerFaultToleranceTest.java | 39 +-
.../apache/uniffle/test/ShuffleServerGrpcTest.java | 49 +-
.../test/ShuffleServerInternalGrpcTest.java | 25 +-
.../test/ShuffleServerOnRandomPortTest.java | 48 +-
.../uniffle/test/ShuffleServerWithHadoopTest.java | 50 +-
.../ShuffleServerWithKerberizedHadoopTest.java | 62 +-
.../ShuffleServerWithLocalOfExceptionTest.java | 20 +-
.../ShuffleServerWithLocalOfLocalOrderTest.java | 44 +-
.../uniffle/test/ShuffleServerWithLocalTest.java | 43 +-
.../test/ShuffleServerWithMemLocalHadoopTest.java | 66 +-
.../uniffle/test/ShuffleServerWithMemoryTest.java | 80 +-
.../uniffle/test/ShuffleWithRssClientTest.java | 101 +-
integration-test/mr/pom.xml | 4 +-
.../apache/uniffle/test/MRIntegrationTestBase.java | 18 +-
integration-test/spark-common/pom.xml | 4 +-
.../org/apache/uniffle/test/AutoAccessTest.java | 19 +-
.../uniffle/test/DynamicFetchClientConfTest.java | 16 +-
.../org/apache/uniffle/test/FailingTasksTest.java | 20 +-
.../test/RSSStageDynamicServerReWriteTest.java | 72 +-
.../apache/uniffle/test/RSSStageResubmitTest.java | 23 +-
.../RepartitionWithHadoopHybridStorageRssTest.java | 32 +-
.../test/RepartitionWithLocalFileRssTest.java | 16 +-
.../RepartitionWithMemoryHybridStorageRssTest.java | 31 +-
.../uniffle/test/RepartitionWithMemoryRssTest.java | 28 +-
.../apache/uniffle/test/RssShuffleManagerTest.java | 14 +-
.../test/ShuffleUnregisterWithHadoopTest.java | 22 +-
.../test/ShuffleUnregisterWithLocalfileTest.java | 29 +-
.../org/apache/uniffle/test/SimpleTestBase.java | 19 +-
...thLocalForMultiPartLocalStorageManagerTest.java | 62 +-
.../uniffle/test/SparkClientWithLocalTest.java | 37 +-
.../uniffle/test/SparkIntegrationTestBase.java | 26 +-
...QLWithDelegationShuffleManagerFallbackTest.java | 18 +-
.../SparkSQLWithDelegationShuffleManagerTest.java | 29 +-
.../uniffle/test/SparkSQLWithMemoryLocalTest.java | 15 +-
.../org/apache/uniffle/test/GetReaderTest.java | 16 +-
.../apache/uniffle/test/AQERepartitionTest.java | 14 +-
.../org/apache/uniffle/test/AQESkewedJoinTest.java | 19 +-
.../uniffle/test/CompressionOverlappingTest.java} | 71 +-
.../ContinuousSelectPartitionStrategyTest.java | 51 +-
.../org/apache/uniffle/test/GetReaderTest.java | 12 +-
.../test/GetShuffleReportForMultiPartTest.java | 51 +-
.../apache/uniffle/test/MapSideCombineTest.java | 15 +-
.../test/PartitionBlockDataReassignBasicTest.java | 29 +-
.../PartitionBlockDataReassignMultiTimesTest.java | 30 +-
.../test/PartitionSplitOfLoadBalanceModeTest.java | 129 +
integration-test/tez/pom.xml | 4 +-
.../uniffle/test/TezIntegrationTestBase.java | 18 +-
.../uniffle/test/TezWordCountWithFailuresTest.java | 16 +-
.../org/apache/uniffle/client/api/ClientInfo.java | 25 +-
.../uniffle/client/api/CoordinatorClient.java | 2 +-
.../uniffle/client/api/ShuffleManagerClient.java | 14 +-
.../uniffle/client/api/ShuffleServerClient.java | 2 +-
.../client/common/ShuffleServerPushCost.java | 109 +
.../common/ShuffleServerPushCostTracker.java | 123 +
.../client/factory/CoordinatorClientFactory.java | 6 +
.../client/factory/ShuffleServerClientFactory.java | 19 +-
.../client/impl/grpc/CoordinatorGrpcClient.java | 13 +-
.../impl/grpc/CoordinatorGrpcRetryableClient.java | 24 +-
.../uniffle/client/impl/grpc/GrpcClient.java | 10 +-
.../client/impl/grpc/ShuffleManagerGrpcClient.java | 30 +-
.../client/impl/grpc/ShuffleServerGrpcClient.java | 160 +-
.../impl/grpc/ShuffleServerGrpcNettyClient.java | 74 +-
.../request/RssGetShuffleAssignmentsRequest.java | 24 -
.../client/request/RssGetShuffleDataRequest.java | 33 +-
.../RssPartitionToShuffleServerRequest.java | 16 +-
.../client/request/RssReassignServersRequest.java | 45 -
.../client/request/RssRegisterShuffleRequest.java | 11 -
.../request/RssReportShuffleReadMetricRequest.java | 151 +
.../RssReportShuffleWriteFailureRequest.java | 4 +
.../RssReportShuffleWriteMetricRequest.java | 198 +
.../client/request/RssSendShuffleDataRequest.java | 12 +-
.../response/RssReassignServersResponse.java | 43 -
...ava => RssReportShuffleReadMetricResponse.java} | 10 +-
...va => RssReportShuffleWriteMetricResponse.java} | 10 +-
pom.xml | 65 +-
proto/src/main/proto/Rss.proto | 139 +-
release/create-package.sh | 10 +-
release/publish_to_svn.sh | 4 +-
rust/experimental/server/.cargo/config.toml | 19 -
rust/experimental/server/Cargo.lock | 4315 --------------------
rust/experimental/server/Cargo.toml | 133 -
rust/experimental/server/README.md | 188 -
rust/experimental/server/build.rs | 43 -
rust/experimental/server/rust-toolchain.toml | 20 -
rust/experimental/server/rustfmt.toml | 26 -
rust/experimental/server/src/app.rs | 943 -----
rust/experimental/server/src/await_tree.rs | 51 -
rust/experimental/server/src/config.rs | 304 --
rust/experimental/server/src/error.rs | 91 -
rust/experimental/server/src/grpc.rs | 876 ----
rust/experimental/server/src/http/await_tree.rs | 49 -
rust/experimental/server/src/http/http_service.rs | 82 -
rust/experimental/server/src/http/jeprof.rs | 144 -
rust/experimental/server/src/http/metrics.rs | 72 -
rust/experimental/server/src/http/mod.rs | 52 -
rust/experimental/server/src/http/pprof.rs | 113 -
rust/experimental/server/src/lib.rs | 279 --
rust/experimental/server/src/main.rs | 313 --
.../server/src/mem_allocator/default.rs | 34 -
.../experimental/server/src/mem_allocator/error.rs | 66 -
.../server/src/mem_allocator/jemalloc.rs | 21 -
rust/experimental/server/src/mem_allocator/mod.rs | 46 -
.../server/src/mem_allocator/profiling.rs | 73 -
.../server/src/mem_allocator/system_std.rs | 21 -
rust/experimental/server/src/metric.rs | 443 --
rust/experimental/server/src/proto/mod.rs | 19 -
rust/experimental/server/src/proto/uniffle.proto | 523 ---
rust/experimental/server/src/readable_size.rs | 323 --
rust/experimental/server/src/runtime/manager.rs | 70 -
rust/experimental/server/src/runtime/metrics.rs | 54 -
rust/experimental/server/src/runtime/mod.rs | 264 --
rust/experimental/server/src/signal.rs | 58 -
rust/experimental/server/src/store/hdfs.rs | 410 --
rust/experimental/server/src/store/hybrid.rs | 922 -----
rust/experimental/server/src/store/local/disk.rs | 409 --
rust/experimental/server/src/store/local/mod.rs | 18 -
rust/experimental/server/src/store/localfile.rs | 720 ----
rust/experimental/server/src/store/mem/mod.rs | 20 -
rust/experimental/server/src/store/mem/ticket.rs | 232 --
rust/experimental/server/src/store/memory.rs | 1105 -----
rust/experimental/server/src/store/mod.rs | 192 -
rust/experimental/server/src/util.rs | 145 -
.../experimental/server/tests/graceful_shutdown.rs | 104 -
rust/experimental/server/tests/lock_benchmark.rs | 138 -
rust/experimental/server/tests/write_read.rs | 91 -
security.md | 2 +-
server/pom.xml | 1 +
.../apache/uniffle/server/HugePartitionUtils.java | 8 +-
.../apache/uniffle/server/LocalStorageChecker.java | 5 +-
.../apache/uniffle/server/RegisterHeartBeat.java | 5 +-
.../org/apache/uniffle/server/ShuffleServer.java | 7 +-
.../apache/uniffle/server/ShuffleServerConf.java | 69 +-
.../uniffle/server/ShuffleServerGrpcService.java | 85 +-
.../uniffle/server/ShuffleServerMetrics.java | 25 +-
.../org/apache/uniffle/server/ShuffleTaskInfo.java | 10 +-
.../apache/uniffle/server/ShuffleTaskManager.java | 72 +-
.../server/block/DefaultShuffleBlockIdManager.java | 12 +
.../block/PartitionedShuffleBlockIdManager.java | 11 +
.../server/block/ShuffleBlockIdManager.java | 2 +
.../uniffle/server/buffer/ShuffleBuffer.java | 2 +
.../server/buffer/ShuffleBufferManager.java | 196 +-
.../server/buffer/ShuffleBufferWithLinkedList.java | 35 +-
.../server/buffer/ShuffleBufferWithSkipList.java | 34 +-
.../apache/uniffle/server/buffer/lab/Chunk.java | 94 +
.../uniffle/server/buffer/lab/ChunkCreator.java | 330 ++
.../org/apache/uniffle/server/buffer/lab/LAB.java | 111 +
.../buffer/lab/LABShuffleBufferWithLinkedList.java | 84 +
.../buffer/lab/LABShuffleBufferWithSkipList.java | 82 +
.../buffer/lab/LABShufflePartitionedBlock.java | 27 +-
.../uniffle/server/buffer/lab/OffheapChunk.java | 23 +-
.../uniffle/server/buffer/lab/SupportsLAB.java | 7 +-
.../apache/uniffle/server/event/PurgeEvent.java | 14 +
.../uniffle/server/event/ShufflePurgeEvent.java | 7 +-
.../org/apache/uniffle/server/merge/Partition.java | 10 +-
.../org/apache/uniffle/server/merge/Shuffle.java | 20 +-
.../uniffle/server/merge/ShuffleMergeManager.java | 29 +-
.../server/netty/ShuffleServerNettyHandler.java | 2 +-
.../server/storage/HadoopStorageManager.java | 3 +-
.../server/storage/LocalStorageManager.java | 8 +-
.../server/storage/SingleStorageManager.java | 5 +-
.../server/MockedShuffleServerGrpcService.java | 13 +-
.../uniffle/server/ShuffleFlushManagerTest.java | 3 +-
.../uniffle/server/ShuffleServerConfTest.java | 4 +-
.../apache/uniffle/server/ShuffleServerTest.java | 2 +-
.../uniffle/server/ShuffleTaskManagerTest.java | 3 +-
.../apache/uniffle/server/StorageCheckerTest.java | 5 +-
.../uniffle/server/buffer/BufferTestBase.java | 2 +
.../LABShuffleBufferWithLinkedListTest.java} | 20 +-
.../buffer/LABShuffleBufferWithSkipListTest.java} | 20 +-
.../server/buffer/ShuffleBufferManagerTest.java | 38 +-
.../buffer/ShuffleBufferWithLinkedListTest.java | 187 +-
.../buffer/ShuffleBufferWithSkipListTest.java | 19 +-
.../server/merge/BlockFlushFileReaderTest.java | 35 +-
.../uniffle/server/merge/MergedResultTest.java | 20 +-
.../server/merge/ShuffleMergeManagerTest.java | 19 +-
.../server/storage/LocalStorageManagerTest.java | 77 +-
.../common/DefaultStorageMediaProvider.java | 18 +-
.../uniffle/storage/common/LocalStorageMeta.java | 21 +-
.../storage/factory/ShuffleHandlerFactory.java | 24 +-
.../uniffle/storage/handler/AsynDeletionEvent.java | 84 +
.../storage/handler/api/ShuffleDeleteHandler.java | 2 +-
.../handler/impl/AsynDeletionEventManager.java | 107 +
.../handler/impl/DataSkippableReadHandler.java | 52 +-
.../handler/impl/HadoopClientReadHandler.java | 35 +-
.../handler/impl/HadoopShuffleDeleteHandler.java | 5 +-
.../handler/impl/HadoopShuffleReadHandler.java | 19 +-
.../handler/impl/LocalFileAsyncDeleteHandler.java | 93 +
...ioWriter.java => LocalFileBlackHoleWriter.java} | 33 +-
.../handler/impl/LocalFileClientReadHandler.java | 50 +-
.../handler/impl/LocalFileDeleteHandler.java | 4 +-
.../storage/handler/impl/LocalFileNioWriter.java | 16 +-
.../handler/impl/LocalFileNullDeviceWriter.java | 19 +-
.../handler/impl/MemoryClientReadHandler.java | 35 +-
.../impl/MultiReplicaClientReadHandler.java | 9 +-
.../impl/PrefetchableClientReadHandler.java | 156 +
.../handler/impl/ShuffleServerReadCost.java | 108 +
...ment.java => ShuffleServerReadCostTracker.java} | 36 +-
.../request/CreateShuffleDeleteHandlerRequest.java | 16 +
.../request/CreateShuffleReadHandlerRequest.java | 60 +-
.../org/apache/uniffle/storage/HadoopTestBase.java | 3 +
.../common/DefaultStorageMediaProviderTest.java | 6 +-
.../handler/impl/DataSkippableReadHandlerTest.java | 37 +-
.../handler/impl/HadoopClientReadHandlerTest.java | 3 +-
.../storage/handler/impl/HadoopHandlerTest.java | 5 +-
.../handler/impl/HadoopShuffleReadHandlerTest.java | 5 +-
.../impl/LocalFileServerReadHandlerTest.java | 4 +-
.../impl/PrefetchableClientReadHandlerTest.java | 117 +
tools/client-simulation-yarn/README.md | 79 +
.../client-simulation-yarn}/pom.xml | 69 +-
.../apache/uniffle/client/simulator/Constants.java | 43 +
.../uniffle/client/simulator/HadoopConfigApp.java | 142 +
.../client/simulator/NMCallBackHandler.java | 77 +
.../simulator/UniffleClientSimOnYarnAppMaster.java | 304 ++
.../simulator/UniffleClientSimOnYarnClient.java | 221 +
.../uniffle/client/simulator/UniffleTask.java | 187 +
.../org/apache/uniffle/client/simulator/Utils.java | 94 +
437 files changed, 13345 insertions(+), 19885 deletions(-)
delete mode 100644 .github/workflows/rust.yml
copy
client-spark/{spark3/src/main/java/org/apache/spark/shuffle/RssShuffleDriverComponents.java
=>
common/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManagerUtils.java}
(53%)
copy
storage/src/main/java/org/apache/uniffle/storage/common/StorageReadMetrics.java
=>
client-spark/common/src/main/java/org/apache/spark/shuffle/events/ShuffleAssignmentInfoEvent.java
(71%)
copy
common/src/main/java/org/apache/uniffle/common/exception/RssException.java =>
client-spark/common/src/main/java/org/apache/spark/shuffle/events/ShuffleMetric.java
(68%)
create mode 100644
client-spark/common/src/main/java/org/apache/spark/shuffle/events/ShuffleReadMetric.java
create mode 100644
client-spark/common/src/main/java/org/apache/spark/shuffle/events/ShuffleWriteMetric.java
create mode 100644
client-spark/common/src/main/java/org/apache/spark/shuffle/events/ShuffleWriteTimes.java
create mode 100644
client-spark/common/src/main/java/org/apache/spark/shuffle/events/TaskReassignInfoEvent.java
create mode 100644
client-spark/common/src/main/java/org/apache/spark/shuffle/events/TaskShuffleReadInfoEvent.java
create mode 100644
client-spark/common/src/main/java/org/apache/spark/shuffle/events/TaskShuffleWriteInfoEvent.java
copy
cli/src/main/java/org/apache/uniffle/client/exception/UniffleRestException.java
=>
client-spark/common/src/main/java/org/apache/spark/shuffle/events/UniffleEvent.java
(79%)
copy
client/src/main/java/org/apache/uniffle/client/impl/TrackingPartitionStatus.java
=>
client-spark/common/src/main/java/org/apache/spark/shuffle/handle/split/PartitionSplitInfo.java
(54%)
create mode 100644
client-spark/common/src/main/java/org/apache/spark/shuffle/writer/OverlappingCompressionDataPusher.java
create mode 100644
client-spark/common/src/main/java/org/apache/uniffle/shuffle/ShuffleIdMappingManager.java
create mode 100644
client-spark/common/src/test/java/org/apache/spark/shuffle/writer/OverlappingCompressionDataPusherTest.java
copy client-spark/{spark3 => extension}/pom.xml (60%)
create mode 100644
client-spark/extension/src/main/resources/META-INF/services/org.apache.spark.status.AppHistoryServerPlugin
copy
coordinator/src/main/java/org/apache/uniffle/coordinator/ClusterManagerFactory.java
=>
client-spark/extension/src/main/scala/org/apache/spark/UniffleHistoryServerPlugin.scala
(60%)
create mode 100644
client-spark/extension/src/main/scala/org/apache/spark/UniffleListener.scala
create mode 100644
client-spark/extension/src/main/scala/org/apache/spark/UnifflePlugin.scala
create mode 100644
client-spark/extension/src/main/scala/org/apache/spark/UniffleStatusStore.scala
create mode 100644
client-spark/extension/src/main/scala/org/apache/spark/ui/ShufflePage.scala
copy client/src/main/java/org/apache/uniffle/client/api/ShuffleReadClient.java
=> client-spark/extension/src/main/scala/org/apache/spark/ui/ShuffleTab.scala
(66%)
create mode 100644
client-spark/spark3/src/main/java/org/apache/spark/shuffle/writer/PartitionLengthStatistic.java
create mode 100644
client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/PartitionLengthStatisticTest.java
create mode 100644
client/src/main/java/org/apache/uniffle/client/impl/DecompressionWorker.java
copy
client/src/main/java/org/apache/uniffle/client/response/{CompressedShuffleBlock.java
=> DecompressedShuffleBlock.java} (65%)
copy
server/src/main/java/org/apache/uniffle/server/merge/MergeEventHandler.java =>
client/src/main/java/org/apache/uniffle/client/response/ShuffleBlock.java (83%)
create mode 100644
client/src/test/java/org/apache/uniffle/client/impl/DecompressionWorkerTest.java
create mode 100644
common/src/main/java/org/apache/uniffle/common/DeferredCompressedBlock.java
copy common/src/main/java/org/apache/uniffle/common/{Arguments.java =>
PartitionSplitMode.java} (77%)
copy common/src/main/java/org/apache/uniffle/common/{ShuffleDataSegment.java
=> ReadSegment.java} (59%)
create mode 100644
common/src/main/java/org/apache/uniffle/common/ShuffleReadTimes.java
create mode 100644
common/src/main/java/org/apache/uniffle/common/compression/StatisticsCodec.java
copy
common/src/main/java/org/apache/uniffle/common/netty/protocol/{GetLocalShuffleDataV2Request.java
=> GetLocalShuffleDataV3Request.java} (61%)
create mode 100644
common/src/main/java/org/apache/uniffle/common/serializer/kryo/KryoDeserializationStream.java
create mode 100644
common/src/main/java/org/apache/uniffle/common/serializer/kryo/KryoSerializationStream.java
copy
common/src/main/java/org/apache/uniffle/common/serializer/{writable/WritableSerializer.java
=> kryo/KryoSerializer.java} (77%)
create mode 100644
common/src/main/java/org/apache/uniffle/common/serializer/kryo/KryoSerializerInstance.java
create mode 100644
common/src/main/java/org/apache/uniffle/common/serializer/kryo/PoolWrapper.java
copy common/src/test/java/org/apache/uniffle/common/{UnionKeyTest.java =>
DeferredCompressedBlockTest.java} (55%)
create mode 100644
common/src/test/java/org/apache/uniffle/common/netty/buffer/FileSegmentManagedBufferTest.java
delete mode 100644
common/src/test/java/org/apache/uniffle/common/port/PortRegistry.java
create mode 100644
common/src/test/java/org/apache/uniffle/common/serializer/KryoSerializerTest.java
delete mode 100644
dashboard/src/main/webapp/src/pages/serverstatus/ExcludeNodeList.vue
mode change 100644 => 100755 dev/scripts/checkshade.sh
copy
integration-test/{spark-common/src/test/java/org/apache/uniffle/test/SparkSQLWithMemoryLocalTest.java
=>
spark3/src/test/java/org/apache/uniffle/test/CompressionOverlappingTest.java}
(51%)
create mode 100644
integration-test/spark3/src/test/java/org/apache/uniffle/test/PartitionSplitOfLoadBalanceModeTest.java
copy
client/src/main/java/org/apache/uniffle/client/impl/TrackingPartitionStatus.java
=> internal-client/src/main/java/org/apache/uniffle/client/api/ClientInfo.java
(67%)
create mode 100644
internal-client/src/main/java/org/apache/uniffle/client/common/ShuffleServerPushCost.java
create mode 100644
internal-client/src/main/java/org/apache/uniffle/client/common/ShuffleServerPushCostTracker.java
delete mode 100644
internal-client/src/main/java/org/apache/uniffle/client/request/RssReassignServersRequest.java
create mode 100644
internal-client/src/main/java/org/apache/uniffle/client/request/RssReportShuffleReadMetricRequest.java
create mode 100644
internal-client/src/main/java/org/apache/uniffle/client/request/RssReportShuffleWriteMetricRequest.java
delete mode 100644
internal-client/src/main/java/org/apache/uniffle/client/response/RssReassignServersResponse.java
copy
internal-client/src/main/java/org/apache/uniffle/client/response/{RssReportShuffleResultResponse.java
=> RssReportShuffleReadMetricResponse.java} (72%)
copy
internal-client/src/main/java/org/apache/uniffle/client/response/{RssReportShuffleResultResponse.java
=> RssReportShuffleWriteMetricResponse.java} (72%)
delete mode 100644 rust/experimental/server/.cargo/config.toml
delete mode 100644 rust/experimental/server/Cargo.lock
delete mode 100644 rust/experimental/server/Cargo.toml
delete mode 100644 rust/experimental/server/README.md
delete mode 100644 rust/experimental/server/build.rs
delete mode 100644 rust/experimental/server/rust-toolchain.toml
delete mode 100644 rust/experimental/server/rustfmt.toml
delete mode 100644 rust/experimental/server/src/app.rs
delete mode 100644 rust/experimental/server/src/await_tree.rs
delete mode 100644 rust/experimental/server/src/config.rs
delete mode 100644 rust/experimental/server/src/error.rs
delete mode 100644 rust/experimental/server/src/grpc.rs
delete mode 100644 rust/experimental/server/src/http/await_tree.rs
delete mode 100644 rust/experimental/server/src/http/http_service.rs
delete mode 100644 rust/experimental/server/src/http/jeprof.rs
delete mode 100644 rust/experimental/server/src/http/metrics.rs
delete mode 100644 rust/experimental/server/src/http/mod.rs
delete mode 100644 rust/experimental/server/src/http/pprof.rs
delete mode 100644 rust/experimental/server/src/lib.rs
delete mode 100644 rust/experimental/server/src/main.rs
delete mode 100644 rust/experimental/server/src/mem_allocator/default.rs
delete mode 100644 rust/experimental/server/src/mem_allocator/error.rs
delete mode 100644 rust/experimental/server/src/mem_allocator/jemalloc.rs
delete mode 100644 rust/experimental/server/src/mem_allocator/mod.rs
delete mode 100644 rust/experimental/server/src/mem_allocator/profiling.rs
delete mode 100644 rust/experimental/server/src/mem_allocator/system_std.rs
delete mode 100644 rust/experimental/server/src/metric.rs
delete mode 100644 rust/experimental/server/src/proto/mod.rs
delete mode 100644 rust/experimental/server/src/proto/uniffle.proto
delete mode 100644 rust/experimental/server/src/readable_size.rs
delete mode 100644 rust/experimental/server/src/runtime/manager.rs
delete mode 100644 rust/experimental/server/src/runtime/metrics.rs
delete mode 100644 rust/experimental/server/src/runtime/mod.rs
delete mode 100644 rust/experimental/server/src/signal.rs
delete mode 100644 rust/experimental/server/src/store/hdfs.rs
delete mode 100644 rust/experimental/server/src/store/hybrid.rs
delete mode 100644 rust/experimental/server/src/store/local/disk.rs
delete mode 100644 rust/experimental/server/src/store/local/mod.rs
delete mode 100644 rust/experimental/server/src/store/localfile.rs
delete mode 100644 rust/experimental/server/src/store/mem/mod.rs
delete mode 100644 rust/experimental/server/src/store/mem/ticket.rs
delete mode 100644 rust/experimental/server/src/store/memory.rs
delete mode 100644 rust/experimental/server/src/store/mod.rs
delete mode 100644 rust/experimental/server/src/util.rs
delete mode 100644 rust/experimental/server/tests/graceful_shutdown.rs
delete mode 100644 rust/experimental/server/tests/lock_benchmark.rs
delete mode 100644 rust/experimental/server/tests/write_read.rs
create mode 100644
server/src/main/java/org/apache/uniffle/server/buffer/lab/Chunk.java
create mode 100644
server/src/main/java/org/apache/uniffle/server/buffer/lab/ChunkCreator.java
create mode 100644
server/src/main/java/org/apache/uniffle/server/buffer/lab/LAB.java
create mode 100644
server/src/main/java/org/apache/uniffle/server/buffer/lab/LABShuffleBufferWithLinkedList.java
create mode 100644
server/src/main/java/org/apache/uniffle/server/buffer/lab/LABShuffleBufferWithSkipList.java
copy
storage/src/main/java/org/apache/uniffle/storage/handler/api/ShuffleWriteHandler.java
=>
server/src/main/java/org/apache/uniffle/server/buffer/lab/LABShufflePartitionedBlock.java
(64%)
copy
common/src/main/java/org/apache/uniffle/common/rpc/MonitoringServerCallListener.java
=> server/src/main/java/org/apache/uniffle/server/buffer/lab/OffheapChunk.java
(64%)
copy common/src/main/java/org/apache/uniffle/common/netty/IOMode.java =>
server/src/main/java/org/apache/uniffle/server/buffer/lab/SupportsLAB.java (91%)
copy
server/src/{main/java/org/apache/uniffle/server/buffer/RequireBufferStatusCode.java
=>
test/java/org/apache/uniffle/server/buffer/LABShuffleBufferWithLinkedListTest.java}
(64%)
copy
server/src/{main/java/org/apache/uniffle/server/buffer/RequireBufferStatusCode.java
=>
test/java/org/apache/uniffle/server/buffer/LABShuffleBufferWithSkipListTest.java}
(64%)
create mode 100644
storage/src/main/java/org/apache/uniffle/storage/handler/AsynDeletionEvent.java
create mode 100644
storage/src/main/java/org/apache/uniffle/storage/handler/impl/AsynDeletionEventManager.java
create mode 100644
storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileAsyncDeleteHandler.java
copy
storage/src/main/java/org/apache/uniffle/storage/handler/impl/{LocalFileNioWriter.java
=> LocalFileBlackHoleWriter.java} (60%)
copy server/src/main/java/org/apache/uniffle/server/Checker.java =>
storage/src/main/java/org/apache/uniffle/storage/handler/impl/LocalFileNullDeviceWriter.java
(61%)
create mode 100644
storage/src/main/java/org/apache/uniffle/storage/handler/impl/PrefetchableClientReadHandler.java
create mode 100644
storage/src/main/java/org/apache/uniffle/storage/handler/impl/ShuffleServerReadCost.java
copy
storage/src/main/java/org/apache/uniffle/storage/handler/impl/{DataFileSegment.java
=> ShuffleServerReadCostTracker.java} (51%)
copy
client-spark/spark3/src/test/java/org/apache/spark/shuffle/FunctionUtilsTests.java
=>
storage/src/test/java/org/apache/uniffle/storage/handler/impl/DataSkippableReadHandlerTest.java
(52%)
create mode 100644
storage/src/test/java/org/apache/uniffle/storage/handler/impl/PrefetchableClientReadHandlerTest.java
create mode 100644 tools/client-simulation-yarn/README.md
copy {client-mr/hadoop3.2 => tools/client-simulation-yarn}/pom.xml (56%)
create mode 100644
tools/client-simulation-yarn/src/main/java/org/apache/uniffle/client/simulator/Constants.java
create mode 100644
tools/client-simulation-yarn/src/main/java/org/apache/uniffle/client/simulator/HadoopConfigApp.java
create mode 100644
tools/client-simulation-yarn/src/main/java/org/apache/uniffle/client/simulator/NMCallBackHandler.java
create mode 100644
tools/client-simulation-yarn/src/main/java/org/apache/uniffle/client/simulator/UniffleClientSimOnYarnAppMaster.java
create mode 100644
tools/client-simulation-yarn/src/main/java/org/apache/uniffle/client/simulator/UniffleClientSimOnYarnClient.java
create mode 100644
tools/client-simulation-yarn/src/main/java/org/apache/uniffle/client/simulator/UniffleTask.java
create mode 100644
tools/client-simulation-yarn/src/main/java/org/apache/uniffle/client/simulator/Utils.java
