zoltar9264 commented on code in PR #22669:
URL: https://github.com/apache/flink/pull/22669#discussion_r1246033453
##########
flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java:
##########
@@ -334,53 +325,61 @@ public SnapshotResult<KeyedStateHandle>
get(CloseableRegistry snapshotCloseableR
}
}
- private void uploadSstFiles(
- @Nonnull Map<StateHandleID, StreamStateHandle> sstFiles,
- @Nonnull Map<StateHandleID, StreamStateHandle> miscFiles,
+ /** upload files and return total uploaded size. */
+ private long uploadSnapshotFiles(
+ @Nonnull List<HandleAndLocalPath> sstFiles,
+ @Nonnull List<HandleAndLocalPath> miscFiles,
@Nonnull CloseableRegistry snapshotCloseableRegistry,
@Nonnull CloseableRegistry tmpResourcesRegistry)
throws Exception {
// write state data
Preconditions.checkState(localBackupDirectory.exists());
- Map<StateHandleID, Path> sstFilePaths = new HashMap<>();
- Map<StateHandleID, Path> miscFilePaths = new HashMap<>();
-
Path[] files = localBackupDirectory.listDirectory();
+ long uploadedSize = 0;
if (files != null) {
+ List<Path> sstFilePaths = new ArrayList<>(files.length);
+ List<Path> miscFilePaths = new ArrayList<>(files.length);
+
createUploadFilePaths(files, sstFiles, sstFilePaths,
miscFilePaths);
final CheckpointedStateScope stateScope =
sharingFilesStrategy ==
SnapshotType.SharingFilesStrategy.NO_SHARING
? CheckpointedStateScope.EXCLUSIVE
: CheckpointedStateScope.SHARED;
- sstFiles.putAll(
+
+ List<HandleAndLocalPath> uploadedSstFiles =
stateUploader.uploadFilesToCheckpointFs(
sstFilePaths,
checkpointStreamFactory,
stateScope,
snapshotCloseableRegistry,
- tmpResourcesRegistry));
- miscFiles.putAll(
+ tmpResourcesRegistry);
+ uploadedSize +=
+ uploadedSstFiles.stream()
+ .mapToLong(e -> e.getHandle().getStateSize())
Review Comment:
Good suggestion, done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
