smengcl commented on code in PR #5104:
URL: https://github.com/apache/ozone/pull/5104#discussion_r1280513602
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java:
##########
@@ -249,31 +356,60 @@ private void waitForDirToExist(Path dir) throws
IOException {
}
}
- private boolean processDir(Path dir, Set<Path> copyFiles,
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private boolean processDir(Path dir, Map<Path, Path> copyFiles,
Map<Path, Path> hardLinkFiles,
Set<Path> toExcludeFiles,
Set<Path> snapshotPaths,
List<String> excluded,
- AtomicLong copySize)
+ AtomicLong copySize, Path destDir)
Review Comment:
nit
```suggestion
AtomicLong copySize,
Path destDir)
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java:
##########
@@ -177,23 +191,95 @@ public static Set<Path> normalizeExcludeList(List<String>
toExcludeList,
return paths;
}
+ /**
+ * Pauses rocksdb compaction threads while creating copies of
+ * compaction logs and hard links of sst backups.
+ * @param tmpdir - Place to create copies/links
+ * @param flush - Whether to flush the db or not.
+ * @return Checkpoint containing snapshot entries expected.
+ */
+ @Override
+ public DBCheckpoint getCheckpoint(Path tmpdir, boolean flush)
+ throws IOException {
+ DBCheckpoint checkpoint;
+
+ // make tmp directories to contain the copies
+ RocksDBCheckpointDiffer differ = getDbStore().getRocksDBCheckpointDiffer();
+ DirectoryData sstBackupDir = new DirectoryData(tmpdir,
+ differ.getSSTBackupDir());
+ DirectoryData compactionLogDir = new DirectoryData(tmpdir,
+ differ.getCompactionLogDir());
+
+ long startTime = System.currentTimeMillis();
+ long pauseCounter = PAUSE_COUNTER.incrementAndGet();
+
+ // Pause compactions, Copy/link files and get checkpoint.
+ try {
+ LOG.info("Compaction pausing {} started.", pauseCounter);
+ differ.incrementTarballRequestCount();
+ FileUtils.copyDirectory(compactionLogDir.getOriginalDir(),
+ compactionLogDir.getTmpDir());
+ OmSnapshotUtils.linkFiles(sstBackupDir.getOriginalDir(),
+ sstBackupDir.getTmpDir());
+ checkpoint = getDbStore().getCheckpoint(flush);
+ } finally {
+ // Unpause the compaction threads.
+ synchronized (getDbStore().getRocksDBCheckpointDiffer()) {
+ differ.decrementTarballRequestCount();
+ differ.notifyAll();
+ long elapsedTime = System.currentTimeMillis() - startTime;
+ LOG.info("Compaction pausing {} ended. Elapsed ms: {}",
+ pauseCounter, elapsedTime);
+ }
+ }
+ return checkpoint;
+ }
+
+
+ // Convenience class for keeping track of the tmp dirs.
+ private static class DirectoryData {
+ private final File originalDir;
+ private final File tmpDir;
+ DirectoryData(Path tmpdir, String dirStr) throws IOException {
+ originalDir = new File(dirStr);
+ tmpDir = new File(tmpdir.toString(), getOriginalDir().getName());
+ if (!tmpDir.exists() && !tmpDir.mkdirs()) {
+ throw new IOException("mkdirs failed: " + tmpDir);
+ }
+ }
+
+ public File getOriginalDir() {
+ return originalDir;
+ }
+
+ public File getTmpDir() {
+ return tmpDir;
+ }
+ }
+
private boolean getFilesForArchive(DBCheckpoint checkpoint,
- Set<Path> copyFiles,
+ Map<Path, Path> copyFiles,
Map<Path, Path> hardLinkFiles,
Set<Path> toExcludeFiles,
boolean includeSnapshotData,
- List<String> excluded)
+ List<String> excluded, Path tmpdir)
Review Comment:
nit
```suggestion
List<String> excluded,
Path tmpdir)
```
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java:
##########
@@ -297,13 +433,27 @@ private boolean processDir(Path dir, Set<Path> copyFiles,
* @param excluded The list of db files that actually were excluded.
*/
@VisibleForTesting
- public static long processFile(Path file, Set<Path> copyFiles,
+ public static long processFile(Path file, Map<Path, Path> copyFiles,
Map<Path, Path> hardLinkFiles,
Set<Path> toExcludeFiles,
- List<String> excluded) throws IOException {
+ List<String> excluded, Path destDir)
Review Comment:
nit
```suggestion
List<String> excluded,
Path destDir)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
