This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1a42c54cff Partial revert changes from #1519 (#2984)
1a42c54cff is described below
commit 1a42c54cff170037b244969ff2a40797ea50ce1e
Author: Mike Miller <mmil...@apache.org>
AuthorDate: Thu Sep 29 16:46:50 2022 +0000
Partial revert changes from #1519 (#2984)
* Revert some changes to FileUtil that used TabletFile. If a tablet is
splitting and exceeds the tserver.tablet.split.midpoint.files.max then
we will reduce the number of index files using temporary files. These
temporary files don't conform to standard TabletFile directories so just
use strings in this special case.
* Closes #2977
---
.../org/apache/accumulo/server/util/FileUtil.java | 58 +++++++++++++---------
.../org/apache/accumulo/tserver/tablet/Tablet.java | 5 +-
2 files changed, 37 insertions(+), 26 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index 3e6cebb523..a945655d90 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.stream.Collectors;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
@@ -108,11 +109,11 @@ public class FileUtil {
return result;
}
- public static Collection<TabletFile> reduceFiles(ServerContext context,
- TableConfiguration tableConf, Text prevEndRow, Text endRow,
Collection<TabletFile> mapFiles,
- int maxFiles, Path tmpDir, int pass) throws IOException {
+ public static Collection<String> reduceFiles(ServerContext context,
TableConfiguration tableConf,
+ Text prevEndRow, Text endRow, Collection<String> mapFiles, int maxFiles,
Path tmpDir,
+ int pass) throws IOException {
- ArrayList<TabletFile> paths = new ArrayList<>(mapFiles);
+ ArrayList<String> paths = new ArrayList<>(mapFiles);
if (paths.size() <= maxFiles)
return paths;
@@ -121,33 +122,33 @@ public class FileUtil {
int start = 0;
- ArrayList<TabletFile> outFiles = new ArrayList<>();
+ ArrayList<String> outFiles = new ArrayList<>();
int count = 0;
while (start < paths.size()) {
int end = Math.min(maxFiles + start, paths.size());
- List<TabletFile> inFiles = paths.subList(start, end);
+ List<String> inFiles = paths.subList(start, end);
start = end;
- TabletFile newMapFile =
- new TabletFile(new Path(String.format("%s/%04d.%s", newDir, count++,
RFile.EXTENSION)));
+ // temporary tablet file does not conform to typical path verified in
TabletFile
+ String newMapFile = String.format("%s/%04d.%s", newDir, count++,
RFile.EXTENSION);
outFiles.add(newMapFile);
- FileSystem ns =
context.getVolumeManager().getFileSystemByPath(newMapFile.getPath());
+ FileSystem ns = context.getVolumeManager().getFileSystemByPath(new
Path(newMapFile));
FileSKVWriter writer = new RFileOperations().newWriterBuilder()
- .forFile(newMapFile.getPathStr(), ns, ns.getConf(),
tableConf.getCryptoService())
+ .forFile(newMapFile, ns, ns.getConf(), tableConf.getCryptoService())
.withTableConfiguration(tableConf).build();
writer.startDefaultLocalityGroup();
List<SortedKeyValueIterator<Key,Value>> iters = new
ArrayList<>(inFiles.size());
FileSKVIterator reader = null;
try {
- for (TabletFile file : inFiles) {
- ns = context.getVolumeManager().getFileSystemByPath(file.getPath());
+ for (String file : inFiles) {
+ ns = context.getVolumeManager().getFileSystemByPath(new Path(file));
reader = FileOperations.getInstance().newIndexReaderBuilder()
- .forFile(file.getPathStr(), ns, ns.getConf(),
tableConf.getCryptoService())
+ .forFile(file, ns, ns.getConf(), tableConf.getCryptoService())
.withTableConfiguration(tableConf).build();
iters.add(reader);
}
@@ -199,8 +200,8 @@ public class FileUtil {
}
public static double estimatePercentageLTE(ServerContext context,
TableConfiguration tableConf,
- String tabletDir, Text prevEndRow, Text endRow, Collection<TabletFile>
mapFiles,
- Text splitRow) throws IOException {
+ String tabletDir, Text prevEndRow, Text endRow, Collection<String>
mapFiles, Text splitRow)
+ throws IOException {
Path tmpDir = null;
@@ -277,9 +278,9 @@ public class FileUtil {
*/
public static SortedMap<Double,Key> findMidPoint(ServerContext context,
TableConfiguration tableConf, String tabletDirectory, Text prevEndRow,
Text endRow,
- Collection<TabletFile> mapFiles, double minSplit, boolean useIndex)
throws IOException {
+ Collection<String> mapFiles, double minSplit, boolean useIndex) throws
IOException {
- Collection<TabletFile> origMapFiles = mapFiles;
+ Collection<String> origMapFiles = mapFiles;
Path tmpDir = null;
@@ -411,22 +412,23 @@ public class FileUtil {
}
private static long countIndexEntries(ServerContext context,
TableConfiguration tableConf,
- Text prevEndRow, Text endRow, Collection<TabletFile> mapFiles, boolean
useIndex,
+ Text prevEndRow, Text endRow, Collection<String> mapFiles, boolean
useIndex,
ArrayList<FileSKVIterator> readers) throws IOException {
long numKeys = 0;
// count the total number of index entries
- for (TabletFile file : mapFiles) {
+ for (String file : mapFiles) {
FileSKVIterator reader = null;
- FileSystem ns =
context.getVolumeManager().getFileSystemByPath(file.getPath());
+ Path path = new Path(file);
+ FileSystem ns = context.getVolumeManager().getFileSystemByPath(path);
try {
if (useIndex)
reader = FileOperations.getInstance().newIndexReaderBuilder()
- .forFile(file.getPathStr(), ns, ns.getConf(),
tableConf.getCryptoService())
+ .forFile(path.toString(), ns, ns.getConf(),
tableConf.getCryptoService())
.withTableConfiguration(tableConf).build();
else
reader = FileOperations.getInstance().newScanReaderBuilder()
- .forFile(file.getPathStr(), ns, ns.getConf(),
tableConf.getCryptoService())
+ .forFile(path.toString(), ns, ns.getConf(),
tableConf.getCryptoService())
.withTableConfiguration(tableConf)
.overRange(new Range(prevEndRow, false, null, true), Set.of(),
false).build();
@@ -450,11 +452,11 @@ public class FileUtil {
if (useIndex)
readers.add(FileOperations.getInstance().newIndexReaderBuilder()
- .forFile(file.getPathStr(), ns, ns.getConf(),
tableConf.getCryptoService())
+ .forFile(path.toString(), ns, ns.getConf(),
tableConf.getCryptoService())
.withTableConfiguration(tableConf).build());
else
readers.add(FileOperations.getInstance().newScanReaderBuilder()
- .forFile(file.getPathStr(), ns, ns.getConf(),
tableConf.getCryptoService())
+ .forFile(path.toString(), ns, ns.getConf(),
tableConf.getCryptoService())
.withTableConfiguration(tableConf)
.overRange(new Range(prevEndRow, false, null, true), Set.of(),
false).build());
@@ -538,4 +540,12 @@ public class FileUtil {
return lastKey;
}
+
+ /**
+ * Convert TabletFiles to Strings in case we need to reduce number of files.
The temporary files
+ * used will have irregular paths that don't conform to TabletFile
verification.
+ */
+ public static Collection<String> toPathStrings(Collection<TabletFile> files)
{
+ return
files.stream().map(TabletFile::getPathStr).collect(Collectors.toList());
+ }
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index decebdfe00..e02c6be82c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1205,7 +1205,7 @@ public class Tablet extends TabletBase {
try {
// we should make .25 below configurable
keys = FileUtil.findMidPoint(context, tableConfiguration,
chooseTabletDir(),
- extent.prevEndRow(), extent.endRow(), files, .25, true);
+ extent.prevEndRow(), extent.endRow(), FileUtil.toPathStrings(files),
.25, true);
} catch (IOException e) {
log.error("Failed to find midpoint {}", e.getMessage());
return null;
@@ -1450,8 +1450,9 @@ public class Tablet extends TabletBase {
splitPoint = findSplitRow(getDatafileManager().getFiles());
} else {
Text tsp = new Text(sp);
+ var fileStrings =
FileUtil.toPathStrings(getDatafileManager().getFiles());
var ratio = FileUtil.estimatePercentageLTE(context,
tableConfiguration, chooseTabletDir(),
- extent.prevEndRow(), extent.endRow(),
getDatafileManager().getFiles(), tsp);
+ extent.prevEndRow(), extent.endRow(), fileStrings, tsp);
splitPoint = new SplitRowSpec(ratio, tsp);
}
