dlmarion commented on code in PR #5026:
URL: https://github.com/apache/accumulo/pull/5026#discussion_r1834986069
##########
core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java:
##########
@@ -19,42 +19,135 @@
package org.apache.accumulo.core.util.compaction;
import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import org.apache.accumulo.core.clientImpl.Namespace;
+import org.apache.accumulo.core.data.NamespaceId;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.lang3.Range;
+
+import com.google.common.base.Preconditions;
public class CompactionJobPrioritizer {
public static final Comparator<CompactionJob> JOB_COMPARATOR =
Comparator.comparingInt(CompactionJob::getPriority)
.thenComparingInt(job -> job.getFiles().size()).reversed();
- public static short createPriority(CompactionKind kind, int totalFiles, int
compactingFiles) {
-
- int prio = totalFiles + compactingFiles;
-
- switch (kind) {
- case USER:
- case CHOP:
- // user-initiated compactions will have a positive priority
- // based on number of files
- if (prio > Short.MAX_VALUE) {
- return Short.MAX_VALUE;
- }
- return (short) prio;
- case SELECTOR:
- case SYSTEM:
- // system-initiated compactions will have a negative priority
- // starting at -32768 and increasing based on number of files
- // maxing out at -1
- if (prio > Short.MAX_VALUE) {
- return -1;
- } else {
- return (short) (Short.MIN_VALUE + prio);
- }
- default:
- throw new AssertionError("Unknown kind " + kind);
+ private static final Map<Pair<TableId,CompactionKind>,Range<Short>>
SYSTEM_TABLE_RANGES =
+ new HashMap<>();
+ private static final Map<Pair<NamespaceId,CompactionKind>,
+ Range<Short>> ACCUMULO_NAMESPACE_RANGES = new HashMap<>();
+
+ // Create ranges of possible priority values where each range has
+ // 2000 possible values. Priority order is:
+ // root table user initiated
+ // root table system initiated
+ // metadata table user initiated
+ // metadata table system initiated
+ // other tables in accumulo namespace user initiated
+ // other tables in accumulo namespace system initiated
+ // user tables that have more files that configured system initiated
+ // user tables user initiated
+ // user tables system initiated
+ static final Range<Short> ROOT_TABLE_USER = Range.of((short) 30768, (short)
32767);
+ static final Range<Short> ROOT_TABLE_SYSTEM = Range.of((short) 28768,
(short) 30767);
+
+ static final Range<Short> METADATA_TABLE_USER = Range.of((short) 26768,
(short) 28767);
+ static final Range<Short> METADATA_TABLE_SYSTEM = Range.of((short) 24768,
(short) 26767);
+
+ static final Range<Short> SYSTEM_NS_USER = Range.of((short) 22768, (short)
24767);
+ static final Range<Short> SYSTEM_NS_SYSTEM = Range.of((short) 20768, (short)
22767);
+
+ static final Range<Short> TABLE_OVER_SIZE = Range.of((short) 18768, (short)
20767);
+
+ static final Range<Short> USER_TABLE_USER = Range.of((short) 1, (short)
18767);
+ static final Range<Short> USER_TABLE_SYSTEM = Range.of((short) -32768,
(short) 0);
+
+ static {
+ // root table
+ SYSTEM_TABLE_RANGES.put(new Pair<>(RootTable.ID, CompactionKind.USER),
ROOT_TABLE_USER);
+ SYSTEM_TABLE_RANGES.put(new Pair<>(RootTable.ID, CompactionKind.SYSTEM),
ROOT_TABLE_SYSTEM);
+
+ // metadata table
+ SYSTEM_TABLE_RANGES.put(new Pair<>(MetadataTable.ID, CompactionKind.USER),
METADATA_TABLE_USER);
+ SYSTEM_TABLE_RANGES.put(new Pair<>(MetadataTable.ID,
CompactionKind.SYSTEM),
+ METADATA_TABLE_SYSTEM);
+
+ // metadata table
+ ACCUMULO_NAMESPACE_RANGES.put(new Pair<>(Namespace.ACCUMULO.id(),
CompactionKind.USER),
+ SYSTEM_NS_USER);
+ ACCUMULO_NAMESPACE_RANGES.put(new Pair<>(Namespace.ACCUMULO.id(),
CompactionKind.SYSTEM),
+ SYSTEM_NS_SYSTEM);
+ }
+
+ public static short createPriority(final NamespaceId nsId, final TableId
tableId,
+ final CompactionKind kind, final int totalFiles, final int
compactingFiles,
+ final int maxFilesPerTablet) {
+
+ Objects.requireNonNull(nsId, "nsId cannot be null");
+ Objects.requireNonNull(tableId, "tableId cannot be null");
+ Preconditions.checkArgument(totalFiles >= 0, "totalFiles is negative %s",
totalFiles);
+ Preconditions.checkArgument(compactingFiles >= 0, "compactingFiles is
negative %s",
+ compactingFiles);
+
+ final Function<Range<Short>,Short> normalPriorityFunction = new
Function<>() {
+ @Override
+ public Short apply(Range<Short> f) {
+ return (short) Math.min(f.getMaximum(), f.getMinimum() + totalFiles +
compactingFiles);
+ }
+ };
+
+ final Function<Range<Short>,Short> tabletOverSizeFunction = new
Function<>() {
+ @Override
+ public Short apply(Range<Short> f) {
+ return (short) Math.min(f.getMaximum(),
+ f.getMinimum() + compactingFiles + (totalFiles -
maxFilesPerTablet));
+ }
+ };
+
+ // Handle the case of a CHOP compaction. For the purposes of determining
+ // a priority, treat them as a USER compaction.
+ CompactionKind calculationKind = kind;
+ if (kind == CompactionKind.CHOP) {
Review Comment:
Implemented in 4c24262
##########
core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java:
##########
@@ -62,73 +66,138 @@ public CompactionJob createJob(CompactionKind kind, String
tablet, int numFiles,
CompactionExecutorIdImpl.externalId("test"), files, kind,
Optional.of(false));
}
+ @Test
+ public void testNonOverlappingRanges() {
+ List<Range<Short>> ranges = new ArrayList<>();
+ ranges.add(ROOT_TABLE_USER);
+ ranges.add(ROOT_TABLE_SYSTEM);
+ ranges.add(METADATA_TABLE_USER);
+ ranges.add(METADATA_TABLE_SYSTEM);
+ ranges.add(SYSTEM_NS_USER);
+ ranges.add(SYSTEM_NS_SYSTEM);
+ ranges.add(TABLE_OVER_SIZE);
+ ranges.add(USER_TABLE_USER);
+ ranges.add(USER_TABLE_SYSTEM);
+
+ for (Range<Short> r1 : ranges) {
+ for (Range<Short> r2 : ranges) {
+ if (r1 == r2) {
+ continue;
+ }
+ assertFalse(r1.isOverlappedBy(r2), r1.toString() + " is overlapped by
" + r2.toString());
+ }
+ }
+
+ Collections.sort(ranges, new Comparator<Range<Short>>() {
+ @Override
+ public int compare(Range<Short> r1, Range<Short> r2) {
+ return Short.compare(r1.getMinimum(), r2.getMinimum());
+ }
+ });
+ // check that the max of the previous range is one less than the
+ // minimum of the current range to make sure there are no holes.
+ short lastMax = Short.MIN_VALUE;
+ for (Range<Short> r : ranges) {
+ if (lastMax != Short.MIN_VALUE) {
+ assertTrue(r.getMinimum() - lastMax == 1);
+ }
+ lastMax = r.getMaximum();
+ }
Review Comment:
Implemented in 4c24262
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
