DomGarguilo commented on a change in pull request #2345: URL: https://github.com/apache/accumulo/pull/2345#discussion_r775626552
########## File path: core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionServicesConfig.java ########## @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.util.compaction; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; + +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.spi.compaction.CompactionServiceId; +import org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner; + +import com.google.common.collect.Sets; + +public class CompactionServicesConfig { + + private final Map<String,String> planners = new HashMap<>(); + private final Map<String,Long> rateLimits = new HashMap<>(); + private final Map<String,Map<String,String>> options = new HashMap<>(); + long defaultRateLimit; + private final Consumer<String> deprecationWarningConsumer; + + public static final CompactionServiceId DEFAULT_SERVICE = CompactionServiceId.of("default"); + + @SuppressWarnings("removal") + private long getDefaultThroughput(AccumuloConfiguration aconf) { + if (aconf.isPropertySet(Property.TSERV_MAJC_THROUGHPUT, true)) { + return aconf.getAsBytes(Property.TSERV_MAJC_THROUGHPUT); + } + + return ConfigurationTypeHelper + .getMemoryAsBytes(Property.TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT.getDefaultValue()); + } + + @SuppressWarnings("removal") + private Map<String,String> getConfiguration(AccumuloConfiguration aconf) { + + Map<String,String> configs = + aconf.getAllPropertiesWithPrefix(Property.TSERV_COMPACTION_SERVICE_PREFIX); + + // check if deprecated properties for compaction executor are set + if (aconf.isPropertySet(Property.TSERV_MAJC_MAXCONCURRENT, true)) { + + String defaultServicePrefix = + Property.TSERV_COMPACTION_SERVICE_PREFIX.getKey() + DEFAULT_SERVICE.canonical() + "."; + + // check if any properties for the default compaction service are set + boolean defaultServicePropsSet = configs.keySet().stream() + .filter(key -> key.startsWith(defaultServicePrefix)).map(Property::getPropertyByKey) + .anyMatch(prop -> prop == null || aconf.isPropertySet(prop, true)); + + if (defaultServicePropsSet) { + + String warning = String.format( + "The deprecated property %s was set. Properties with the prefix %s " + + "were also set, which replace the deprecated properties. The deprecated " + + "property was therefore ignored.", + Property.TSERV_MAJC_MAXCONCURRENT.getKey(), defaultServicePrefix); + + deprecationWarningConsumer.accept(warning); + + } else { + String numThreads = aconf.get(Property.TSERV_MAJC_MAXCONCURRENT); + + // Its possible a user has configured the other compaction services, but not the default + // service. In this case want to produce a config with the default service configs + // overridden using deprecated configs. + + HashMap<String,String> configsCopy = new HashMap<>(configs); + + Map<String,String> defaultServiceConfigs = + Map.of(defaultServicePrefix + "planner", DefaultCompactionPlanner.class.getName(), + defaultServicePrefix + "planner.opts.executors", + "[{'name':'deprecated', 'numThreads':" + numThreads + "}]"); + + configsCopy.putAll(defaultServiceConfigs); + + String warning = String.format( + "The deprecated property %s was set. Properties with the prefix %s " + + "were not set, these should replace the deprecated properties. The old " + + "properties were automatically mapped to the new properties in process " + + "creating : %s.", + Property.TSERV_MAJC_MAXCONCURRENT.getKey(), defaultServicePrefix, + defaultServiceConfigs); + + deprecationWarningConsumer.accept(warning); + + configs = Map.copyOf(configsCopy); + } + } + + return configs; + + } + + public CompactionServicesConfig(AccumuloConfiguration aconf, + Consumer<String> deprecationWarningConsumer) { + this.deprecationWarningConsumer = deprecationWarningConsumer; + Map<String,String> configs = getConfiguration(aconf); + + configs.forEach((prop, val) -> { + + var suffix = prop.substring(Property.TSERV_COMPACTION_SERVICE_PREFIX.getKey().length()); + String[] tokens = suffix.split("\\."); + if (tokens.length == 4 && tokens[1].equals("planner") && tokens[2].equals("opts")) { + getOptions().computeIfAbsent(tokens[0], k -> new HashMap<>()).put(tokens[3], val); + } else if (tokens.length == 2 && tokens[1].equals("planner")) { + getPlanners().put(tokens[0], val); + } else if (tokens.length == 3 && tokens[1].equals("rate") && tokens[2].equals("limit")) { + var eprop = Property.getPropertyByKey(prop); + if (eprop == null || aconf.isPropertySet(eprop, true) + || !isDeprecatedThroughputSet(aconf)) { + getRateLimits().put(tokens[0], ConfigurationTypeHelper.getFixedMemoryAsBytes(val)); Review comment: Addressed in 76b4e70 ########## File path: server/base/src/main/java/org/apache/accumulo/server/conf/CheckCompactionConfig.java ########## @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.conf; + +import java.io.FileNotFoundException; +import java.nio.file.Path; +import java.util.Set; + +import org.apache.accumulo.core.cli.Help; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.SiteConfiguration; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; +import org.apache.accumulo.core.spi.compaction.CompactionPlanner; +import org.apache.accumulo.core.spi.compaction.CompactionServiceId; +import org.apache.accumulo.core.util.ConfigurationImpl; +import org.apache.accumulo.core.util.compaction.CompactionPlannerInitParams; +import org.apache.accumulo.core.util.compaction.CompactionServicesConfig; +import org.apache.accumulo.start.spi.KeywordExecutable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; + +@AutoService(KeywordExecutable.class) +public class CheckCompactionConfig implements KeywordExecutable { + + private final static Logger log = LoggerFactory.getLogger(CheckCompactionConfig.class); + + static class Opts extends Help { + @Parameter(description = "<path/to/props/file>", required = true) Review comment: Addressed in 76b4e70 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
