Michael Blow has submitted this change and it was merged. Change subject: Make Default Dir a Command Line Option ......................................................................
Make Default Dir a Command Line Option - Avoid using static field to hold default directory, in favor of an option. - Improve output on failed regexadm match (indicate where actual and expected diverge) Change-Id: I7855c8f344eea9c9b6a394d85413a062a3ddb609 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1751 Reviewed-by: Michael Blow <mb...@apache.org> Tested-by: Michael Blow <mb...@apache.org> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java D asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm D asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm D asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java 16 files changed, 403 insertions(+), 327 deletions(-) Approvals: Michael Blow: Looks good to me, approved; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java index 53fc7ec..28a42f5 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java @@ -53,6 +53,7 @@ public class AsterixHyracksIntegrationUtil { static class LoggerHolder { static final Logger LOGGER = Logger.getLogger(AsterixHyracksIntegrationUtil.class.getName()); + private LoggerHolder() { } } @@ -62,14 +63,13 @@ public static final int DEFAULT_HYRACKS_CC_CLUSTER_PORT = 1099; public ClusterControllerService cc; - public NodeControllerService[] ncs; + public NodeControllerService[] ncs = new NodeControllerService[0]; public IHyracksClientConnection hcc; private ConfigManager configManager; private List<String> nodeNames; public void init(boolean deleteOldInstanceData) throws Exception { - ncs = new NodeControllerService[0]; // ensure that ncs is not null final ICCApplication ccApplication = createCCApplication(); configManager = new ConfigManager(); ccApplication.registerConfig(configManager); @@ -81,26 +81,26 @@ deleteTransactionLogs(); removeTestStorageFiles(); } - final List<NCConfig> ncConfigs = new ArrayList<>(); - nodeNames.forEach(nodeId -> ncConfigs.add(createNCConfig(nodeId, configManager))); - final PropertiesAccessor accessor = PropertiesAccessor.getInstance(configManager.getAppConfig()); - ncConfigs.forEach((ncConfig1) -> fixupIODevices(ncConfig1, accessor)); + final List<NodeControllerService> nodeControllers = new ArrayList<>(); + for (String nodeId : nodeNames) { + final INCApplication ncApplication = createNCApplication(); + ConfigManager ncConfigManager = new ConfigManager(); + ncApplication.registerConfig(ncConfigManager); + nodeControllers.add( + new NodeControllerService(fixupIODevices(createNCConfig(nodeId, ncConfigManager)), ncApplication)); + } ; cc.start(); // Starts ncs. nodeNames = ccConfig.getConfigManager().getNodeNames(); - List<NodeControllerService> nodeControllers = new ArrayList<>(); List<Thread> startupThreads = new ArrayList<>(); - for (NCConfig ncConfig : ncConfigs) { - final INCApplication ncApplication = createNCApplication(); - NodeControllerService nodeControllerService = new NodeControllerService(ncConfig, ncApplication); - nodeControllers.add(nodeControllerService); - Thread ncStartThread = new Thread("IntegrationUtil-" + ncConfig.getNodeId()) { + for (NodeControllerService nc : nodeControllers) { + Thread ncStartThread = new Thread("IntegrationUtil-" + nc.getId()) { @Override public void run() { try { - nodeControllerService.start(); + nc.start(); } catch (Exception e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); } @@ -113,8 +113,8 @@ for (Thread thread : startupThreads) { thread.join(); } - for (NCConfig ncConfig : ncConfigs) { - for (String ioDevice : ncConfig.getIODevices()) { + for (NodeControllerService nc : nodeControllers) { + for (String ioDevice : nc.getConfiguration().getIODevices()) { if (!new File(ioDevice).isAbsolute()) { throw new IllegalStateException("iodevice not absolute: " + ioDevice); } @@ -123,7 +123,7 @@ // Wait until cluster becomes active ClusterStateManager.INSTANCE.waitForState(ClusterState.ACTIVE); hcc = new HyracksConnection(cc.getConfig().getClientListenAddress(), cc.getConfig().getClientListenPort()); - ncs = nodeControllers.toArray(new NodeControllerService[nodeControllers.size()]); + this.ncs = nodeControllers.toArray(new NodeControllerService[nodeControllers.size()]); } protected CCConfig createCCConfig(ConfigManager configManager) throws IOException { @@ -159,7 +159,8 @@ return new NCApplication(); } - private NCConfig fixupIODevices(NCConfig ncConfig, PropertiesAccessor accessor) { + private NCConfig fixupIODevices(NCConfig ncConfig) throws IOException, AsterixException { + PropertiesAccessor accessor = PropertiesAccessor.getInstance(ncConfig.getAppConfig()); String tempPath = System.getProperty(IO_DIR_KEY); if (tempPath.endsWith(File.separator)) { tempPath = tempPath.substring(0, tempPath.length() - 1); @@ -182,7 +183,8 @@ ioDeviceDir.mkdirs(); ioDevices.add(iodevicePath); } - configManager.set(ncConfig.getNodeId(), NCConfig.Option.IODEVICES, ioDevices.toArray(new String[0])); + ncConfig.getConfigManager().set(ncConfig.getNodeId(), NCConfig.Option.IODEVICES, + ioDevices.toArray(new String[0])); return ncConfig; } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java index 466203c..9678e09 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java @@ -36,9 +36,10 @@ static void registerConfigOptions(IConfigManager configManager) { AsterixProperties.registerConfigOptions(configManager); - ControllerConfig.defaultDir = FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "asterixdb"); - NCConfig.defaultAppClass = NCApplication.class.getName(); - CCConfig.defaultAppClass = CCApplication.class.getName(); + ControllerConfig.Option.DEFAULT_DIR + .setDefaultValue(FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "asterixdb")); + NCConfig.Option.APP_CLASS.setDefaultValue(NCApplication.class.getName()); + CCConfig.Option.APP_CLASS.setDefaultValue(CCApplication.class.getName()); try { InputStream propertyStream = ApplicationConfigurator.class.getClassLoader() .getResourceAsStream("git.properties"); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java index 340b1ce..f994e98 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiServletTest.java @@ -37,7 +37,9 @@ import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import com.fasterxml.jackson.databind.ObjectMapper; @@ -48,11 +50,21 @@ public class VersionApiServletTest { - @Test - public void testGet() throws Exception { + @Before + public void setup() throws Exception { // Starts test asterixdb cluster. SqlppExecutionTest.setUp(); + } + @After + public void teardown() throws Exception { + // Tears down the asterixdb cluster. + SqlppExecutionTest.tearDown(); + } + + + @Test + public void testGet() throws Exception { // Configures a test version api servlet. VersionApiServlet servlet = new VersionApiServlet(new ConcurrentHashMap<>(), new String[] { "/" }); Map<String, String> propMap = new HashMap<>(); @@ -113,8 +125,5 @@ // Checks the response contains all the expected keys. Assert.assertEquals(actualResponse.toString(), expectedResponse.toString()); - - // Tears down the asterixdb cluster. - SqlppExecutionTest.tearDown(); } } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java index acd4540..2018e5a 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/config/ConfigUsageTest.java @@ -85,8 +85,9 @@ ConfigManager configManager = new ConfigManager(); CCApplication application = new CCApplication(); application.registerConfig(configManager); - ControllerConfig.defaultDir = ControllerConfig.defaultDir.replace(System.getProperty("java.io.tmpdir"), - "${java.io.tmpdir}/"); + ControllerConfig.Option.DEFAULT_DIR + .setDefaultValue(((String) ControllerConfig.Option.DEFAULT_DIR.defaultValue()) + .replace(System.getProperty("java.io.tmpdir"), "${java.io.tmpdir}/")); return configManager; } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 3c5f823..4b24108 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -188,24 +188,24 @@ if (lineExpected.isEmpty()) { continue; } - throwLineChanged(scriptFile, lineExpected, "<EOF>", num); + throw createLineChangedException(scriptFile, lineExpected, "<EOF>", num); } // Comparing result equality but ignore "Time"-prefixed fields. (for metadata tests.) String[] lineSplitsExpected = lineExpected.split("Time"); String[] lineSplitsActual = lineActual.split("Time"); if (lineSplitsExpected.length != lineSplitsActual.length) { - throwLineChanged(scriptFile, lineExpected, lineActual, num); + throw createLineChangedException(scriptFile, lineExpected, lineActual, num); } if (!equalStrings(lineSplitsExpected[0], lineSplitsActual[0], regex)) { - throwLineChanged(scriptFile, lineExpected, lineActual, num); + throw createLineChangedException(scriptFile, lineExpected, lineActual, num); } for (int i = 1; i < lineSplitsExpected.length; i++) { String[] splitsByCommaExpected = lineSplitsExpected[i].split(","); String[] splitsByCommaActual = lineSplitsActual[i].split(","); if (splitsByCommaExpected.length != splitsByCommaActual.length) { - throwLineChanged(scriptFile, lineExpected, lineActual, num); + throw createLineChangedException(scriptFile, lineExpected, lineActual, num); } for (int j = 1; j < splitsByCommaExpected.length; j++) { if (splitsByCommaExpected[j].indexOf("DatasetId") >= 0) { @@ -214,7 +214,7 @@ continue; } if (!equalStrings(splitsByCommaExpected[j], splitsByCommaActual[j], regex)) { - throwLineChanged(scriptFile, lineExpected, lineActual, num); + throw createLineChangedException(scriptFile, lineExpected, lineActual, num); } } } @@ -223,7 +223,7 @@ } lineActual = readerActual.readLine(); if (lineActual != null) { - throwLineChanged(scriptFile, "<EOF>", lineActual, num); + throw createLineChangedException(scriptFile, "<EOF>", lineActual, num); } } catch (Exception e) { System.err.println("Actual results file: " + actualFile.toString()); @@ -235,9 +235,9 @@ } - private void throwLineChanged(File scriptFile, String lineExpected, String lineActual, int num) - throws ComparisonException { - throw new ComparisonException("Result for " + scriptFile + " changed at line " + num + ":\n< " + private ComparisonException createLineChangedException(File scriptFile, String lineExpected, String lineActual, + int num) { + return new ComparisonException("Result for " + scriptFile + " changed at line " + num + ":\n< " + truncateIfLong(lineExpected) + "\n> " + truncateIfLong(lineActual)); } @@ -346,7 +346,6 @@ public void runScriptAndCompareWithResultRegex(File scriptFile, File expectedFile, File actualFile) throws Exception { - System.err.println("Expected results file: " + expectedFile.toString()); String lineExpected, lineActual; try (BufferedReader readerExpected = new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8")); @@ -384,11 +383,7 @@ throw new Exception("Result for " + scriptFile + ": expected pattern '" + expression + "' not found in result: " + actual); } - } catch (Exception e) { - System.err.println("Actual results file: " + actualFile.toString()); - throw e; } - } public void runScriptAndCompareWithResultRegexAdm(File scriptFile, File expectedFile, File actualFile) @@ -399,6 +394,22 @@ IOUtils.copy(new FileInputStream(expectedFile), expected, StandardCharsets.UTF_8); Pattern pattern = Pattern.compile(expected.toString(), Pattern.DOTALL | Pattern.MULTILINE); if (!pattern.matcher(actual.toString()).matches()) { + // figure out where the problem first occurs... + StringBuilder builder = new StringBuilder(); + String [] lines = expected.toString().split("\\n"); + int endOfMatch = 0; + final StringBuffer actualBuffer = actual.getBuffer(); + for (int i = 0; i < lines.length; i++) { + builder.append(lines[i]).append('\n'); + Pattern partPatten = Pattern.compile(builder.toString(), Pattern.DOTALL | Pattern.MULTILINE); + final Matcher matcher = partPatten.matcher(actualBuffer); + if (!matcher.lookingAt()) { + final int eol = actualBuffer.indexOf("\n", endOfMatch); + String actualLine = actualBuffer.substring(endOfMatch, eol == -1 ? actualBuffer.length() : eol); + throw createLineChangedException(scriptFile, lines[i], actualLine, i + 1); + } + endOfMatch = matcher.end(); + } throw new Exception("Result for " + scriptFile + ": actual file did not match expected result"); } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm deleted file mode 100644 index 251aeb6..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm +++ /dev/null @@ -1,75 +0,0 @@ -{ - "cc" : { - "configUri" : "http://127.0.0.1:19002/admin/cluster/cc/config", - "statsUri" : "http://127.0.0.1:19002/admin/cluster/cc/stats", - "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/cc/threaddump" - }, - "config" : { - "active.memory.global.budget" : 67108864, - "compiler.framesize" : 32768, - "compiler.groupmemory" : 163840, - "compiler.joinmemory" : 262144, - "compiler.parallelism" : 0, - "compiler.sortmemory" : 327680, - "instance.name" : "DEFAULT_INSTANCE", - "log.level" : "INFO", - "max.wait.active.cluster" : 60, - "messaging.frame.count" : 512, - "messaging.frame.size" : 4096, - "metadata.callback.port" : 0, - "metadata.listen.port" : 0, - "metadata.node" : "asterix_nc1", - "metadata.registration.timeout.secs" : 60, - "replication.log.batchsize" : 4096, - "replication.log.buffer.numpages" : 8, - "replication.log.buffer.pagesize" : 131072, - "replication.max.remote.recovery.attempts" : 5, - "replication.timeout" : 30, - "txn.commitprofiler.enabled" : false, - "txn.commitprofiler.reportinterval" : 5, - "txn.job.recovery.memorysize" : 67108864, - "txn.lock.escalationthreshold" : 1000, - "txn.lock.shrinktimer" : 5000, - "txn.lock.timeout.sweepthreshold" : 10000, - "txn.lock.timeout.waitthreshold" : 60000, - "txn.log.buffer.numpages" : 8, - "txn.log.buffer.pagesize" : 131072, - "txn.log.checkpoint.history" : 0, - "txn.log.checkpoint.lsnthreshold" : 67108864, - "txn.log.checkpoint.pollfrequency" : 120, - "txn.log.partitionsize" : 268435456 - }, - "diagnosticsUri" : "http://127.0.0.1:19002/admin/diagnostics", - "fullShutdownUri" : "http://127.0.0.1:19002/admin/shutdown?all=true", - "metadata_node" : "asterix_nc1", - "ncs" : [ { - "configUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc1/config", - "node_id" : "asterix_nc1", - "partitions" : [ { - "active" : true, - "partition_id" : "partition_0" - }, { - "active" : true, - "partition_id" : "partition_1" - } ], - "state" : "ACTIVE", - "statsUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc1/stats", - "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc1/threaddump" - }, { - "configUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc2/config", - "node_id" : "asterix_nc2", - "partitions" : [ { - "active" : true, - "partition_id" : "partition_2" - }, { - "active" : true, - "partition_id" : "partition_3" - } ], - "state" : "ACTIVE", - "statsUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc2/stats", - "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc2/threaddump" - } ], - "shutdownUri" : "http://127.0.0.1:19002/admin/shutdown", - "state" : "ACTIVE", - "versionUri" : "http://127.0.0.1:19002/admin/version" -} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm new file mode 100644 index 0000000..5ec13a4 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -0,0 +1,76 @@ +\{ + "cc" : \{ + "configUri" : "http://127\.0\.0\.1:19002/admin/cluster/cc/config", + "statsUri" : "http://127\.0\.0\.1:19002/admin/cluster/cc/stats", + "threadDumpUri" : "http://127\.0\.0\.1:19002/admin/cluster/cc/threaddump" + \}, + "config" : \{ + "active\.memory\.global\.budget" : 67108864, + "compiler\.framesize" : 32768, + "compiler\.groupmemory" : 163840, + "compiler\.joinmemory" : 262144, + "compiler\.parallelism" : 0, + "compiler\.sortmemory" : 327680, + "default\.dir" : "/.*/asterixdb", + "instance\.name" : "DEFAULT_INSTANCE", + "log\.level" : "INFO", + "max\.wait\.active\.cluster" : 60, + "messaging\.frame\.count" : 512, + "messaging\.frame\.size" : 4096, + "metadata\.callback\.port" : 0, + "metadata\.listen\.port" : 0, + "metadata\.node" : "asterix_nc1", + "metadata\.registration\.timeout\.secs" : 60, + "replication\.log\.batchsize" : 4096, + "replication\.log\.buffer\.numpages" : 8, + "replication\.log\.buffer\.pagesize" : 131072, + "replication\.max\.remote\.recovery\.attempts" : 5, + "replication\.timeout" : 30, + "txn\.commitprofiler\.enabled" : false, + "txn\.commitprofiler\.reportinterval" : 5, + "txn\.job\.recovery\.memorysize" : 67108864, + "txn\.lock\.escalationthreshold" : 1000, + "txn\.lock\.shrinktimer" : 5000, + "txn\.lock\.timeout\.sweepthreshold" : 10000, + "txn\.lock\.timeout\.waitthreshold" : 60000, + "txn\.log\.buffer\.numpages" : 8, + "txn\.log\.buffer\.pagesize" : 131072, + "txn\.log\.checkpoint\.history" : 0, + "txn\.log\.checkpoint\.lsnthreshold" : 67108864, + "txn\.log\.checkpoint\.pollfrequency" : 120, + "txn\.log\.partitionsize" : 268435456 + \}, + "diagnosticsUri" : "http://127\.0\.0\.1:19002/admin/diagnostics", + "fullShutdownUri" : "http://127\.0\.0\.1:19002/admin/shutdown\?all=true", + "metadata_node" : "asterix_nc1", + "ncs" : \[ \{ + "configUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc1/config", + "node_id" : "asterix_nc1", + "partitions" : \[ \{ + "active" : true, + "partition_id" : "partition_0" + \}, \{ + "active" : true, + "partition_id" : "partition_1" + \} \], + "state" : "ACTIVE", + "statsUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc1/stats", + "threadDumpUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc1/threaddump" + \}, \{ + "configUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc2/config", + "node_id" : "asterix_nc2", + "partitions" : \[ \{ + "active" : true, + "partition_id" : "partition_2" + \}, \{ + "active" : true, + "partition_id" : "partition_3" + \} \], + "state" : "ACTIVE", + "statsUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc2/stats", + "threadDumpUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc2/threaddump" + \} \], + "shutdownUri" : "http://127\.0\.0\.1:19002/admin/shutdown", + "state" : "ACTIVE", + "versionUri" : "http://127\.0\.0\.1:19002/admin/version" +\} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm deleted file mode 100644 index 485a983..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm +++ /dev/null @@ -1,75 +0,0 @@ -{ - "cc" : { - "configUri" : "http://127.0.0.1:19002/admin/cluster/cc/config", - "statsUri" : "http://127.0.0.1:19002/admin/cluster/cc/stats", - "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/cc/threaddump" - }, - "config" : { - "active.memory.global.budget" : 67108864, - "compiler.framesize" : 32768, - "compiler.groupmemory" : 163840, - "compiler.joinmemory" : 262144, - "compiler.parallelism" : -1, - "compiler.sortmemory" : 327680, - "instance.name" : "DEFAULT_INSTANCE", - "log.level" : "WARNING", - "max.wait.active.cluster" : 60, - "messaging.frame.count" : 512, - "messaging.frame.size" : 4096, - "metadata.callback.port" : 0, - "metadata.listen.port" : 0, - "metadata.node" : "asterix_nc1", - "metadata.registration.timeout.secs" : 60, - "replication.log.batchsize" : 4096, - "replication.log.buffer.numpages" : 8, - "replication.log.buffer.pagesize" : 131072, - "replication.max.remote.recovery.attempts" : 5, - "replication.timeout" : 30, - "txn.commitprofiler.enabled" : false, - "txn.commitprofiler.reportinterval" : 5, - "txn.job.recovery.memorysize" : 67108864, - "txn.lock.escalationthreshold" : 1000, - "txn.lock.shrinktimer" : 5000, - "txn.lock.timeout.sweepthreshold" : 10000, - "txn.lock.timeout.waitthreshold" : 60000, - "txn.log.buffer.numpages" : 8, - "txn.log.buffer.pagesize" : 131072, - "txn.log.checkpoint.history" : 0, - "txn.log.checkpoint.lsnthreshold" : 67108864, - "txn.log.checkpoint.pollfrequency" : 120, - "txn.log.partitionsize" : 268435456 - }, - "diagnosticsUri" : "http://127.0.0.1:19002/admin/diagnostics", - "fullShutdownUri" : "http://127.0.0.1:19002/admin/shutdown?all=true", - "metadata_node" : "asterix_nc1", - "ncs" : [ { - "configUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc1/config", - "node_id" : "asterix_nc1", - "partitions" : [ { - "active" : true, - "partition_id" : "partition_0" - }, { - "active" : true, - "partition_id" : "partition_1" - } ], - "state" : "ACTIVE", - "statsUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc1/stats", - "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc1/threaddump" - }, { - "configUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc2/config", - "node_id" : "asterix_nc2", - "partitions" : [ { - "active" : true, - "partition_id" : "partition_2" - }, { - "active" : true, - "partition_id" : "partition_3" - } ], - "state" : "ACTIVE", - "statsUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc2/stats", - "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc2/threaddump" - } ], - "shutdownUri" : "http://127.0.0.1:19002/admin/shutdown", - "state" : "ACTIVE", - "versionUri" : "http://127.0.0.1:19002/admin/version" -} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm new file mode 100644 index 0000000..d845b2a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -0,0 +1,76 @@ +\{ + "cc" : \{ + "configUri" : "http://127\.0\.0\.1:19002/admin/cluster/cc/config", + "statsUri" : "http://127\.0\.0\.1:19002/admin/cluster/cc/stats", + "threadDumpUri" : "http://127\.0\.0\.1:19002/admin/cluster/cc/threaddump" + \}, + "config" : \{ + "active\.memory\.global\.budget" : 67108864, + "compiler\.framesize" : 32768, + "compiler\.groupmemory" : 163840, + "compiler\.joinmemory" : 262144, + "compiler\.parallelism" : -1, + "compiler\.sortmemory" : 327680, + "default\.dir" : "/.*/asterixdb", + "instance\.name" : "DEFAULT_INSTANCE", + "log\.level" : "WARNING", + "max\.wait\.active\.cluster" : 60, + "messaging\.frame\.count" : 512, + "messaging\.frame\.size" : 4096, + "metadata\.callback\.port" : 0, + "metadata\.listen\.port" : 0, + "metadata\.node" : "asterix_nc1", + "metadata\.registration\.timeout\.secs" : 60, + "replication\.log\.batchsize" : 4096, + "replication\.log\.buffer\.numpages" : 8, + "replication\.log\.buffer\.pagesize" : 131072, + "replication\.max\.remote\.recovery\.attempts" : 5, + "replication\.timeout" : 30, + "txn\.commitprofiler\.enabled" : false, + "txn\.commitprofiler\.reportinterval" : 5, + "txn\.job\.recovery\.memorysize" : 67108864, + "txn\.lock\.escalationthreshold" : 1000, + "txn\.lock\.shrinktimer" : 5000, + "txn\.lock\.timeout\.sweepthreshold" : 10000, + "txn\.lock\.timeout\.waitthreshold" : 60000, + "txn\.log\.buffer\.numpages" : 8, + "txn\.log\.buffer\.pagesize" : 131072, + "txn\.log\.checkpoint\.history" : 0, + "txn\.log\.checkpoint\.lsnthreshold" : 67108864, + "txn\.log\.checkpoint\.pollfrequency" : 120, + "txn\.log\.partitionsize" : 268435456 + \}, + "diagnosticsUri" : "http://127\.0\.0\.1:19002/admin/diagnostics", + "fullShutdownUri" : "http://127\.0\.0\.1:19002/admin/shutdown\?all=true", + "metadata_node" : "asterix_nc1", + "ncs" : \[ \{ + "configUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc1/config", + "node_id" : "asterix_nc1", + "partitions" : \[ \{ + "active" : true, + "partition_id" : "partition_0" + \}, \{ + "active" : true, + "partition_id" : "partition_1" + \} \], + "state" : "ACTIVE", + "statsUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc1/stats", + "threadDumpUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc1/threaddump" + \}, \{ + "configUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc2/config", + "node_id" : "asterix_nc2", + "partitions" : \[ \{ + "active" : true, + "partition_id" : "partition_2" + \}, \{ + "active" : true, + "partition_id" : "partition_3" + \} \], + "state" : "ACTIVE", + "statsUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc2/stats", + "threadDumpUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc2/threaddump" + \} \], + "shutdownUri" : "http://127\.0\.0\.1:19002/admin/shutdown", + "state" : "ACTIVE", + "versionUri" : "http://127\.0\.0\.1:19002/admin/version" +\} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm deleted file mode 100644 index ec0ae4a..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm +++ /dev/null @@ -1,75 +0,0 @@ -{ - "cc" : { - "configUri" : "http://127.0.0.1:19002/admin/cluster/cc/config", - "statsUri" : "http://127.0.0.1:19002/admin/cluster/cc/stats", - "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/cc/threaddump" - }, - "config" : { - "active.memory.global.budget" : 67108864, - "compiler.framesize" : 32768, - "compiler.groupmemory" : 163840, - "compiler.joinmemory" : 262144, - "compiler.parallelism" : 3, - "compiler.sortmemory" : 327680, - "instance.name" : "DEFAULT_INSTANCE", - "log.level" : "WARNING", - "max.wait.active.cluster" : 60, - "messaging.frame.count" : 512, - "messaging.frame.size" : 4096, - "metadata.callback.port" : 0, - "metadata.listen.port" : 0, - "metadata.node" : "asterix_nc1", - "metadata.registration.timeout.secs" : 60, - "replication.log.batchsize" : 4096, - "replication.log.buffer.numpages" : 8, - "replication.log.buffer.pagesize" : 131072, - "replication.max.remote.recovery.attempts" : 5, - "replication.timeout" : 30, - "txn.commitprofiler.enabled" : false, - "txn.commitprofiler.reportinterval" : 5, - "txn.job.recovery.memorysize" : 67108864, - "txn.lock.escalationthreshold" : 1000, - "txn.lock.shrinktimer" : 5000, - "txn.lock.timeout.sweepthreshold" : 10000, - "txn.lock.timeout.waitthreshold" : 60000, - "txn.log.buffer.numpages" : 8, - "txn.log.buffer.pagesize" : 131072, - "txn.log.checkpoint.history" : 0, - "txn.log.checkpoint.lsnthreshold" : 67108864, - "txn.log.checkpoint.pollfrequency" : 120, - "txn.log.partitionsize" : 268435456 - }, - "diagnosticsUri" : "http://127.0.0.1:19002/admin/diagnostics", - "fullShutdownUri" : "http://127.0.0.1:19002/admin/shutdown?all=true", - "metadata_node" : "asterix_nc1", - "ncs" : [ { - "configUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc1/config", - "node_id" : "asterix_nc1", - "partitions" : [ { - "active" : true, - "partition_id" : "partition_0" - }, { - "active" : true, - "partition_id" : "partition_1" - } ], - "state" : "ACTIVE", - "statsUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc1/stats", - "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc1/threaddump" - }, { - "configUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc2/config", - "node_id" : "asterix_nc2", - "partitions" : [ { - "active" : true, - "partition_id" : "partition_2" - }, { - "active" : true, - "partition_id" : "partition_3" - } ], - "state" : "ACTIVE", - "statsUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc2/stats", - "threadDumpUri" : "http://127.0.0.1:19002/admin/cluster/node/asterix_nc2/threaddump" - } ], - "shutdownUri" : "http://127.0.0.1:19002/admin/shutdown", - "state" : "ACTIVE", - "versionUri" : "http://127.0.0.1:19002/admin/version" -} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm new file mode 100644 index 0000000..1315220 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -0,0 +1,76 @@ +\{ + "cc" : \{ + "configUri" : "http://127\.0\.0\.1:19002/admin/cluster/cc/config", + "statsUri" : "http://127\.0\.0\.1:19002/admin/cluster/cc/stats", + "threadDumpUri" : "http://127\.0\.0\.1:19002/admin/cluster/cc/threaddump" + \}, + "config" : \{ + "active\.memory\.global\.budget" : 67108864, + "compiler\.framesize" : 32768, + "compiler\.groupmemory" : 163840, + "compiler\.joinmemory" : 262144, + "compiler\.parallelism" : 3, + "compiler\.sortmemory" : 327680, + "default\.dir" : "/.*/asterixdb", + "instance\.name" : "DEFAULT_INSTANCE", + "log\.level" : "WARNING", + "max\.wait\.active\.cluster" : 60, + "messaging\.frame\.count" : 512, + "messaging\.frame\.size" : 4096, + "metadata\.callback\.port" : 0, + "metadata\.listen\.port" : 0, + "metadata\.node" : "asterix_nc1", + "metadata\.registration\.timeout\.secs" : 60, + "replication\.log\.batchsize" : 4096, + "replication\.log\.buffer\.numpages" : 8, + "replication\.log\.buffer\.pagesize" : 131072, + "replication\.max\.remote\.recovery\.attempts" : 5, + "replication\.timeout" : 30, + "txn\.commitprofiler\.enabled" : false, + "txn\.commitprofiler\.reportinterval" : 5, + "txn\.job\.recovery\.memorysize" : 67108864, + "txn\.lock\.escalationthreshold" : 1000, + "txn\.lock\.shrinktimer" : 5000, + "txn\.lock\.timeout\.sweepthreshold" : 10000, + "txn\.lock\.timeout\.waitthreshold" : 60000, + "txn\.log\.buffer\.numpages" : 8, + "txn\.log\.buffer\.pagesize" : 131072, + "txn\.log\.checkpoint\.history" : 0, + "txn\.log\.checkpoint\.lsnthreshold" : 67108864, + "txn\.log\.checkpoint\.pollfrequency" : 120, + "txn\.log\.partitionsize" : 268435456 + \}, + "diagnosticsUri" : "http://127\.0\.0\.1:19002/admin/diagnostics", + "fullShutdownUri" : "http://127\.0\.0\.1:19002/admin/shutdown\?all=true", + "metadata_node" : "asterix_nc1", + "ncs" : \[ \{ + "configUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc1/config", + "node_id" : "asterix_nc1", + "partitions" : \[ \{ + "active" : true, + "partition_id" : "partition_0" + \}, \{ + "active" : true, + "partition_id" : "partition_1" + \} \], + "state" : "ACTIVE", + "statsUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc1/stats", + "threadDumpUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc1/threaddump" + \}, \{ + "configUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc2/config", + "node_id" : "asterix_nc2", + "partitions" : \[ \{ + "active" : true, + "partition_id" : "partition_2" + \}, \{ + "active" : true, + "partition_id" : "partition_3" + \} \], + "state" : "ACTIVE", + "statsUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc2/stats", + "threadDumpUri" : "http://127\.0\.0\.1:19002/admin/cluster/node/asterix_nc2/threaddump" + \} \], + "shutdownUri" : "http://127\.0\.0\.1:19002/admin/shutdown", + "state" : "ACTIVE", + "versionUri" : "http://127\.0\.0\.1:19002/admin/version" +\} \ No newline at end of file diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java index e3a5fb9..1d09fff 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java @@ -18,12 +18,14 @@ */ package org.apache.asterix.common.config; -import java.util.function.Supplier; +import java.util.function.Function; +import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.config.IOption; import org.apache.hyracks.api.config.IOptionType; import org.apache.hyracks.api.config.Section; import org.apache.hyracks.control.common.config.OptionTypes; +import org.apache.hyracks.control.common.controllers.ControllerConfig; import org.apache.hyracks.control.common.controllers.NCConfig; import org.apache.hyracks.util.file.FileUtil; @@ -33,29 +35,34 @@ INITIAL_RUN(OptionTypes.BOOLEAN, false, "A flag indicating if it's the first time the NC is started"), CORE_DUMP_DIR( OptionTypes.STRING, - (Supplier<String>) () -> FileUtil.joinPath(NCConfig.defaultDir, "coredump"), - "The directory where node core dumps should be written"), + appConfig -> FileUtil.joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "coredump"), + "The directory where node core dumps should be written", + "<value of " + ControllerConfig.Option.DEFAULT_DIR.cmdline() + ">/coredump"), TXN_LOG_DIR( OptionTypes.STRING, - (Supplier<String>) () -> FileUtil.joinPath(NCConfig.defaultDir, "txn-log"), - "The directory where transaction logs should be stored"), - STORAGE_SUBDIR(OptionTypes.STRING, "storage", "The subdirectory name under each iodevice used for storage"), - ; + appConfig -> FileUtil.joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "txn-log"), + "The directory where transaction logs should be stored", + "<value of " + ControllerConfig.Option.DEFAULT_DIR.cmdline() + ">/txn-log"), + STORAGE_SUBDIR(OptionTypes.STRING, "storage", "The subdirectory name under each iodevice used for storage"),; private final IOptionType type; private final Object defaultValue; private final String description; + private final String defaultValueDescription; <T> Option(IOptionType<T> type, T defaultValue, String description) { this.type = type; this.defaultValue = defaultValue; this.description = description; + this.defaultValueDescription = null; } - <T> Option(IOptionType<T> type, Supplier<T> defaultValue, String description) { + <T> Option(IOptionType<T> type, Function<IApplicationConfig, T> defaultValue, String description, + String defaultValueDescription) { this.type = type; this.defaultValue = defaultValue; this.description = description; + this.defaultValueDescription = defaultValueDescription; } @Override @@ -79,9 +86,15 @@ } @Override + public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) { + return defaultValueDescription; + } + + @Override public boolean hidden() { return this == INITIAL_RUN; } + } public NodeProperties(PropertiesAccessor accessor) { diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java index 63163bb..2a83bb7 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java @@ -142,7 +142,10 @@ } } else { registeredOptions.add(option); - optionSetters.put(option, (node, value, isDefault) -> correctedMap(node, isDefault).put(option, value)); + optionSetters.put(option, + (node, value, + isDefault) -> correctedMap(option.section() == Section.NC ? node : null, isDefault) + .put(option, value)); if (LOGGER.isLoggable(Level.FINE)) { optionSetters.put(option, (node, value, isDefault) -> LOGGER .fine((isDefault ? "defaulting" : "setting ") + option.toIniString() + " to " + value)); @@ -184,8 +187,7 @@ return map == null ? null : map.get(key); } - public void processConfig() - throws CmdLineException, IOException { + public void processConfig() throws CmdLineException, IOException { if (!configured) { for (List<IConfigurator> configuratorList : configurators.values()) { for (IConfigurator configurator : configuratorList) { @@ -222,8 +224,7 @@ @SuppressWarnings({ "squid:S106", "squid:S1147" }) // use of System.err, System.exit() private List<String> processCommandLine(Collection<Section> sections, OptionHandlerFilter usageFilter, - BiConsumer<IOption, Object> setAction) - throws CmdLineException { + BiConsumer<IOption, Object> setAction) throws CmdLineException { final Args4jBean bean = new Args4jBean(); CmdLineParser cmdLineParser = new CmdLineParser(bean); final List<String> appArgs = new ArrayList<>(); @@ -277,12 +278,12 @@ for (IOption option : iniPointerOptions) { Object pointer = get(option); if (pointer instanceof String) { - ini = ConfigUtils.loadINIFile((String)pointer); + ini = ConfigUtils.loadINIFile((String) pointer); } else if (pointer instanceof URL) { - ini = ConfigUtils.loadINIFile((URL)pointer); + ini = ConfigUtils.loadINIFile((URL) pointer); } else if (pointer != null) { - throw new IllegalArgumentException("config file pointer options must be of type String (for file) or " + - "URL, instead of " + option.type().targetType()); + throw new IllegalArgumentException("config file pointer options must be of type String (for file) or " + + "URL, instead of " + option.type().targetType()); } } if (ini == null) { diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java index 51451ce..f83df3a 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/CCConfig.java @@ -26,8 +26,9 @@ import java.net.InetAddress; import java.util.ArrayList; import java.util.List; -import java.util.function.Supplier; +import java.util.function.Function; +import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.config.IOption; import org.apache.hyracks.api.config.IOptionType; import org.apache.hyracks.api.config.Section; @@ -38,10 +39,8 @@ @SuppressWarnings("SameParameterValue") public class CCConfig extends ControllerConfig { - public static String defaultAppClass; - public enum Option implements IOption { - APP_CLASS(STRING, (Supplier<String>)() -> defaultAppClass), + APP_CLASS(STRING, (String) null), ADDRESS(STRING, InetAddress.getLoopbackAddress().getHostAddress()), CLUSTER_LISTEN_ADDRESS(STRING, ADDRESS), CLUSTER_LISTEN_PORT(INTEGER, 1099), @@ -58,32 +57,40 @@ RESULT_TTL(LONG, 86400000L), // TODO(mblow): add time unit RESULT_SWEEP_THRESHOLD(LONG, 60000L), // TODO(mblow): add time unit @SuppressWarnings("RedundantCast") // not redundant- false positive from IDEA - ROOT_DIR(STRING, (Supplier<String>)() -> FileUtil.joinPath(defaultDir, "ClusterControllerService")), + ROOT_DIR(STRING, (Function<IApplicationConfig, String>) appConfig -> + FileUtil.joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), + "ClusterControllerService"), "<value of " + ControllerConfig.Option.DEFAULT_DIR.cmdline() + + ">/ClusterControllerService"), CLUSTER_TOPOLOGY(STRING), JOB_QUEUE_CLASS(STRING, "org.apache.hyracks.control.cc.scheduler.FIFOJobQueue"), JOB_QUEUE_CAPACITY(INTEGER, 4096), JOB_MANAGER_CLASS(STRING, "org.apache.hyracks.control.cc.job.JobManager"); private final IOptionType parser; - private final Object defaultValue; + private Object defaultValue; + private final String defaultValueDescription; <T> Option(IOptionType<T> parser) { - this(parser, (T)null); + this(parser, (T) null); } <T> Option(IOptionType<T> parser, Option defaultOption) { this.parser = parser; this.defaultValue = defaultOption; + defaultValueDescription = null; } <T> Option(IOptionType<T> parser, T defaultValue) { this.parser = parser; this.defaultValue = defaultValue; + defaultValueDescription = null; } - <T> Option(IOptionType<T> parser, Supplier<T> defaultValue) { + <T> Option(IOptionType<T> parser, Function<IApplicationConfig, T> defaultValue, + String defaultValueDescription) { this.parser = parser; this.defaultValue = defaultValue; + this.defaultValueDescription = defaultValueDescription; } @Override @@ -129,16 +136,16 @@ case HEARTBEAT_MAX_MISSES: return "Sets the maximum number of missed heartbeats before a node is marked as dead"; case PROFILE_DUMP_PERIOD: - return "Sets the time duration between two profile dumps from each node controller in " + - "milliseconds; 0 to disable"; + return "Sets the time duration between two profile dumps from each node controller in " + + "milliseconds; 0 to disable"; case JOB_HISTORY_SIZE: return "Limits the number of historical jobs remembered by the system to the specified value"; case RESULT_TTL: - return "Limits the amount of time results for asynchronous jobs should be retained by the system " + - "in milliseconds"; + return "Limits the amount of time results for asynchronous jobs should be retained by the system " + + "in milliseconds"; case RESULT_SWEEP_THRESHOLD: - return "The duration within which an instance of the result cleanup should be invoked in " + - "milliseconds"; + return "The duration within which an instance of the result cleanup should be invoked in " + + "milliseconds"; case ROOT_DIR: return "Sets the root folder used for file operations"; case CLUSTER_TOPOLOGY: @@ -153,6 +160,15 @@ throw new IllegalStateException("NYI: " + this); } } + + public void setDefaultValue(Object defaultValue) { + this.defaultValue = defaultValue; + } + + @Override + public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) { + return defaultValueDescription; + } } private final ConfigManager configManager; diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java index 05f7854..a9b3f97 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java @@ -31,19 +31,21 @@ public class ControllerConfig implements Serializable { - public static String defaultDir = FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "hyracks"); - public enum Option implements IOption { - CONFIG_FILE(OptionTypes.STRING, "Specify path to master configuration file"), - CONFIG_FILE_URL(OptionTypes.URL, "Specify URL to master configuration file"); + CONFIG_FILE(OptionTypes.STRING, "Specify path to master configuration file", null), + CONFIG_FILE_URL(OptionTypes.URL, "Specify URL to master configuration file", null), + DEFAULT_DIR(OptionTypes.STRING, "Directory where files are written to by default", + FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "hyracks")), + ; private final IOptionType type; private final String description; + private String defaultValue; - - Option(IOptionType type, String description) { + Option(IOptionType type, String description, String defaultValue) { this.type = type; this.description = description; + this.defaultValue = defaultValue; } @Override @@ -63,7 +65,11 @@ @Override public Object defaultValue() { - return null; + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java index 9cf4da3..86cc19e 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java @@ -28,7 +28,7 @@ import java.net.InetAddress; import java.util.ArrayList; import java.util.List; -import java.util.function.Supplier; +import java.util.function.Function; import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.config.IOption; @@ -40,8 +40,6 @@ public class NCConfig extends ControllerConfig { private static final long serialVersionUID = 3L; - public static String defaultAppClass; - public enum Option implements IOption { ADDRESS(STRING, InetAddress.getLoopbackAddress().getHostAddress()), PUBLIC_ADDRESS(STRING, ADDRESS), @@ -49,11 +47,11 @@ CLUSTER_LISTEN_PORT(INTEGER, 0), NCSERVICE_ADDRESS(STRING, PUBLIC_ADDRESS), NCSERVICE_PORT(INTEGER, 9090), - CLUSTER_ADDRESS(STRING, (String)null), + CLUSTER_ADDRESS(STRING, (String) null), CLUSTER_PORT(INTEGER, 1099), CLUSTER_PUBLIC_ADDRESS(STRING, PUBLIC_ADDRESS), CLUSTER_PUBLIC_PORT(INTEGER, CLUSTER_LISTEN_PORT), - NODE_ID(STRING, (String)null), + NODE_ID(STRING, (String) null), DATA_LISTEN_ADDRESS(STRING, ADDRESS), DATA_LISTEN_PORT(INTEGER, 0), DATA_PUBLIC_ADDRESS(STRING, PUBLIC_ADDRESS), @@ -67,36 +65,42 @@ MESSAGING_PUBLIC_ADDRESS(STRING, PUBLIC_ADDRESS), MESSAGING_PUBLIC_PORT(INTEGER, MESSAGING_LISTEN_PORT), CLUSTER_CONNECT_RETRIES(INTEGER, 5), - @SuppressWarnings("RedundantCast") // not redundant- false positive from IDEA - IODEVICES(STRING_ARRAY, (Supplier<String []>)() -> new String [] { FileUtil.joinPath(defaultDir, "iodevice") }), + IODEVICES(STRING_ARRAY, appConfig -> new String[] { + FileUtil.joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "iodevice") }, + "<value of " + ControllerConfig.Option.DEFAULT_DIR.cmdline() + ">/iodevice"), NET_THREAD_COUNT(INTEGER, 1), NET_BUFFER_COUNT(INTEGER, 1), RESULT_TTL(LONG, 86400000L), RESULT_SWEEP_THRESHOLD(LONG, 60000L), RESULT_MANAGER_MEMORY(INTEGER_BYTE_UNIT, -1), @SuppressWarnings("RedundantCast") // not redundant- false positive from IDEA - APP_CLASS(STRING, (Supplier<String>)() -> defaultAppClass), + APP_CLASS(STRING, (String) null), NCSERVICE_PID(INTEGER, -1), COMMAND(STRING, "hyracksnc"), - JVM_ARGS(STRING, (String)null), + JVM_ARGS(STRING, (String) null), VIRTUAL_NC(BOOLEAN, false); private final IOptionType parser; - private final Object defaultValue; + private final String defaultValueDescription; + private Object defaultValue; <T> Option(IOptionType<T> parser, Option defaultOption) { this.parser = parser; this.defaultValue = defaultOption; + defaultValueDescription = null; } <T> Option(IOptionType<T> parser, T defaultValue) { this.parser = parser; this.defaultValue = defaultValue; + defaultValueDescription = null; } - <T> Option(IOptionType<T> parser, Supplier<T> defaultValue) { + <T> Option(IOptionType<T> parser, Function<IApplicationConfig, T> defaultValue, + String defaultValueDescription) { this.parser = parser; this.defaultValue = defaultValue; + this.defaultValueDescription = defaultValueDescription; } @Override @@ -113,13 +117,13 @@ public String description() { switch (this) { case ADDRESS: - return "Default IP Address to bind listeners on this NC. All services will bind on this address " + - "unless a service-specific listen address is supplied."; + return "Default IP Address to bind listeners on this NC. All services will bind on this address " + + "unless a service-specific listen address is supplied."; case CLUSTER_LISTEN_ADDRESS: return "IP Address to bind cluster listener on this NC"; case PUBLIC_ADDRESS: - return "Default public address that other processes should use to contact this NC. All services " + - "will advertise this address unless a service-specific public address is supplied."; + return "Default public address that other processes should use to contact this NC. All services " + + "will advertise this address unless a service-specific public address is supplied."; case NCSERVICE_ADDRESS: return "Address the CC should use to contact the NCService associated with this NC"; case NCSERVICE_PORT: @@ -135,8 +139,8 @@ case CLUSTER_PUBLIC_PORT: return "Public IP port to announce cluster listener"; case NODE_ID: - return "Logical name of node controller unique within the cluster (required unless specified in " + - "config file)"; + return "Logical name of node controller unique within the cluster (required unless specified in " + + "config file)"; case DATA_LISTEN_ADDRESS: return "IP Address to bind data listener"; case DATA_LISTEN_PORT: @@ -170,11 +174,11 @@ case NET_BUFFER_COUNT: return "Number of network buffers per input/output channel"; case RESULT_TTL: - return "Limits the amount of time results for asynchronous jobs should be retained by the system " + - "in milliseconds"; + return "Limits the amount of time results for asynchronous jobs should be retained by the system " + + "in milliseconds"; case RESULT_SWEEP_THRESHOLD: - return "The duration within which an instance of the result cleanup should be invoked in " + - "milliseconds"; + return "The duration within which an instance of the result cleanup should be invoked in " + + "milliseconds"; case RESULT_MANAGER_MEMORY: return "Memory usable for result caching at this Node Controller in bytes"; case APP_CLASS: @@ -192,7 +196,6 @@ } } - @Override public IOptionType type() { return parser; @@ -203,10 +206,20 @@ return defaultValue; } + public void setDefaultValue(Object defaultValue) { + this.defaultValue = defaultValue; + } + @Override public boolean hidden() { return this == VIRTUAL_NC; } + + @Override + public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) { + return defaultValueDescription; + } + } private List<String> appArgs = new ArrayList<>(); -- To view, visit https://asterix-gerrit.ics.uci.edu/1751 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I7855c8f344eea9c9b6a394d85413a062a3ddb609 Gerrit-PatchSet: 7 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <mb...@apache.org> Gerrit-Reviewer: Michael Blow <mb...@apache.org> Gerrit-Reviewer: Till Westmann <ti...@apache.org>