This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new bdea5ef [ASTERIXDB-2781] Allow multi-options
bdea5ef is described below
commit bdea5ef2d07df9215c9b31c2aecfb3053bdcb542
Author: Ian Maxon <ima...@apache.org>
AuthorDate: Mon Sep 21 20:53:43 2020 -0700
[ASTERIXDB-2781] Allow multi-options
- Allow options that are array typed to be specified with
multi-options in INI configs
- Change PYTHON_ADDITIONAL_PKGS to be STRING_ARRAY
Change-Id: I5ff9338524407e14a16640a08fc0abeb74a3ebde
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/8024
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lycha...@couchbase.com>
Reviewed-by: Michael Blow <mb...@apache.org>
---
asterixdb/asterix-app/src/main/resources/cc.conf | 6 ++++--
asterixdb/asterix-app/src/test/resources/cc.conf | 3 ++-
.../ExternalScalarPythonFunctionEvaluator.java | 8 +++-----
.../control/common/config/ConfigManager.java | 24 ++++++++++++++++------
.../hyracks/control/common/config/ConfigUtils.java | 7 +++++++
.../hyracks/control/common/config/OptionTypes.java | 10 +++++++++
.../control/common/controllers/NCConfig.java | 2 +-
7 files changed, 45 insertions(+), 15 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf
b/asterixdb/asterix-app/src/main/resources/cc.conf
index 9e9ac51..ccd35f8 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -18,7 +18,8 @@
[nc/asterix_nc1]
txn.log.dir=target/tmp/asterix_nc1/txnlog
core.dump.dir=target/tmp/asterix_nc1/coredump
-iodevices=asterix_nc1/iodevice1,asterix_nc1/iodevice2
+iodevices=asterix_nc1/iodevice1
+iodevices=asterix_nc1/iodevice2
nc.api.port=19004
#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
@@ -26,7 +27,8 @@ nc.api.port=19004
ncservice.port=9091
txn.log.dir=target/tmp/asterix_nc2/txnlog
core.dump.dir=target/tmp/asterix_nc2/coredump
-iodevices=asterix_nc2/iodevice1,asterix_nc2/iodevice2
+iodevices=asterix_nc2/iodevice1
+iodevices=asterix_nc1/iodevice2
nc.api.port=19005
#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf
b/asterixdb/asterix-app/src/test/resources/cc.conf
index a113e3a..158eaa0 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -18,7 +18,8 @@
[nc/asterix_nc1]
txn.log.dir=target/tmp/asterix_nc1/txnlog
core.dump.dir=target/tmp/asterix_nc1/coredump
-iodevices=target/tmp/asterix_nc1/iodevice1,../asterix-server/target/tmp/asterix_nc1/iodevice2
+iodevices=target/tmp/asterix_nc1/iodevice1,
+iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2
nc.api.port=19004
#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
index a5eccce..2353c6d 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
@@ -91,11 +91,9 @@ class ExternalScalarPythonFunctionEvaluator extends
ExternalScalarFunctionEvalua
File pythonPath = new File(pythonPathCmd);
List<String> sitePkgs = new ArrayList<>();
sitePkgs.add(SITE_PACKAGES);
- String addlSitePackagesRaw =
-
ctx.getServiceContext().getAppConfig().getString((NCConfig.Option.PYTHON_ADDITIONAL_PACKAGES));
- if (addlSitePackagesRaw != null) {
-
sitePkgs.addAll(Arrays.asList(addlSitePackagesRaw.split(File.pathSeparator)));
- }
+ String[] addlSitePackages =
+
ctx.getServiceContext().getAppConfig().getStringArray((NCConfig.Option.PYTHON_ADDITIONAL_PACKAGES));
+ sitePkgs.addAll(Arrays.asList(addlSitePackages));
if (cfg.getBoolean(NCConfig.Option.PYTHON_USE_BUNDLED_MSGPACK)) {
sitePkgs.add("ipc" + File.separator + SITE_PACKAGES +
File.separator);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
index c175dd4..0ba9090 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.control.common.config;
+import static
org.apache.hyracks.control.common.config.OptionTypes.COLLECTION_TYPES;
+
import java.io.IOException;
import java.io.Serializable;
import java.lang.reflect.Array;
@@ -340,17 +342,27 @@ public class ConfigManager implements IConfigManager,
Serializable {
throw new HyracksException("Unknown section in ini: " +
section.getName());
}
Map<String, IOption> optionMap = getSectionOptionMap(rootSection);
- for (Map.Entry<String, String> iniOption : section.entrySet()) {
- String name = iniOption.getKey();
+ for (String name : section.keySet()) {
final IOption option = optionMap == null ? null :
optionMap.get(name);
if (option == null) {
handleUnknownOption(section, name);
return;
}
- final String value = iniOption.getValue();
- LOGGER.debug("setting " + option.toIniString() + " to " +
value);
- final Object parsed = option.type().parse(value);
- invokeSetters(option, parsed, node);
+ final List<String> values = section.getAll(name);
+ if (values.size() <= 1) {
+ LOGGER.debug("setting " + option.toIniString() + " to " +
values.get(0));
+ final Object parsed = option.type().parse(values.get(0));
+ invokeSetters(option, parsed, node);
+ } else {
+ if (option.type().targetType().isArray()) {
+ Object[] val = values.stream()
+ .map(v -> ((String)
(COLLECTION_TYPES.get(option.type()).parse(v)))).toArray();
+ invokeSetters(option, Arrays.copyOf(val, val.length,
option.type().targetType()), node);
+ } else {
+ throw new HyracksException(
+ "Multiple option values specified for unary
option" + option.toIniString());
+ }
+ }
}
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigUtils.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigUtils.java
index 1cc739c..c95a3d1 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigUtils.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigUtils.java
@@ -37,6 +37,7 @@ import org.apache.hyracks.api.config.IOption;
import org.apache.hyracks.api.config.Section;
import org.apache.hyracks.api.config.SerializedOption;
import org.apache.hyracks.control.common.controllers.ControllerConfig;
+import org.ini4j.Config;
import org.ini4j.Ini;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
@@ -88,6 +89,9 @@ public class ConfigUtils {
public static Ini loadINIFile(String configFile) throws IOException {
Ini ini = new Ini();
+ Config conf = new Config();
+ conf.setMultiOption(true);
+ ini.setConfig(conf);
File conffile = new File(configFile);
if (!conffile.exists()) {
throw new FileNotFoundException(configFile);
@@ -98,6 +102,9 @@ public class ConfigUtils {
public static Ini loadINIFile(URL configURL) throws IOException {
Ini ini = new Ini();
+ Config conf = new Config();
+ conf.setMultiOption(true);
+ ini.setConfig(conf);
ini.load(configURL);
return ini;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java
index 7088e08..0dcb7a4 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java
@@ -20,7 +20,10 @@ package org.apache.hyracks.control.common.config;
import java.net.MalformedURLException;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.stream.Stream;
import org.apache.hyracks.api.config.IOptionType;
@@ -388,6 +391,13 @@ public class OptionTypes {
}
};
+ static Map<IOptionType, IOptionType> COLLECTION_TYPES;
+ static {
+ Map<IOptionType, IOptionType> collTypes = new HashMap<>();
+ collTypes.put(STRING_ARRAY, STRING);
+ COLLECTION_TYPES = Collections.unmodifiableMap(collTypes);
+ }
+
private OptionTypes() {
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index ce299b0..eaf0418 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -93,7 +93,7 @@ public class NCConfig extends ControllerConfig {
IO_WORKERS_PER_PARTITION(POSITIVE_INTEGER, 2),
IO_QUEUE_SIZE(POSITIVE_INTEGER, 10),
PYTHON_CMD(STRING, (String) null),
- PYTHON_ADDITIONAL_PACKAGES(STRING, (String) null),
+ PYTHON_ADDITIONAL_PACKAGES(STRING_ARRAY, (String[]) null),
PYTHON_USE_BUNDLED_MSGPACK(BOOLEAN, true),
PYTHON_ARGS(STRING_ARRAY, (String[]) null);
