This is an automated email from the ASF dual-hosted git repository. imaxon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new bdea5ef [ASTERIXDB-2781] Allow multi-options bdea5ef is described below commit bdea5ef2d07df9215c9b31c2aecfb3053bdcb542 Author: Ian Maxon <ima...@apache.org> AuthorDate: Mon Sep 21 20:53:43 2020 -0700 [ASTERIXDB-2781] Allow multi-options - Allow options that are array typed to be specified with multi-options in INI configs - Change PYTHON_ADDITIONAL_PKGS to be STRING_ARRAY Change-Id: I5ff9338524407e14a16640a08fc0abeb74a3ebde Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/8024 Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Dmitry Lychagin <dmitry.lycha...@couchbase.com> Reviewed-by: Michael Blow <mb...@apache.org> --- asterixdb/asterix-app/src/main/resources/cc.conf | 6 ++++-- asterixdb/asterix-app/src/test/resources/cc.conf | 3 ++- .../ExternalScalarPythonFunctionEvaluator.java | 8 +++----- .../control/common/config/ConfigManager.java | 24 ++++++++++++++++------ .../hyracks/control/common/config/ConfigUtils.java | 7 +++++++ .../hyracks/control/common/config/OptionTypes.java | 10 +++++++++ .../control/common/controllers/NCConfig.java | 2 +- 7 files changed, 45 insertions(+), 15 deletions(-) diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf index 9e9ac51..ccd35f8 100644 --- a/asterixdb/asterix-app/src/main/resources/cc.conf +++ b/asterixdb/asterix-app/src/main/resources/cc.conf @@ -18,7 +18,8 @@ [nc/asterix_nc1] txn.log.dir=target/tmp/asterix_nc1/txnlog core.dump.dir=target/tmp/asterix_nc1/coredump -iodevices=asterix_nc1/iodevice1,asterix_nc1/iodevice2 +iodevices=asterix_nc1/iodevice1 +iodevices=asterix_nc1/iodevice2 nc.api.port=19004 #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006 @@ -26,7 +27,8 @@ nc.api.port=19004 ncservice.port=9091 txn.log.dir=target/tmp/asterix_nc2/txnlog core.dump.dir=target/tmp/asterix_nc2/coredump -iodevices=asterix_nc2/iodevice1,asterix_nc2/iodevice2 +iodevices=asterix_nc2/iodevice1 +iodevices=asterix_nc1/iodevice2 nc.api.port=19005 #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007 diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf index a113e3a..158eaa0 100644 --- a/asterixdb/asterix-app/src/test/resources/cc.conf +++ b/asterixdb/asterix-app/src/test/resources/cc.conf @@ -18,7 +18,8 @@ [nc/asterix_nc1] txn.log.dir=target/tmp/asterix_nc1/txnlog core.dump.dir=target/tmp/asterix_nc1/coredump -iodevices=target/tmp/asterix_nc1/iodevice1,../asterix-server/target/tmp/asterix_nc1/iodevice2 +iodevices=target/tmp/asterix_nc1/iodevice1, +iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2 nc.api.port=19004 #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006 diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java index a5eccce..2353c6d 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java @@ -91,11 +91,9 @@ class ExternalScalarPythonFunctionEvaluator extends ExternalScalarFunctionEvalua File pythonPath = new File(pythonPathCmd); List<String> sitePkgs = new ArrayList<>(); sitePkgs.add(SITE_PACKAGES); - String addlSitePackagesRaw = - ctx.getServiceContext().getAppConfig().getString((NCConfig.Option.PYTHON_ADDITIONAL_PACKAGES)); - if (addlSitePackagesRaw != null) { - sitePkgs.addAll(Arrays.asList(addlSitePackagesRaw.split(File.pathSeparator))); - } + String[] addlSitePackages = + ctx.getServiceContext().getAppConfig().getStringArray((NCConfig.Option.PYTHON_ADDITIONAL_PACKAGES)); + sitePkgs.addAll(Arrays.asList(addlSitePackages)); if (cfg.getBoolean(NCConfig.Option.PYTHON_USE_BUNDLED_MSGPACK)) { sitePkgs.add("ipc" + File.separator + SITE_PACKAGES + File.separator); } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java index c175dd4..0ba9090 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java @@ -18,6 +18,8 @@ */ package org.apache.hyracks.control.common.config; +import static org.apache.hyracks.control.common.config.OptionTypes.COLLECTION_TYPES; + import java.io.IOException; import java.io.Serializable; import java.lang.reflect.Array; @@ -340,17 +342,27 @@ public class ConfigManager implements IConfigManager, Serializable { throw new HyracksException("Unknown section in ini: " + section.getName()); } Map<String, IOption> optionMap = getSectionOptionMap(rootSection); - for (Map.Entry<String, String> iniOption : section.entrySet()) { - String name = iniOption.getKey(); + for (String name : section.keySet()) { final IOption option = optionMap == null ? null : optionMap.get(name); if (option == null) { handleUnknownOption(section, name); return; } - final String value = iniOption.getValue(); - LOGGER.debug("setting " + option.toIniString() + " to " + value); - final Object parsed = option.type().parse(value); - invokeSetters(option, parsed, node); + final List<String> values = section.getAll(name); + if (values.size() <= 1) { + LOGGER.debug("setting " + option.toIniString() + " to " + values.get(0)); + final Object parsed = option.type().parse(values.get(0)); + invokeSetters(option, parsed, node); + } else { + if (option.type().targetType().isArray()) { + Object[] val = values.stream() + .map(v -> ((String) (COLLECTION_TYPES.get(option.type()).parse(v)))).toArray(); + invokeSetters(option, Arrays.copyOf(val, val.length, option.type().targetType()), node); + } else { + throw new HyracksException( + "Multiple option values specified for unary option" + option.toIniString()); + } + } } } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigUtils.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigUtils.java index 1cc739c..c95a3d1 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigUtils.java @@ -37,6 +37,7 @@ import org.apache.hyracks.api.config.IOption; import org.apache.hyracks.api.config.Section; import org.apache.hyracks.api.config.SerializedOption; import org.apache.hyracks.control.common.controllers.ControllerConfig; +import org.ini4j.Config; import org.ini4j.Ini; import org.kohsuke.args4j.CmdLineException; import org.kohsuke.args4j.CmdLineParser; @@ -88,6 +89,9 @@ public class ConfigUtils { public static Ini loadINIFile(String configFile) throws IOException { Ini ini = new Ini(); + Config conf = new Config(); + conf.setMultiOption(true); + ini.setConfig(conf); File conffile = new File(configFile); if (!conffile.exists()) { throw new FileNotFoundException(configFile); @@ -98,6 +102,9 @@ public class ConfigUtils { public static Ini loadINIFile(URL configURL) throws IOException { Ini ini = new Ini(); + Config conf = new Config(); + conf.setMultiOption(true); + ini.setConfig(conf); ini.load(configURL); return ini; } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java index 7088e08..0dcb7a4 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/OptionTypes.java @@ -20,7 +20,10 @@ package org.apache.hyracks.control.common.config; import java.net.MalformedURLException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Stream; import org.apache.hyracks.api.config.IOptionType; @@ -388,6 +391,13 @@ public class OptionTypes { } }; + static Map<IOptionType, IOptionType> COLLECTION_TYPES; + static { + Map<IOptionType, IOptionType> collTypes = new HashMap<>(); + collTypes.put(STRING_ARRAY, STRING); + COLLECTION_TYPES = Collections.unmodifiableMap(collTypes); + } + private OptionTypes() { } } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java index ce299b0..eaf0418 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java @@ -93,7 +93,7 @@ public class NCConfig extends ControllerConfig { IO_WORKERS_PER_PARTITION(POSITIVE_INTEGER, 2), IO_QUEUE_SIZE(POSITIVE_INTEGER, 10), PYTHON_CMD(STRING, (String) null), - PYTHON_ADDITIONAL_PACKAGES(STRING, (String) null), + PYTHON_ADDITIONAL_PACKAGES(STRING_ARRAY, (String[]) null), PYTHON_USE_BUNDLED_MSGPACK(BOOLEAN, true), PYTHON_ARGS(STRING_ARRAY, (String[]) null);