Xikui Wang has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/2078
Change subject: [NO ISSUE][*DB] Enable loading external UDF for AsterixHyracksITUtil ...................................................................... [NO ISSUE][*DB] Enable loading external UDF for AsterixHyracksITUtil - user model changes: no - storage format changes: no - interface changes: no Add external UDF support for AsterixHyracksIntegrationUtil. Developer may use following system properties to install external UDF for local IT env. "external.lib" : udf package path. "external.lib.dataverse": dataverse name (default: test). "external.lib.libname": library name (default: testlib). Change-Id: I333238b944b1c6d48a2b80b87a25f600522e0cc4 --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java R asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java R asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java 7 files changed, 43 insertions(+), 20 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/78/2078/1 diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java index f5e94b1..678926e 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java @@ -29,11 +29,14 @@ import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.asterix.app.external.ExternalUDFLibrarian; import org.apache.asterix.common.api.IClusterManagementWork.ClusterState; +import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.config.PropertiesAccessor; import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.hyracks.bootstrap.CCApplication; import org.apache.asterix.hyracks.bootstrap.NCApplication; import org.apache.commons.io.FileUtils; @@ -90,7 +93,7 @@ ncApplication.registerConfig(ncConfigManager); nodeControllers.add( new NodeControllerService(fixupIODevices(createNCConfig(nodeId, ncConfigManager)), ncApplication)); - } ; + } cc.start(); @@ -119,6 +122,22 @@ ((ICcApplicationContext) cc.getApplicationContext()).getClusterStateManager().waitForState(ClusterState.ACTIVE); hcc = new HyracksConnection(cc.getConfig().getClientListenAddress(), cc.getConfig().getClientListenPort()); this.ncs = nodeControllers.toArray(new NodeControllerService[nodeControllers.size()]); + } + + public void initLocalInstance(boolean deleteOldInstanceData, String externalLibPath) throws Exception { + List<ILibraryManager> libraryManagers = new ArrayList<>(); + ExternalUDFLibrarian librarian = new ExternalUDFLibrarian(libraryManagers); + librarian.cleanup(); + init(deleteOldInstanceData); + if (externalLibPath != null && externalLibPath.length() != 0) { + libraryManagers.add(((ICcApplicationContext) cc.getApplicationContext()).getLibraryManager()); + for (NodeControllerService nc : ncs) { + INcApplicationContext runtimeCtx = (INcApplicationContext) nc.getApplicationContext(); + libraryManagers.add(runtimeCtx.getLibraryManager()); + } + librarian.install(System.getProperty("external.lib.dataverse", "test"), + System.getProperty("external.lib.libname", "testlib"), externalLibPath); + } } public ClusterControllerService getClusterControllerService() { @@ -263,14 +282,16 @@ public static void main(String[] args) throws Exception { AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil(); try { - integrationUtil.run(Boolean.getBoolean("cleanup.start"), Boolean.getBoolean("cleanup.shutdown")); + integrationUtil.runLocalInstance(Boolean.getBoolean("cleanup.start"), + Boolean.getBoolean("cleanup.shutdown"), System.getProperty("external.lib", "")); } catch (Exception e) { LOGGER.log(Level.WARNING, "Unexpected exception", e); System.exit(1); } } - protected void run(boolean cleanupOnStart, boolean cleanupOnShutdown) throws Exception { + protected void runLocalInstance(boolean cleanupOnStart, boolean cleanupOnShutdown, String loadExternalLibs) + throws Exception { Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { @@ -283,7 +304,7 @@ }); System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, "asterix-build-configuration.xml"); - init(cleanupOnStart); + initLocalInstance(cleanupOnStart, loadExternalLibs); while (true) { Thread.sleep(10000); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java index 5f86c28..4271d55 100755 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java @@ -346,7 +346,7 @@ if (jarsInLibDir.length > 1) { throw new Exception("Incorrect library structure: found multiple library jars"); } - if (jarsInLibDir.length < 0) { + if (jarsInLibDir.length <= 0) { throw new Exception("Incorrect library structure: could not find library jar"); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java similarity index 95% rename from asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java rename to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java index 76fa308..69b7e3c 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.asterix.app.external; import java.io.File; @@ -28,17 +29,16 @@ import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.event.service.AsterixEventServiceUtil; -import org.apache.asterix.test.common.ITestLibrarian; import org.apache.commons.io.FileUtils; import org.apache.hyracks.algebricks.common.utils.Pair; -public class TestLibrarian implements ITestLibrarian { +public class ExternalUDFLibrarian implements IExternalUDFLibrarian { // The following list includes a library manager for the CC // and library managers for NCs (one-per-NC). private final List<ILibraryManager> libraryManagers; - public TestLibrarian(List<ILibraryManager> libraryManagers) { + public ExternalUDFLibrarian(List<ILibraryManager> libraryManagers) { this.libraryManagers = libraryManagers; } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java similarity index 93% rename from asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java rename to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java index d661710..9b1d19d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java @@ -16,7 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.test.common; + +package org.apache.asterix.app.external; import java.io.IOException; import java.rmi.RemoteException; @@ -24,7 +25,7 @@ import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.exceptions.AsterixException; -public interface ITestLibrarian { +public interface IExternalUDFLibrarian { public void install(String dvName, String libName, String libPath) throws IOException, Exception; public void uninstall(String dvName, String libName) throws RemoteException, AsterixException, ACIDException; diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java index 104f80b..432e4f1 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java @@ -26,7 +26,7 @@ import java.util.Map; import java.util.logging.Logger; -import org.apache.asterix.app.external.TestLibrarian; +import org.apache.asterix.app.external.ExternalUDFLibrarian; import org.apache.asterix.app.nc.NCAppRuntimeContext; import org.apache.asterix.app.nc.TransactionSubsystem; import org.apache.asterix.common.config.DatasetConfig.IndexType; @@ -129,7 +129,7 @@ File outdir = new File(PATH_ACTUAL); outdir.mkdirs(); // remove library directory - TestLibrarian.removeLibraryDir(); + ExternalUDFLibrarian.removeLibraryDir(); ExecutionTestUtil.setUp(cleanupOnStart, testConfigFileName == null ? TEST_CONFIG_FILE_NAME : testConfigFileName, ExecutionTestUtil.integrationUtil, runHDFS); @@ -143,7 +143,7 @@ } public void deInit() throws Exception { - TestLibrarian.removeLibraryDir(); + ExternalUDFLibrarian.removeLibraryDir(); ExecutionTestUtil.tearDown(cleanupOnStop); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 4ca9131..0d66256 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -58,6 +58,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.asterix.app.external.IExternalUDFLibrarian; import org.apache.asterix.common.api.Duration; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.utils.Servlets; @@ -134,7 +135,7 @@ */ protected final List<InetSocketAddress> endpoints; protected int endpointSelector; - protected ITestLibrarian librarian; + protected IExternalUDFLibrarian librarian; private Map<File, TestLoop> testLoops = new HashMap<>(); public TestExecutor() { @@ -153,7 +154,7 @@ this.endpoints = endpoints; } - public void setLibrarian(ITestLibrarian librarian) { + public void setLibrarian(IExternalUDFLibrarian librarian) { this.librarian = librarian; } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java index 03f42f5..b0e5785 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java @@ -31,7 +31,7 @@ import java.util.Collection; import java.util.List; -import org.apache.asterix.app.external.TestLibrarian; +import org.apache.asterix.app.external.ExternalUDFLibrarian; import org.apache.asterix.common.config.ClusterProperties; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.test.common.TestExecutor; @@ -56,7 +56,7 @@ private static final List<String> badTestCases = new ArrayList<>(); private static TestExecutor testExecutor; - private static TestLibrarian librarian; + private static ExternalUDFLibrarian librarian; private static final int repeat = Integer.getInteger("test.repeat", 1); public static void setUp(String configFile, TestExecutor executor) throws Exception { @@ -64,8 +64,8 @@ File outdir = new File(PATH_ACTUAL); outdir.mkdirs(); List<ILibraryManager> libraryManagers = ExecutionTestUtil.setUp(cleanupOnStart, configFile); - TestLibrarian.removeLibraryDir(); - librarian = new TestLibrarian(libraryManagers); + ExternalUDFLibrarian.removeLibraryDir(); + librarian = new ExternalUDFLibrarian(libraryManagers); testExecutor.setLibrarian(librarian); if (repeat != 1) { System.out.println("FYI: each test will be run " + repeat + " times."); @@ -79,7 +79,7 @@ // Check whether there are leaked threads. checkThreadLeaks(); } finally { - TestLibrarian.removeLibraryDir(); + ExternalUDFLibrarian.removeLibraryDir(); ExecutionTestUtil.tearDown(cleanupOnStop); ExecutionTestUtil.integrationUtil.removeTestStorageFiles(); if (!badTestCases.isEmpty()) { -- To view, visit https://asterix-gerrit.ics.uci.edu/2078 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I333238b944b1c6d48a2b80b87a25f600522e0cc4 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang <xkk...@gmail.com>