This is an automated email from the ASF dual-hosted git repository. imaxon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new f2c9308 [NO ISSUE] Fix UDF Metadata Transactions f2c9308 is described below commit f2c93085c81628eb9bc7e563259eaf9e4fe7a3bc Author: Ian Maxon <ian@maxons.email> AuthorDate: Wed Apr 22 01:43:28 2020 -0700 [NO ISSUE] Fix UDF Metadata Transactions - Move UDF Metadata Transactions to be initiated from the CC directly - Remove old UDF initialization code - Make DROP DATAVERSE remove libraries - Make NCs properly read libraries on init with multipart dataverse names Change-Id: Ibda23a2e8308937f343d80eff04ede9a235da0d2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/4923 Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Murtadha Hubail <mhub...@apache.org> --- .../asterix/api/http/server/UdfApiServlet.java | 166 ++++++++++++--- .../asterix/app/external/ExternalLibraryUtils.java | 225 +++------------------ .../asterix/app/message/AbstractUdfMessage.java | 8 +- .../asterix/app/message/DeleteUdfMessage.java | 6 +- .../apache/asterix/app/message/LoadUdfMessage.java | 4 +- .../app/nc/task/ExternalLibrarySetupTask.java | 2 +- .../asterix/app/translator/QueryTranslator.java | 14 ++ .../asterix/hyracks/bootstrap/CCApplication.java | 2 +- .../asterix/app/external/ExternalUDFLibrarian.java | 16 +- .../deterministic.5.ddl.sqlpp} | 0 .../getCapital.4.ddl.sqlpp} | 0 .../getCapital_open.4.ddl.sqlpp} | 0 .../getCapital_open/getCapital_open.4.lib.sqlpp | 19 -- ...ay_sum.5.ddl.sqlpp => my_array_sum.4.ddl.sqlpp} | 0 .../my_array_sum/my_array_sum.4.lib.sqlpp | 19 -- .../external-library/mysum/mysum.7.lib.sqlpp | 19 -- .../mysum_dropinuse.0.ddl.sqlpp} | 3 +- .../mysum_dropinuse.1.lib.sqlpp} | 2 +- .../mysum_dropinuse.2.ddl.sqlpp} | 3 +- .../mysum_dropinuse.3.query.sqlpp} | 3 +- .../mysum_dropinuse.4.lib.sqlpp} | 0 .../mysum_dropinuse.5.ddl.sqlpp} | 3 +- .../mysum_dropinuse.6.lib.sqlpp} | 0 .../mysum_dropinuse.7.ddl.sqlpp} | 0 ...ion.5.ddl.sqlpp => type_validation.4.ddl.sqlpp} | 0 .../udf_metadata.4.ddl.sqlpp} | 0 .../upperCase.4.ddl.sqlpp} | 2 +- ...lpp => feed-with-external-function.6.ddl.sqlpp} | 0 .../feed-with-external-function.6.lib.sqlpp | 19 -- .../mysum_dropinuse/mysum_dropinuse.1.adm | 1 + .../resources/runtimets/testsuite_it_sqlpp.xml | 6 + .../asterix/common/exceptions/ErrorCode.java | 1 + .../src/main/resources/asx_errormsg/en.properties | 1 + .../org/apache/asterix/metadata/MetadataNode.java | 5 + 34 files changed, 220 insertions(+), 329 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java index 54a62df..0f5d542 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java @@ -19,21 +19,34 @@ package org.apache.asterix.api.http.server; import java.io.File; +import java.io.PrintWriter; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.rmi.RemoteException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.ConcurrentMap; import org.apache.asterix.app.external.ExternalLibraryUtils; -import org.apache.asterix.app.message.DeleteUdfMessage; import org.apache.asterix.app.message.LoadUdfMessage; import org.apache.asterix.common.dataflow.ICcApplicationContext; +import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.messaging.api.ICCMessageBroker; import org.apache.asterix.common.messaging.api.INcAddressedMessage; import org.apache.asterix.common.metadata.DataverseName; +import org.apache.asterix.common.metadata.IMetadataLockUtil; +import org.apache.asterix.common.metadata.LockList; +import org.apache.asterix.metadata.MetadataManager; +import org.apache.asterix.metadata.MetadataTransactionContext; +import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.asterix.metadata.entities.DatasourceAdapter; +import org.apache.asterix.metadata.entities.Dataverse; +import org.apache.asterix.metadata.entities.Function; +import org.apache.asterix.metadata.entities.Library; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.api.deployment.DeploymentId; @@ -75,6 +88,8 @@ public class UdfApiServlet extends BasicAuthServlet { @Override protected void post(IServletRequest request, IServletResponse response) { + + PrintWriter responseWriter = response.writer(); FullHttpRequest req = request.getHttpRequest(); Pair<String, DataverseName> resourceNames; try { @@ -85,8 +100,17 @@ public class UdfApiServlet extends BasicAuthServlet { } String resourceName = resourceNames.first; DataverseName dataverse = resourceNames.second; + IMetadataLockUtil mdLockUtil = appCtx.getMetadataLockUtil(); + MetadataTransactionContext mdTxnCtx = null; + LockList mdLockList = null; File udf = null; try { + MetadataManager.INSTANCE.init(); + mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); + MetadataProvider metadataProvider = MetadataProvider.create(appCtx, null); + mdLockList = metadataProvider.getLocks(); + mdLockUtil.createLibraryBegin(appCtx.getMetadataLockManager(), metadataProvider.getLocks(), dataverse, + resourceName); File workingDir = new File(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(), UDF_TMP_DIR_PREFIX); if (!workingDir.exists()) { @@ -102,35 +126,104 @@ public class UdfApiServlet extends BasicAuthServlet { fc.write(content); } } - IHyracksClientConnection hcc = appCtx.getHcc(); - DeploymentId udfName = new DeploymentId(makeDeploymentId(dataverse, resourceName)); - ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(dataverse, resourceName); - if (cl != null) { - deleteUdf(dataverse, resourceName); - } - hcc.deployBinary(udfName, Arrays.asList(udf.toString()), true); - ExternalLibraryUtils.setUpExternaLibrary(appCtx.getLibraryManager(), false, - FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(), - "applications", udfName.toString())); - - long reqId = broker.newRequestId(); - List<INcAddressedMessage> requests = new ArrayList<>(); - List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes()); - ncs.forEach(s -> requests.add(new LoadUdfMessage(dataverse, resourceName, reqId))); - broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT); + setupBinariesAndClassloaders(dataverse, resourceName, udf); + installLibrary(mdTxnCtx, dataverse, resourceName); + MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); } catch (Exception e) { + try { + ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName); + } catch (Exception e2) { + e.addSuppressed(e2); + } response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); + responseWriter.write(e.getMessage()); + responseWriter.flush(); LOGGER.error(e); - return; - } finally { + if (mdTxnCtx != null) { + try { + MetadataManager.INSTANCE.abortTransaction(mdTxnCtx); + } catch (RemoteException r) { + LOGGER.error("Unable to abort metadata transaction", r); + } + } if (udf != null) { udf.delete(); } + return; + } finally { + if (mdLockList != null) { + mdLockList.unlock(); + } } response.setStatus(HttpResponseStatus.OK); } + private void setupBinariesAndClassloaders(DataverseName dataverse, String resourceName, File udf) throws Exception { + IHyracksClientConnection hcc = appCtx.getHcc(); + DeploymentId udfName = new DeploymentId(makeDeploymentId(dataverse, resourceName)); + ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(dataverse, resourceName); + if (cl != null) { + //prepare to replace the binary + ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName); + } + hcc.deployBinary(udfName, Arrays.asList(udf.toString()), true); + //setup for CC + ExternalLibraryUtils.setUpExternaLibrary(appCtx.getLibraryManager(), + FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(), + "applications", udfName.toString())); + //setup NCs + long reqId = broker.newRequestId(); + List<INcAddressedMessage> requests = new ArrayList<>(); + List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes()); + ncs.forEach(s -> requests.add(new LoadUdfMessage(dataverse, resourceName, reqId))); + broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT); + } + + private static void installLibrary(MetadataTransactionContext mdTxnCtx, DataverseName dataverse, String libraryName) + throws RemoteException, AlgebricksException { + Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName); + // Get the dataverse + Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse); + if (dv == null) { + throw new AsterixException(ErrorCode.UNKNOWN_DATAVERSE); + } + if (libraryInMetadata != null) { + //replacing binary, library already exists + return; + } + // Add library + MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName)); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Added library " + libraryName + " to Metadata"); + } + } + + private static void deleteLibrary(MetadataTransactionContext mdTxnCtx, DataverseName dataverse, String libraryName) + throws RemoteException, AlgebricksException { + Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse); + if (dv == null) { + throw new AsterixException(ErrorCode.UNKNOWN_DATAVERSE); + } + Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName); + if (library == null) { + throw new AsterixException(ErrorCode.UNKNOWN_LIBRARY); + } + List<Function> functions = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverse); + for (Function function : functions) { + if (libraryName.equals(function.getLibrary())) { + throw new AsterixException(ErrorCode.METADATA_DROP_LIBRARY_IN_USE, libraryName); + } + } + List<DatasourceAdapter> adapters = MetadataManager.INSTANCE.getDataverseAdapters(mdTxnCtx, dataverse); + for (DatasourceAdapter adapter : adapters) { + if (libraryName.equals(adapter.getLibrary())) { + throw new AsterixException(ErrorCode.METADATA_DROP_LIBRARY_IN_USE, libraryName); + } + } + MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName); + } + public static String makeDeploymentId(DataverseName dv, String resourceName) { List<String> dvParts = dv.getParts(); dvParts.add(resourceName); @@ -138,16 +231,6 @@ public class UdfApiServlet extends BasicAuthServlet { return dvWithLibrarySuffix.getCanonicalForm(); } - private void deleteUdf(DataverseName dataverse, String resourceName) throws Exception { - long reqId = broker.newRequestId(); - List<INcAddressedMessage> requests = new ArrayList<>(); - List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes()); - ncs.forEach(s -> requests.add(new DeleteUdfMessage(dataverse, resourceName, reqId))); - broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT); - appCtx.getLibraryManager().deregisterLibraryClassLoader(dataverse, resourceName); - appCtx.getHcc().unDeployBinary(new DeploymentId(makeDeploymentId(dataverse, resourceName))); - } - @Override protected void delete(IServletRequest request, IServletResponse response) { Pair<String, DataverseName> resourceNames; @@ -157,14 +240,37 @@ public class UdfApiServlet extends BasicAuthServlet { response.setStatus(HttpResponseStatus.BAD_REQUEST); return; } + PrintWriter responseWriter = response.writer(); String resourceName = resourceNames.first; DataverseName dataverse = resourceNames.second; + IMetadataLockUtil mdLockUtil = appCtx.getMetadataLockUtil(); + MetadataTransactionContext mdTxnCtx = null; + LockList mdLockList = null; try { - deleteUdf(dataverse, resourceName); + MetadataManager.INSTANCE.init(); + mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); + MetadataProvider metadataProvider = MetadataProvider.create(appCtx, null); + mdLockList = metadataProvider.getLocks(); + mdLockUtil.dropLibraryBegin(appCtx.getMetadataLockManager(), metadataProvider.getLocks(), dataverse, + resourceName); + deleteLibrary(mdTxnCtx, dataverse, resourceName); + ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName); + MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); } catch (Exception e) { + try { + MetadataManager.INSTANCE.abortTransaction(mdTxnCtx); + } catch (RemoteException r) { + LOGGER.error("Unable to abort metadata transaction", r); + } response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); + responseWriter.write(e.getMessage()); + responseWriter.flush(); LOGGER.error(e); return; + } finally { + if (mdLockList != null) { + mdLockList.unlock(); + } } response.setStatus(HttpResponseStatus.OK); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java index 758bd45..47a460e 100755 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java @@ -18,31 +18,23 @@ */ package org.apache.asterix.app.external; +import static org.apache.asterix.api.http.server.UdfApiServlet.UDF_RESPONSE_TIMEOUT; +import static org.apache.asterix.api.http.server.UdfApiServlet.makeDeploymentId; + import java.io.File; import java.io.FilenameFilter; -import java.io.IOException; import java.net.URL; import java.net.URLClassLoader; -import java.rmi.RemoteException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import org.apache.asterix.common.exceptions.ACIDException; -import org.apache.asterix.common.exceptions.AsterixException; -import org.apache.asterix.common.functions.FunctionSignature; +import org.apache.asterix.app.message.DeleteUdfMessage; +import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.library.ILibraryManager; +import org.apache.asterix.common.messaging.api.ICCMessageBroker; +import org.apache.asterix.common.messaging.api.INcAddressedMessage; import org.apache.asterix.common.metadata.DataverseName; -import org.apache.asterix.metadata.MetadataManager; -import org.apache.asterix.metadata.MetadataTransactionContext; -import org.apache.asterix.metadata.entities.DatasourceAdapter; -import org.apache.asterix.metadata.entities.Dataverse; -import org.apache.asterix.metadata.entities.Function; -import org.apache.asterix.metadata.entities.Library; -import org.apache.asterix.metadata.utils.MetadataUtil; -import org.apache.asterix.runtime.formats.NonTaggedDataFormat; -import org.apache.logging.log4j.Level; +import org.apache.hyracks.api.deployment.DeploymentId; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -54,30 +46,17 @@ public class ExternalLibraryUtils { private ExternalLibraryUtils() { } - public static void setUpExternaLibrary(ILibraryManager externalLibraryManager, boolean isMetadataNode, - String libraryPath) throws Exception { - // start by un-installing removed libraries (Metadata Node only) - Map<DataverseName, List<String>> uninstalledLibs = null; - if (isMetadataNode) { - uninstalledLibs = uninstallLibraries(); - } - - // get the directory of the to be installed libraries - String[] pathSplit = libraryPath.split("\\."); - String[] dvSplit = pathSplit[pathSplit.length - 2].split("/"); - DataverseName dataverse = DataverseName.createSinglePartName(dvSplit[dvSplit.length - 1]); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT - String name = pathSplit[pathSplit.length - 1].trim(); - File installLibDir = new File(libraryPath); - - // directory exists? - if (installLibDir.exists()) { - registerClassLoader(externalLibraryManager, dataverse, name, libraryPath); - configureLibrary(externalLibraryManager, dataverse, name, installLibDir, uninstalledLibs, isMetadataNode); - } + public static void setUpExternaLibrary(ILibraryManager externalLibraryManager, String libraryPath) + throws Exception { + // get the installed library dirs + String[] parts = libraryPath.split(File.separator); + DataverseName catenatedDv = DataverseName.createFromCanonicalForm(parts[parts.length - 1]); + String libraryName = catenatedDv.getParts().get(catenatedDv.getParts().size() - 1); + DataverseName dvName = DataverseName.create(catenatedDv.getParts(), 0, catenatedDv.getParts().size() - 1); + registerClassLoader(externalLibraryManager, dvName, libraryName, libraryPath); } - public static void setUpInstalledLibraries(ILibraryManager externalLibraryManager, boolean isMetadataNode, - File appDir) throws Exception { + public static void setUpInstalledLibraries(ILibraryManager externalLibraryManager, File appDir) throws Exception { File[] libs = appDir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { @@ -86,170 +65,20 @@ public class ExternalLibraryUtils { }); if (libs != null) { for (File lib : libs) { - setUpExternaLibrary(externalLibraryManager, isMetadataNode, lib.getAbsolutePath()); - } - } - } - - /** - * un-install libraries. - * - * @return a map from dataverse -> list of uninstalled libraries. - * @throws Exception - */ - private static Map<DataverseName, List<String>> uninstallLibraries() throws Exception { - Map<DataverseName, List<String>> uninstalledLibs = new HashMap<>(); - // get the directory of the un-install libraries - File uninstallLibDir = getLibraryUninstallDir(); - String[] uninstallLibNames; - // directory exists? - if (uninstallLibDir.exists()) { - // list files - uninstallLibNames = uninstallLibDir.list(nonHiddenFileNameFilter); - for (String uninstallLibName : uninstallLibNames) { - // Get the <dataverse name - library name> pair - String[] components = uninstallLibName.split("\\."); - DataverseName dataverse = DataverseName.createSinglePartName(components[0]); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT - String libName = components[1]; - // un-install - uninstallLibrary(dataverse, libName); - // delete the library file - new File(uninstallLibDir, uninstallLibName).delete(); - // add the library to the list of uninstalled libraries - List<String> uinstalledLibsInDv = uninstalledLibs.get(dataverse); - if (uinstalledLibsInDv == null) { - uinstalledLibsInDv = new ArrayList<>(); - uninstalledLibs.put(dataverse, uinstalledLibsInDv); - } - uinstalledLibsInDv.add(libName); + setUpExternaLibrary(externalLibraryManager, lib.getAbsolutePath()); } } - return uninstalledLibs; } - /** - * Remove the library from metadata completely. - * TODO Currently, external libraries only include functions and adapters. we need to extend this to include: - * 1. external data source - * 2. data parser - * - * @param dataverse - * @param libraryName - * @return true if the library was found and removed, false otherwise - * @throws AsterixException - * @throws RemoteException - * @throws ACIDException - */ - public static boolean uninstallLibrary(DataverseName dataverse, String libraryName) - throws AsterixException, RemoteException, ACIDException { - MetadataTransactionContext mdTxnCtx = null; - try { - // begin transaction - mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); - // make sure dataverse exists - Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse); - if (dv == null) { - return false; - } - // make sure library exists - Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName); - if (library == null) { - return false; - } - - // get dataverse functions - List<Function> functions = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverse); - for (Function function : functions) { - // does function belong to library? - if (function.getName().startsWith(libraryName + "#")) { - // drop the function - MetadataManager.INSTANCE.dropFunction(mdTxnCtx, - new FunctionSignature(dataverse, function.getName(), function.getArity())); - } - } - - // get the dataverse adapters - List<DatasourceAdapter> adapters = MetadataManager.INSTANCE.getDataverseAdapters(mdTxnCtx, dataverse); - for (DatasourceAdapter adapter : adapters) { - // belong to the library? - if (adapter.getAdapterIdentifier().getName().startsWith(libraryName + "#")) { - // remove adapter <! we didn't check if there are feeds which use this adapter> - MetadataManager.INSTANCE.dropAdapter(mdTxnCtx, dataverse, adapter.getAdapterIdentifier().getName()); - } - } - // drop the library itself - MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName); - MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); - } catch (Exception e) { - MetadataManager.INSTANCE.abortTransaction(mdTxnCtx); - throw new AsterixException(e); - } - return true; - } - - private static void addLibraryToMetadata(Map<DataverseName, List<String>> uninstalledLibs, DataverseName dataverse, - String libraryName) throws ACIDException, RemoteException { - // Modify metadata accordingly - List<String> uninstalledLibsInDv = uninstalledLibs.get(dataverse); - // was this library just un-installed? - boolean wasUninstalled = uninstalledLibsInDv != null && uninstalledLibsInDv.contains(libraryName); - MetadataTransactionContext mdTxnCtx = null; - try { - mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); - Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName); - if (libraryInMetadata != null && !wasUninstalled) { - // exists in metadata and was not un-installed, we return. - // Another place which shows that our metadata transactions are broken - // (we didn't call commit before!!!) - MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); - return; - } - - // Add library - MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName)); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("Added library " + libraryName + " to Metadata"); - } - - // Get the dataverse - Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse); - if (dv == null) { - MetadataManager.INSTANCE.addDataverse(mdTxnCtx, new Dataverse(dataverse, - NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT, MetadataUtil.PENDING_NO_OP)); - } - - MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); - } catch (Exception e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.log(Level.ERROR, "Exception in installing library " + libraryName, e); - } - MetadataManager.INSTANCE.abortTransaction(mdTxnCtx); - } - } - - /** - * Each element of a library is installed as part of a transaction. Any - * failure in installing an element does not effect installation of other - * libraries. - */ - protected static void configureLibrary(ILibraryManager libraryManager, DataverseName dataverse, String libraryName, - final File libraryDir, Map<DataverseName, List<String>> uninstalledLibs, boolean isMetadataNode) - throws Exception { - - String[] libraryDescriptors = libraryDir.list((dir, name) -> name.endsWith(".xml")); - - if (libraryDescriptors == null) { - throw new IOException("Unable to list files in directory " + libraryDir); - } - - if (libraryDescriptors.length > 1) { - throw new IllegalStateException("More than 1 library descriptors defined"); - } - - // Prepare possible parameters - if (isMetadataNode) { - addLibraryToMetadata(uninstalledLibs, dataverse, libraryName); - } + public static void deleteDeployedUdf(ICCMessageBroker broker, ICcApplicationContext appCtx, + DataverseName dataverseName, String lib) throws Exception { + long reqId = broker.newRequestId(); + List<INcAddressedMessage> requests = new ArrayList<>(); + List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes()); + ncs.forEach(s -> requests.add(new DeleteUdfMessage(dataverseName, lib, reqId))); + broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT); + appCtx.getLibraryManager().deregisterLibraryClassLoader(dataverseName, lib); + appCtx.getHcc().unDeployBinary(new DeploymentId(makeDeploymentId(dataverseName, lib))); } /** diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java index 1c212fe..b8c77e7 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java @@ -46,12 +46,9 @@ public abstract class AbstractUdfMessage extends CcIdentifiedMessage implements @Override public void handle(INcApplicationContext appCtx) { ILibraryManager mgr = appCtx.getLibraryManager(); - String mdNodeName = appCtx.getMetadataProperties().getMetadataNodeName(); - String nodeName = appCtx.getServiceContext().getNodeId(); INCMessageBroker broker = (INCMessageBroker) appCtx.getServiceContext().getMessageBroker(); - boolean isMdNode = mdNodeName.equals(nodeName); try { - handleAction(mgr, isMdNode, appCtx); + handleAction(mgr, appCtx); broker.sendMessageToCC(getCcId(), new UdfResponseMessage(reqId, null)); } catch (Exception e) { try { @@ -64,7 +61,6 @@ public abstract class AbstractUdfMessage extends CcIdentifiedMessage implements } - protected abstract void handleAction(ILibraryManager mgr, boolean isMdNode, INcApplicationContext appCtx) - throws Exception; + protected abstract void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) throws Exception; } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java index 3fb58fa..7bcfa9b 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java @@ -18,7 +18,6 @@ */ package org.apache.asterix.app.message; -import org.apache.asterix.app.external.ExternalLibraryUtils; import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.common.metadata.DataverseName; @@ -32,10 +31,7 @@ public class DeleteUdfMessage extends AbstractUdfMessage { } @Override - protected void handleAction(ILibraryManager mgr, boolean isMdNode, INcApplicationContext appCtx) throws Exception { - if (isMdNode) { - ExternalLibraryUtils.uninstallLibrary(dataverseName, libraryName); - } + protected void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) { mgr.deregisterLibraryClassLoader(dataverseName, libraryName); } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java index 6dffb28..7ccdfd4 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java @@ -33,8 +33,8 @@ public class LoadUdfMessage extends AbstractUdfMessage { } @Override - protected void handleAction(ILibraryManager mgr, boolean isMdNode, INcApplicationContext appCtx) throws Exception { - ExternalLibraryUtils.setUpExternaLibrary(mgr, isMdNode, + protected void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) throws Exception { + ExternalLibraryUtils.setUpExternaLibrary(mgr, FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(), "applications", dataverseName.getCanonicalForm() + "." + libraryName)); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java index 1ca2b78..40262ac 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java @@ -38,7 +38,7 @@ public class ExternalLibrarySetupTask implements INCLifecycleTask { public void perform(CcId ccId, IControllerService cs) throws HyracksDataException { INcApplicationContext appContext = (INcApplicationContext) cs.getApplicationContext(); try { - ExternalLibraryUtils.setUpInstalledLibraries(appContext.getLibraryManager(), metadataNode, + ExternalLibraryUtils.setUpInstalledLibraries(appContext.getLibraryManager(), cs.getContext().getServerCtx().getAppDir()); } catch (Exception e) { throw HyracksDataException.create(e); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 16ca37f..814c96e 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -50,6 +50,7 @@ import org.apache.asterix.api.http.server.ApiServlet; import org.apache.asterix.app.active.ActiveEntityEventsListener; import org.apache.asterix.app.active.ActiveNotificationHandler; import org.apache.asterix.app.active.FeedEventsListener; +import org.apache.asterix.app.external.ExternalLibraryUtils; import org.apache.asterix.app.result.ExecutionError; import org.apache.asterix.app.result.ResultHandle; import org.apache.asterix.app.result.ResultReader; @@ -78,6 +79,7 @@ import org.apache.asterix.common.exceptions.RuntimeDataException; import org.apache.asterix.common.exceptions.WarningCollector; import org.apache.asterix.common.exceptions.WarningUtil; import org.apache.asterix.common.functions.FunctionSignature; +import org.apache.asterix.common.messaging.api.ICCMessageBroker; import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.common.metadata.IMetadataLockUtil; import org.apache.asterix.common.utils.JobUtils; @@ -1307,6 +1309,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen boolean bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); List<JobSpecification> jobsToExecute = new ArrayList<>(); + List<Library> librariesToDelete = new ArrayList<>(); + ICCMessageBroker broker = (ICCMessageBroker) appCtx.getServiceContext().getMessageBroker(); try { Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName); if (dv == null) { @@ -1326,6 +1330,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen function.getDataverseName() + "." + function.getName() + "@" + function.getArity()); } } + MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; // # disconnect all feeds from any datasets in the dataverse. @@ -1381,6 +1386,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } jobsToExecute.add(DataverseUtil.dropDataverseJobSpec(dv, metadataProvider)); + librariesToDelete = MetadataManager.INSTANCE.getDataverseLibraries(mdTxnCtx, dataverseName); // #. mark PendingDropOp on the dataverse record by // first, deleting the dataverse record from the DATAVERSE_DATASET // second, inserting the dataverse record with the PendingDropOp value into the @@ -1397,6 +1403,10 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen runJob(hcc, jobSpec); } + for (Library lib : librariesToDelete) { + ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverseName, lib.getName()); + } + mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); @@ -1434,6 +1444,10 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen for (JobSpecification jobSpec : jobsToExecute) { runJob(hcc, jobSpec); } + + for (Library lib : librariesToDelete) { + ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverseName, lib.getName()); + } } catch (Exception e2) { // do no throw exception since still the metadata needs to be compensated. e.addSuppressed(e2); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java index bc10d4f..ae85381 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java @@ -157,7 +157,7 @@ public class CCApplication extends BaseCCApplication { ReplicationProperties repProp = new ReplicationProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig())); INcLifecycleCoordinator lifecycleCoordinator = createNcLifeCycleCoordinator(repProp.isReplicationEnabled()); - ExternalLibraryUtils.setUpInstalledLibraries(libraryManager, false, ccServiceCtx.getServerCtx().getAppDir()); + ExternalLibraryUtils.setUpInstalledLibraries(libraryManager, ccServiceCtx.getServerCtx().getAppDir()); componentProvider = new StorageComponentProvider(); ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions())); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java index fd40a01..0c3ac81 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.URL; import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.commons.io.IOUtils; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.auth.AuthScope; @@ -39,7 +40,6 @@ import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.hyracks.algebricks.common.utils.Pair; -import org.apache.hyracks.api.exceptions.HyracksException; @SuppressWarnings("squid:S134") public class ExternalUDFLibrarian implements IExternalUDFLibrarian { @@ -75,7 +75,7 @@ public class ExternalUDFLibrarian implements IExternalUDFLibrarian { HttpResponse response = hc.execute(post, hcCtx); response.getEntity().consumeContent(); if (response.getStatusLine().getStatusCode() != 200) { - throw new HyracksException(response.getStatusLine().toString()); + throw new AsterixException(response.getStatusLine().toString()); } } @@ -93,9 +93,17 @@ public class ExternalUDFLibrarian implements IExternalUDFLibrarian { hcCtx.setAuthCache(ac); HttpDelete del = new HttpDelete(url.toString()); HttpResponse response = hc.execute(del, hcCtx); + String resp = null; + int respCode = response.getStatusLine().getStatusCode(); + if (respCode == 500) { + resp = IOUtils.toString(response.getEntity().getContent()); + } response.getEntity().consumeContent(); - if (response.getStatusLine().getStatusCode() != 200) { - throw new AsterixException(response.getStatusLine().toString()); + if (resp == null && respCode != 200) { + resp = response.getStatusLine().toString(); + } + if (resp != null) { + throw new AsterixException(resp); } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.ddl.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.5.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.5.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.ddl.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.5.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.lib.sqlpp deleted file mode 100644 index 41880cb..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.lib.sqlpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -uninstall externallibtest testlib admin admin diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.ddl.sqlpp similarity index 100% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.lib.sqlpp deleted file mode 100644 index 41880cb..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.lib.sqlpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -uninstall externallibtest testlib admin admin diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.lib.sqlpp deleted file mode 100644 index 41880cb..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.lib.sqlpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -uninstall externallibtest testlib admin admin diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.0.ddl.sqlpp similarity index 91% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.0.ddl.sqlpp index 172bed4..76cc70d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.0.ddl.sqlpp @@ -16,4 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -uninstall externallibtest testlib admin admin \ No newline at end of file +DROP DATAVERSE externallibtest if exists; +CREATE DATAVERSE externallibtest; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp similarity index 88% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.lib.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp index 41880cb..3dc6eb6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -uninstall externallibtest testlib admin admin +install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.2.ddl.sqlpp similarity index 83% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.2.ddl.sqlpp index e44b339..fd815d1 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.2.ddl.sqlpp @@ -16,5 +16,6 @@ * specific language governing permissions and limitations * under the License. */ + USE externallibtest; -uninstall externallibtest testlib admin admin +create function mysum(a: int32, b: int32) returns int32 language java as "testlib","org.apache.asterix.external.library.MySumFactory"; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.3.query.sqlpp similarity index 95% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.3.query.sqlpp index 2b27030..a23c197 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.3.query.sqlpp @@ -16,5 +16,6 @@ * specific language governing permissions and limitations * under the License. */ +use externallibtest; -DROP DATAVERSE externallibtest; +mysum(9.7, 4); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.4.lib.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.lib.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.4.lib.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.5.ddl.sqlpp similarity index 92% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.5.ddl.sqlpp index e44b339..f8e2ae7 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.5.ddl.sqlpp @@ -16,5 +16,6 @@ * specific language governing permissions and limitations * under the License. */ + USE externallibtest; -uninstall externallibtest testlib admin admin +drop function externallibtest.mysum@2; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.6.lib.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.lib.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.6.lib.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.7.ddl.sqlpp similarity index 100% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.7.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.ddl.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.ddl.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.6.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.ddl.sqlpp similarity index 96% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.ddl.sqlpp index 65733e4..1a46de9 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.ddl.sqlpp @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -DROP DATAVERSE externallibtest; + drop dataverse externallibtest; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.7.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.ddl.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.7.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp deleted file mode 100644 index 6b2fb1f..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -uninstall udfs testlib admin admin diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysum_dropinuse/mysum_dropinuse.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysum_dropinuse/mysum_dropinuse.1.adm new file mode 100644 index 0000000..ca7bf83 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysum_dropinuse/mysum_dropinuse.1.adm @@ -0,0 +1 @@ +13 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml index 91d4969..b6cf0cc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml @@ -44,6 +44,12 @@ </compilation-unit> </test-case> <test-case FilePath="external-library"> + <compilation-unit name="mysum_dropinuse"> + <output-dir compare="Text">mysum_dropinuse</output-dir> + <expected-error>Library testlib is being used. It cannot be dropped</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="external-library"> <compilation-unit name="my_array_sum"> <output-dir compare="Text">my_array_sum</output-dir> </compilation-unit> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 3d30360..7fef69e 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -314,6 +314,7 @@ public class ErrorCode { public static final int FAILED_TO_PARSE_METADATA = 3115; public static final int INPUT_DECODE_FAILURE = 3116; public static final int FAILED_TO_PARSE_MALFORMED_LOG_RECORD = 3117; + public static final int METADATA_DROP_LIBRARY_IN_USE = 3118; // Lifecycle management errors public static final int DUPLICATE_PARTITION_ID = 4000; diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index fc356e9..4461ff1 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -312,6 +312,7 @@ 3115 = Failed to parse record metadata 3116 = Failed to decode input 3117 = Failed to parse record, malformed log record +3118 = Library %1$s is being used. It cannot be dropped # Lifecycle management errors 4000 = Partition id %1$s for node %2$s already in use by node %3$s diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java index cf7562f..ca98328 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java @@ -534,6 +534,11 @@ public class MetadataNode implements IMetadataNode { true); } + //Drop libraries, similarly. + for (Library lib : getDataverseLibraries(txnId, dataverseName)) { + dropLibrary(txnId, lib.getDataverseName(), lib.getName()); + } + List<Dataset> dataverseDatasets; Dataset ds; dataverseDatasets = getDataverseDatasets(txnId, dataverseName);