abdullah alamoudi has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/1017
Change subject: Add Asterix Extension Manager ...................................................................... Add Asterix Extension Manager More extension support is added. A user can now provide implementations for the IExtension interface which will give them more control over the behavior of the system and give them the ability to add custom features. Change-Id: I280268495cc3aad00f898cba21f7299f7120ce5c --- M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java M asterixdb/asterix-app/pom.xml M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java A asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/ExtensionManager.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java A asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtensionProperties.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ServletUtil.java M asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd M asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java A asterixdb/asterix-extension/pom.xml A asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/ExtensionId.java A asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/IExtension.java A asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/IExtensionStatement.java M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java M asterixdb/pom.xml 19 files changed, 417 insertions(+), 92 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/17/1017/1 diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index 1894cc3..9f7dc2f 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -278,7 +278,6 @@ return new ALogicalPlanImpl(new MutableObject<>(leafOperator)); } - @SuppressWarnings("unchecked") @Override public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt) throws AlgebricksException { diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml index 7d0a9ba..9cb68d8 100644 --- a/asterixdb/asterix-app/pom.xml +++ b/asterixdb/asterix-app/pom.xml @@ -317,6 +317,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-extension</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-test-support</artifactId> <version>0.2.18-SNAPSHOT</version> diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java index 4bb09d4..a2b5917 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java @@ -26,6 +26,7 @@ import java.util.logging.Logger; import org.apache.asterix.active.ActiveManager; +import org.apache.asterix.app.cc.ExtensionManager; import org.apache.asterix.common.api.AsterixThreadExecutor; import org.apache.asterix.common.api.IAsterixAppRuntimeContext; import org.apache.asterix.common.api.IDatasetLifecycleManager; @@ -130,6 +131,7 @@ private final int metadataRmiPort; private ILibraryManager libraryManager; + private ExtensionManager extensionManager; public AsterixAppRuntimeContext(INCApplicationContext ncApplicationContext, int metadataRmiPort) throws AsterixException { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java index 7a47ca9..584bf82 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java @@ -71,6 +71,12 @@ this.sqlppCompilationProvider = new SqlppCompilationProvider(); } + public APIServlet(ILangCompilationProvider aqlCompilationProvider, + ILangCompilationProvider sqlppCompilationProvider) { + this.aqlCompilationProvider = aqlCompilationProvider; + this.sqlppCompilationProvider = sqlppCompilationProvider; + } + @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException { // Query language diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/ExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/ExtensionManager.java new file mode 100644 index 0000000..bb58d07 --- /dev/null +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/ExtensionManager.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.app.cc; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServlet; + +import org.apache.asterix.api.http.servlet.APIServlet; +import org.apache.asterix.api.http.servlet.AQLAPIServlet; +import org.apache.asterix.api.http.servlet.ClusterAPIServlet; +import org.apache.asterix.api.http.servlet.ConnectorAPIServlet; +import org.apache.asterix.api.http.servlet.DDLAPIServlet; +import org.apache.asterix.api.http.servlet.QueryAPIServlet; +import org.apache.asterix.api.http.servlet.QueryResultAPIServlet; +import org.apache.asterix.api.http.servlet.QueryServiceServlet; +import org.apache.asterix.api.http.servlet.QueryStatusAPIServlet; +import org.apache.asterix.api.http.servlet.ShutdownAPIServlet; +import org.apache.asterix.api.http.servlet.UpdateAPIServlet; +import org.apache.asterix.api.http.servlet.VersionAPIServlet; +import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.common.utils.ServletUtil; +import org.apache.asterix.compiler.provider.AqlCompilationProvider; +import org.apache.asterix.compiler.provider.SqlppCompilationProvider; +import org.apache.asterix.extension.api.ExtensionId; +import org.apache.asterix.extension.api.IExtension; +import org.apache.asterix.extension.api.IExtensionStatement; +import org.apache.asterix.lang.common.base.Statement; +import org.apache.asterix.metadata.declared.AqlMetadataProvider; +import org.apache.hyracks.api.client.IHyracksClientConnection; +import org.apache.hyracks.api.exceptions.HyracksDataException; + +public class ExtensionManager { + + private static Map<ExtensionId, IExtension> extensions = new HashMap<>(); + + public static void handleExtensionStatement(AqlMetadataProvider metadataProvider, Statement stmt, + IHyracksClientConnection hcc) throws HyracksDataException { + if (!(stmt instanceof IExtensionStatement)) { + throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.INVALID_EXTENSION_STATEMENT, + " Unable to identify handler for Statement with kind %1$s since it doesn't implement %2$s", + stmt.getKind(), IExtensionStatement.class.getSimpleName()); + } + IExtensionStatement extensionStatement = (IExtensionStatement) stmt; + IExtension extension = extensions.get(extensionStatement.getExtensionId()); + if (extension == null) { + throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.EXTENSION_NOT_FOUND, + " Unable to identify handler for IExtensionStatement since its extension %1$s is not registered with the extension manager", + extensionStatement.getExtensionId().toString()); + } + extension.handle(metadataProvider, extensionStatement, hcc); + } + + private static void addExtension(ExtensionId extensionId, IExtension extension) { + extensions.put(extensionId, extension); + } + + public static void init(List<String> extensionClassNames) { + // extension manager should instantiate, validate and add extensions + } + + public static APIServlet createWebAPIServlet() { + // if an extension provide different aql compiler or sqlpp compiler, extension manager should pass them + return new APIServlet(); + } + + public static HttpServlet createServlet(String servlet) { + switch (servlet) { + case ServletUtil.AQL: + return new AQLAPIServlet(new AqlCompilationProvider()); + case ServletUtil.AQL_DDL: + return new DDLAPIServlet(new AqlCompilationProvider()); + case ServletUtil.AQL_QUERY: + return new QueryAPIServlet(new AqlCompilationProvider()); + case ServletUtil.AQL_UPDATE: + return new UpdateAPIServlet(new AqlCompilationProvider()); + case ServletUtil.CLUSTER_STATE: + return new ClusterAPIServlet(); + case ServletUtil.CONNECTOR: + return new ConnectorAPIServlet(); + case ServletUtil.QUERY_RESULT: + return new QueryResultAPIServlet(); + case ServletUtil.QUERY_SERVICE: + return new QueryServiceServlet(); + case ServletUtil.QUERY_STATUS: + return new QueryStatusAPIServlet(); + case ServletUtil.SHUTDOWN: + return new ShutdownAPIServlet(); + case ServletUtil.SQLPP: + // Note: The servlet class name should be changed since it is used with different languages + return new AQLAPIServlet(new SqlppCompilationProvider()); + case ServletUtil.SQLPP_DDL: + return new DDLAPIServlet(new SqlppCompilationProvider()); + case ServletUtil.SQLPP_QUERY: + return new QueryAPIServlet(new SqlppCompilationProvider()); + case ServletUtil.SQLPP_UPDATE: + return new UpdateAPIServlet(new SqlppCompilationProvider()); + case ServletUtil.VERSION: + return new VersionAPIServlet(); + default: + // Check if an extension provide a servlet for the passed parameter + break; + } + return null; + } +} diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java index 05d9b3d..579dffa 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java @@ -43,6 +43,7 @@ import org.apache.asterix.active.IActiveEntityEventsListener; import org.apache.asterix.api.common.APIFramework; import org.apache.asterix.api.common.SessionConfig; +import org.apache.asterix.app.cc.ExtensionManager; import org.apache.asterix.app.external.ExternalIndexingOperations; import org.apache.asterix.app.external.FeedJoint; import org.apache.asterix.app.external.FeedOperations; @@ -378,8 +379,12 @@ case Statement.Kind.RUN: handleRunStatement(metadataProvider, stmt, hcc); break; + case Statement.Kind.FUNCTION_DECL: + // No op + break; default: // Default should delegate unknown statement to extension-manager + ExtensionManager.handleExtensionStatement(metadataProvider, stmt, hcc); break; } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java index 324194d..1ec7b4a 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java @@ -21,7 +21,6 @@ import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.asterix.api.http.servlet.APIServlet; import org.apache.asterix.api.http.servlet.AQLAPIServlet; import org.apache.asterix.api.http.servlet.ClusterAPIServlet; import org.apache.asterix.api.http.servlet.ConnectorAPIServlet; @@ -34,6 +33,7 @@ import org.apache.asterix.api.http.servlet.ShutdownAPIServlet; import org.apache.asterix.api.http.servlet.UpdateAPIServlet; import org.apache.asterix.api.http.servlet.VersionAPIServlet; +import org.apache.asterix.app.cc.ExtensionManager; import org.apache.asterix.app.external.ActiveLifecycleListener; import org.apache.asterix.app.external.ExternalLibraryUtils; import org.apache.asterix.common.api.AsterixThreadFactory; @@ -41,8 +41,7 @@ import org.apache.asterix.common.config.AsterixExternalProperties; import org.apache.asterix.common.config.AsterixMetadataProperties; import org.apache.asterix.common.library.ILibraryManager; -import org.apache.asterix.common.utils.ServletUtil.Servlets; -import org.apache.asterix.compiler.provider.AqlCompilationProvider; +import org.apache.asterix.common.utils.ServletUtil; import org.apache.asterix.compiler.provider.SqlppCompilationProvider; import org.apache.asterix.event.service.ILookupService; import org.apache.asterix.external.library.ExternalLibraryManager; @@ -72,7 +71,7 @@ private Server webServer; private Server jsonAPIServer; - private Server feedServer; + private Server activeServer; private static IAsterixStateProxy proxy; private ICCApplicationContext appCtx; @@ -103,14 +102,17 @@ AsterixAppContextInfo.getInstance().getCCApplicationContext() .addJobLifecycleListener(ActiveLifecycleListener.INSTANCE); + // instantiate and add extensions + ExtensionManager.init(AsterixAppContextInfo.getInstance().getExtensionsProperies().getExtensionClassNames()); + AsterixExternalProperties externalProperties = AsterixAppContextInfo.getInstance().getExternalProperties(); setupWebServer(externalProperties); webServer.start(); setupJSONAPIServer(externalProperties); jsonAPIServer.start(); - setupFeedServer(externalProperties); - feedServer.start(); + setupActiveServer(externalProperties); + activeServer.start(); ClusterManager.INSTANCE.registerSubscriber(GlobalRecoveryManager.INSTANCE); @@ -128,11 +130,11 @@ // Stop servers webServer.stop(); jsonAPIServer.stop(); - feedServer.stop(); + activeServer.stop(); // Make sure servers are stopped before proceeding webServer.join(); jsonAPIServer.join(); - feedServer.join(); + activeServer.join(); } private IHyracksClientConnection getNewHyracksClientConnection() throws Exception { @@ -152,7 +154,7 @@ context.setAttribute(HYRACKS_CONNECTION_ATTR, hcc); webServer.setHandler(context); - context.addServlet(new ServletHolder(new APIServlet()), "/*"); + context.addServlet(new ServletHolder(ExtensionManager.createWebAPIServlet()), "/*"); } private void setupJSONAPIServer(AsterixExternalProperties externalProperties) throws Exception { @@ -168,36 +170,33 @@ jsonAPIServer.setHandler(context); // AQL rest APIs. - context.addServlet(new ServletHolder(new QueryAPIServlet(new AqlCompilationProvider())), - Servlets.AQL_QUERY.getPath()); - context.addServlet(new ServletHolder(new UpdateAPIServlet(new AqlCompilationProvider())), - Servlets.AQL_UPDATE.getPath()); - context.addServlet(new ServletHolder(new DDLAPIServlet(new AqlCompilationProvider())), - Servlets.AQL_DDL.getPath()); - context.addServlet(new ServletHolder(new AQLAPIServlet(new AqlCompilationProvider())), Servlets.AQL.getPath()); + context.addServlet(new ServletHolder(ExtensionManager.createServlet(ServletUtil.AQL_QUERY)), + ServletUtil.AQL_QUERY); + context.addServlet(new ServletHolder(ExtensionManager.createServlet(ServletUtil.AQL_UPDATE)), + ServletUtil.AQL_UPDATE); + context.addServlet(new ServletHolder(ExtensionManager.createServlet(ServletUtil.AQL_DDL)), ServletUtil.AQL_DDL); + context.addServlet(new ServletHolder(ExtensionManager.createServlet(ServletUtil.AQL)), ServletUtil.AQL); // SQL++ rest APIs. context.addServlet(new ServletHolder(new QueryAPIServlet(new SqlppCompilationProvider())), - Servlets.SQLPP_QUERY.getPath()); + ServletUtil.SQLPP_QUERY); context.addServlet(new ServletHolder(new UpdateAPIServlet(new SqlppCompilationProvider())), - Servlets.SQLPP_UPDATE.getPath()); - context.addServlet(new ServletHolder(new DDLAPIServlet(new SqlppCompilationProvider())), - Servlets.SQLPP_DDL.getPath()); - context.addServlet(new ServletHolder(new AQLAPIServlet(new SqlppCompilationProvider())), - Servlets.SQLPP.getPath()); + ServletUtil.SQLPP_UPDATE); + context.addServlet(new ServletHolder(new DDLAPIServlet(new SqlppCompilationProvider())), ServletUtil.SQLPP_DDL); + context.addServlet(new ServletHolder(new AQLAPIServlet(new SqlppCompilationProvider())), ServletUtil.SQLPP); // Other APIs. - context.addServlet(new ServletHolder(new QueryStatusAPIServlet()), Servlets.QUERY_STATUS.getPath()); - context.addServlet(new ServletHolder(new QueryResultAPIServlet()), Servlets.QUERY_RESULT.getPath()); - context.addServlet(new ServletHolder(new QueryServiceServlet()), Servlets.QUERY_SERVICE.getPath()); - context.addServlet(new ServletHolder(new ConnectorAPIServlet()), Servlets.CONNECTOR.getPath()); - context.addServlet(new ServletHolder(new ShutdownAPIServlet()), Servlets.SHUTDOWN.getPath()); - context.addServlet(new ServletHolder(new VersionAPIServlet()), Servlets.VERSION.getPath()); - context.addServlet(new ServletHolder(new ClusterAPIServlet()), Servlets.CLUSTER_STATE.getPath()); + context.addServlet(new ServletHolder(new QueryStatusAPIServlet()), ServletUtil.QUERY_STATUS); + context.addServlet(new ServletHolder(new QueryResultAPIServlet()), ServletUtil.QUERY_RESULT); + context.addServlet(new ServletHolder(new QueryServiceServlet()), ServletUtil.QUERY_SERVICE); + context.addServlet(new ServletHolder(new ConnectorAPIServlet()), ServletUtil.CONNECTOR); + context.addServlet(new ServletHolder(new ShutdownAPIServlet()), ServletUtil.SHUTDOWN); + context.addServlet(new ServletHolder(new VersionAPIServlet()), ServletUtil.VERSION); + context.addServlet(new ServletHolder(new ClusterAPIServlet()), ServletUtil.CLUSTER_STATE); } - private void setupFeedServer(AsterixExternalProperties externalProperties) throws Exception { - feedServer = new Server(externalProperties.getFeedServerPort()); + private void setupActiveServer(AsterixExternalProperties externalProperties) throws Exception { + activeServer = new Server(externalProperties.getFeedServerPort()); ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); context.setContextPath("/"); @@ -205,7 +204,7 @@ IHyracksClientConnection hcc = getNewHyracksClientConnection(); context.setAttribute(HYRACKS_CONNECTION_ATTR, hcc); - feedServer.setHandler(context); + activeServer.setHandler(context); context.addServlet(new ServletHolder(new FeedServlet()), "/"); } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtensionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtensionProperties.java new file mode 100644 index 0000000..ff681c8 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtensionProperties.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import java.util.List; + +public class AsterixExtensionProperties extends AbstractAsterixProperties { + + public AsterixExtensionProperties(AsterixPropertiesAccessor accessor) { + super(accessor); + } + + public List<String> getExtensionClassNames() { + return accessor.getExtensionClassNames(); + } + +} diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java index 7309f0c..c87b33e 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java @@ -51,8 +51,8 @@ private final String instanceName; private final String metadataNodeName; - private final List<String> nodeNames = new ArrayList<>();; - private final Map<String, String[]> stores = new HashMap<>();; + private final List<String> nodeNames = new ArrayList<>(); + private final Map<String, String[]> stores = new HashMap<>(); private final Map<String, String> coredumpConfig = new HashMap<>(); // This can be removed when asterix-configuration.xml is no longer required. @@ -63,8 +63,12 @@ private final Map<String, ClusterPartition[]> nodePartitionsMap; private final SortedMap<Integer, ClusterPartition> clusterPartitions = new TreeMap<>(); + // For extensions + private final List<String> extensionClassNames = new ArrayList<>(); + /** * Constructor which reads asterix-configuration.xml, the old way. + * * @throws AsterixException */ public AsterixPropertiesAccessor() throws AsterixException { @@ -116,6 +120,11 @@ stores.put(store.getNcId(), nodeStores); nodePartitionsMap.put(store.getNcId(), nodePartitions); nodeNames.add(store.getNcId()); + } + if (asterixConfiguration.getExtensions() != null) { + for (String extensionClassName : asterixConfiguration.getExtensions().getExtensionClassName()) { + getExtensionClassNames().add(extensionClassName); + } } asterixConfigurationParams = new HashMap<String, Property>(); for (Property p : asterixConfiguration.getProperty()) { @@ -271,4 +280,8 @@ public SortedMap<Integer, ClusterPartition> getClusterPartitions() { return clusterPartitions; } + + public List<String> getExtensionClassNames() { + return extensionClassNames; + } } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 66a01e0..7d54ec2 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -21,6 +21,8 @@ public class ErrorCode { public static final String ASTERIX = "ASX"; public static final int ERROR_CASTING_FIELD = 0; + public static final int INVALID_EXTENSION_STATEMENT = 1; + public static final int EXTENSION_NOT_FOUND = 2; private ErrorCode() { } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ServletUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ServletUtil.java index b75d16c..5563258 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ServletUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ServletUtil.java @@ -20,33 +20,21 @@ public class ServletUtil { - public enum Servlets { - AQL("/aql"), - AQL_QUERY("/query"), - AQL_UPDATE("/update"), - AQL_DDL("/ddl"), - SQLPP("/sqlpp"), - SQLPP_QUERY("/query/sqlpp"), - SQLPP_UPDATE("/update/sqlpp"), - SQLPP_DDL("/ddl/sqlpp"), - QUERY_STATUS("/query/status"), - QUERY_RESULT("/query/result"), - QUERY_SERVICE("/query/service"), - CONNECTOR("/connector"), - SHUTDOWN("/admin/shutdown"), - VERSION("/admin/version"), - CLUSTER_STATE("/admin/cluster"); - - private final String path; - - private Servlets(String path) { - this.path = path; - } - - public String getPath() { - return path; - } - } + public static final String AQL = "/aql"; + public static final String AQL_QUERY = "/query"; + public static final String AQL_UPDATE = "/update"; + public static final String AQL_DDL = "/ddl"; + public static final String SQLPP = "/sqlpp"; + public static final String SQLPP_QUERY = "/query/sqlpp"; + public static final String SQLPP_UPDATE = "/update/sqlpp"; + public static final String SQLPP_DDL = "/ddl/sqlpp"; + public static final String QUERY_STATUS = "/query/status"; + public static final String QUERY_RESULT = "/query/result"; + public static final String QUERY_SERVICE = "/query/service"; + public static final String CONNECTOR = "/connector"; + public static final String SHUTDOWN = "/admin/shutdown"; + public static final String VERSION = "/admin/version"; + public static final String CLUSTER_STATE = "/admin/cluster"; private ServletUtil() { throw new AssertionError("No objects of this class should be created."); diff --git a/asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd b/asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd index bb99319..9116207 100644 --- a/asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd +++ b/asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd @@ -56,6 +56,9 @@ <xs:element name="txnLogDirPath" type="xs:string" /> + <xs:element + name="extensionClassName" + type="xs:string" /> <!-- definition of complex elements --> <xs:element name="store"> @@ -94,6 +97,17 @@ </xs:sequence> </xs:complexType> </xs:element> + + <xs:element name="extensions"> + <xs:complexType> + <xs:sequence> + <xs:element ref="mg:extensionClassName" + minOccurs="0" + maxOccurs="unbounded"> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> <xs:element name="asterixConfiguration"> @@ -117,6 +131,10 @@ <xs:element ref="mg:transactionLogDir" maxOccurs="unbounded" /> + <xs:element ref="mg:extensions" + minOccurs="0" + maxOccurs="1"> + </xs:element> <xs:element ref="mg:property" minOccurs="0" diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java index 58c6a91..b76f41d 100644 --- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java @@ -40,13 +40,12 @@ import java.util.logging.Logger; import org.apache.asterix.common.config.GlobalConfig; -import org.apache.asterix.common.utils.ServletUtil.Servlets; +import org.apache.asterix.common.utils.ServletUtil; import org.apache.asterix.test.server.ITestServer; import org.apache.asterix.test.server.TestServerProvider; import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.asterix.testframework.context.TestCaseContext.OutputFormat; import org.apache.asterix.testframework.context.TestFileContext; -import org.apache.asterix.testframework.xml.TestCase; import org.apache.asterix.testframework.xml.TestCase.CompilationUnit; import org.apache.asterix.testframework.xml.TestGroup; import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler; @@ -113,10 +112,10 @@ public void runScriptAndCompareWithResult(File scriptFile, PrintWriter print, File expectedFile, File actualFile) throws Exception { System.err.println("Expected results file: " + expectedFile.toString()); - BufferedReader readerExpected = new BufferedReader( - new InputStreamReader(new FileInputStream(expectedFile), "UTF-8")); - BufferedReader readerActual = new BufferedReader( - new InputStreamReader(new FileInputStream(actualFile), "UTF-8")); + BufferedReader readerExpected = + new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8")); + BufferedReader readerActual = + new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), "UTF-8")); String lineExpected, lineActual; int num = 1; try { @@ -278,8 +277,8 @@ return statusCode; } - public InputStream executeQuery(String str, OutputFormat fmt, String url, - List<CompilationUnit.Parameter> params) throws Exception { + public InputStream executeQuery(String str, OutputFormat fmt, String url, List<CompilationUnit.Parameter> params) + throws Exception { HttpMethod method = constructHttpMethod(str, url, "query", false, params); // Set accepted output response type method.setRequestHeader("Accept", fmt.mimeType()); @@ -398,7 +397,7 @@ } private InputStream getHandleResult(String handle, OutputFormat fmt) throws Exception { - final String url = "http://" + host + ":" + port + Servlets.QUERY_RESULT.getPath(); + final String url = "http://" + host + ":" + port + ServletUtil.QUERY_RESULT; // Create a method instance. GetMethod method = new GetMethod(url); @@ -515,7 +514,6 @@ executeTest(actualPath, testCaseCtx, pb, isDmlRecoveryTest, null); } - public void executeTest(TestCaseContext testCaseCtx, TestFileContext ctx, String statement, boolean isDmlRecoveryTest, ProcessBuilder pb, CompilationUnit cUnit, MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath) throws Exception { @@ -525,9 +523,9 @@ switch (ctx.getType()) { case "ddl": if (ctx.getFile().getName().endsWith("aql")) { - executeDDL(statement, "http://" + host + ":" + port + Servlets.AQL_DDL.getPath()); + executeDDL(statement, "http://" + host + ":" + port + ServletUtil.AQL_DDL); } else { - executeDDL(statement, "http://" + host + ":" + port + Servlets.SQLPP_DDL.getPath()); + executeDDL(statement, "http://" + host + ":" + port + ServletUtil.SQLPP_DDL); } break; case "update": @@ -536,9 +534,9 @@ statement = statement.replaceAll("nc1://", "127.0.0.1://../../../../../../asterix-app/"); } if (ctx.getFile().getName().endsWith("aql")) { - executeUpdate(statement, "http://" + host + ":" + port + Servlets.AQL_UPDATE.getPath()); + executeUpdate(statement, "http://" + host + ":" + port + ServletUtil.AQL_UPDATE); } else { - executeUpdate(statement, "http://" + host + ":" + port + Servlets.SQLPP_UPDATE.getPath()); + executeUpdate(statement, "http://" + host + ":" + port + ServletUtil.SQLPP_UPDATE); } break; case "query": @@ -556,25 +554,25 @@ if (ctx.getFile().getName().endsWith("aql")) { if (ctx.getType().equalsIgnoreCase("query")) { resultStream = executeQuery(statement, fmt, - "http://" + host + ":" + port + Servlets.AQL_QUERY.getPath(), cUnit.getParameter()); + "http://" + host + ":" + port + ServletUtil.AQL_QUERY, cUnit.getParameter()); } else if (ctx.getType().equalsIgnoreCase("async")) { resultStream = executeAnyAQLAsync(statement, false, fmt, - "http://" + host + ":" + port + Servlets.AQL.getPath()); + "http://" + host + ":" + port + ServletUtil.AQL); } else if (ctx.getType().equalsIgnoreCase("asyncdefer")) { resultStream = executeAnyAQLAsync(statement, true, fmt, - "http://" + host + ":" + port + Servlets.AQL.getPath()); + "http://" + host + ":" + port + ServletUtil.AQL); } } else { if (ctx.getType().equalsIgnoreCase("query")) { resultStream = executeQueryService(statement, fmt, - "http://" + host + ":" + port + Servlets.QUERY_SERVICE.getPath(), cUnit.getParameter()); + "http://" + host + ":" + port + ServletUtil.QUERY_SERVICE, cUnit.getParameter()); resultStream = ResultExtractor.extract(resultStream); } else if (ctx.getType().equalsIgnoreCase("async")) { resultStream = executeAnyAQLAsync(statement, false, fmt, - "http://" + host + ":" + port + Servlets.SQLPP.getPath()); + "http://" + host + ":" + port + ServletUtil.SQLPP); } else if (ctx.getType().equalsIgnoreCase("asyncdefer")) { resultStream = executeAnyAQLAsync(statement, true, fmt, - "http://" + host + ":" + port + Servlets.SQLPP.getPath()); + "http://" + host + ":" + port + ServletUtil.SQLPP); } } if (queryCount.intValue() >= expectedResultFileCtxs.size()) { @@ -583,8 +581,8 @@ } expectedResultFile = expectedResultFileCtxs.get(queryCount.intValue()).getFile(); - File actualResultFile = testCaseCtx.getActualResultFile(cUnit, expectedResultFile, - new File(actualPath)); + File actualResultFile = + testCaseCtx.getActualResultFile(cUnit, expectedResultFile, new File(actualPath)); actualResultFile.getParentFile().mkdirs(); writeOutputToFile(actualResultFile, resultStream); @@ -600,14 +598,14 @@ break; case "txnqbc": // qbc represents query before crash resultStream = executeQuery(statement, OutputFormat.forCompilationUnit(cUnit), - "http://" + host + ":" + port + Servlets.AQL_QUERY.getPath(), cUnit.getParameter()); + "http://" + host + ":" + port + ServletUtil.AQL_QUERY, cUnit.getParameter()); qbcFile = getTestCaseQueryBeforeCrashFile(actualPath, testCaseCtx, cUnit); qbcFile.getParentFile().mkdirs(); writeOutputToFile(qbcFile, resultStream); break; case "txnqar": // qar represents query after recovery resultStream = executeQuery(statement, OutputFormat.forCompilationUnit(cUnit), - "http://" + host + ":" + port + Servlets.AQL_QUERY.getPath(), cUnit.getParameter()); + "http://" + host + ":" + port + ServletUtil.AQL_QUERY, cUnit.getParameter()); File qarFile = new File(actualPath + File.separator + testCaseCtx.getTestCase().getFilePath().replace(File.separator, "_") + "_" + cUnit.getName() + "_qar.adm"); @@ -618,7 +616,7 @@ break; case "txneu": // eu represents erroneous update try { - executeUpdate(statement, "http://" + host + ":" + port + Servlets.AQL_UPDATE.getPath()); + executeUpdate(statement, "http://" + host + ":" + port + ServletUtil.AQL_UPDATE); } catch (Exception e) { // An exception is expected. failed = true; @@ -646,7 +644,7 @@ break; case "errddl": // a ddlquery that expects error try { - executeDDL(statement, "http://" + host + ":" + port + Servlets.AQL_DDL.getPath()); + executeDDL(statement, "http://" + host + ":" + port + ServletUtil.AQL_DDL); } catch (Exception e) { // expected error happens failed = true; @@ -686,8 +684,8 @@ case "cstate": // cluster state query try { fmt = OutputFormat.forCompilationUnit(cUnit); - resultStream = executeClusterStateQuery(fmt, - "http://" + host + ":" + port + Servlets.CLUSTER_STATE.getPath()); + resultStream = + executeClusterStateQuery(fmt, "http://" + host + ":" + port + ServletUtil.CLUSTER_STATE); expectedResultFile = expectedResultFileCtxs.get(queryCount.intValue()).getFile(); actualResultFile = testCaseCtx.getActualResultFile(cUnit, expectedResultFile, new File(actualPath)); actualResultFile.getParentFile().mkdirs(); diff --git a/asterixdb/asterix-extension/pom.xml b/asterixdb/asterix-extension/pom.xml new file mode 100644 index 0000000..ec1ba9e --- /dev/null +++ b/asterixdb/asterix-extension/pom.xml @@ -0,0 +1,16 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.asterix</groupId> + <artifactId>apache-asterixdb</artifactId> + <version>0.8.9-SNAPSHOT</version> + </parent> + <artifactId>asterix-extension</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-lang-common</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> \ No newline at end of file diff --git a/asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/ExtensionId.java b/asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/ExtensionId.java new file mode 100644 index 0000000..3b9469a --- /dev/null +++ b/asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/ExtensionId.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.extension.api; + +public class ExtensionId { + + private final String name; + private final int version; + + public ExtensionId(String name, int version) { + this.name = name; + this.version = version; + } + + @Override + public int hashCode() { + return getName().hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (o instanceof ExtensionId) { + ExtensionId oExtensionId = (ExtensionId) o; + return version == oExtensionId.version && name.equals(oExtensionId.getName()); + } + return false; + } + + public int getVersion() { + return version; + } + + public String getName() { + return name; + } + +} diff --git a/asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/IExtension.java b/asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/IExtension.java new file mode 100644 index 0000000..f25574c --- /dev/null +++ b/asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/IExtension.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.extension.api; + +import org.apache.asterix.metadata.declared.AqlMetadataProvider; +import org.apache.hyracks.api.client.IHyracksClientConnection; + +public interface IExtension { + + void handle(AqlMetadataProvider metadataProvider, IExtensionStatement extensionStatement, + IHyracksClientConnection hcc); + +} diff --git a/asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/IExtensionStatement.java b/asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/IExtensionStatement.java new file mode 100644 index 0000000..964eb04 --- /dev/null +++ b/asterixdb/asterix-extension/src/main/java/org/apache/asterix/extension/api/IExtensionStatement.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.extension.api; + +import org.apache.asterix.lang.common.base.Statement; + +public interface IExtensionStatement extends Statement { + public ExtensionId getExtensionId(); +} diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java index bb1e554..b6af65e 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java @@ -23,6 +23,7 @@ import org.apache.asterix.common.cluster.IGlobalRecoveryMaanger; import org.apache.asterix.common.config.AsterixBuildProperties; import org.apache.asterix.common.config.AsterixCompilerProperties; +import org.apache.asterix.common.config.AsterixExtensionProperties; import org.apache.asterix.common.config.AsterixExternalProperties; import org.apache.asterix.common.config.AsterixFeedProperties; import org.apache.asterix.common.config.AsterixMetadataProperties; @@ -52,6 +53,7 @@ private final ICCApplicationContext appCtx; private AsterixCompilerProperties compilerProperties; + private AsterixExtensionProperties extentionsProperties; private AsterixExternalProperties externalProperties; private AsterixMetadataProperties metadataProperties; private AsterixStorageProperties storageProperties; @@ -81,13 +83,14 @@ propertiesAccessor = new AsterixPropertiesAccessor(); } INSTANCE.compilerProperties = new AsterixCompilerProperties(propertiesAccessor); + INSTANCE.extentionsProperties = new AsterixExtensionProperties(propertiesAccessor); INSTANCE.externalProperties = new AsterixExternalProperties(propertiesAccessor); INSTANCE.metadataProperties = new AsterixMetadataProperties(propertiesAccessor); INSTANCE.storageProperties = new AsterixStorageProperties(propertiesAccessor); INSTANCE.txnProperties = new AsterixTransactionProperties(propertiesAccessor); INSTANCE.feedProperties = new AsterixFeedProperties(propertiesAccessor); - INSTANCE.replicationProperties = new AsterixReplicationProperties(propertiesAccessor, - AsterixClusterProperties.INSTANCE.getCluster()); + INSTANCE.replicationProperties = + new AsterixReplicationProperties(propertiesAccessor, AsterixClusterProperties.INSTANCE.getCluster()); INSTANCE.hcc = hcc; INSTANCE.buildProperties = new AsterixBuildProperties(propertiesAccessor); Logger.getLogger("org.apache").setLevel(INSTANCE.externalProperties.getLogLevel()); @@ -173,4 +176,8 @@ public ILibraryManager getLibraryManager() { return libraryManager; } + + public AsterixExtensionProperties getExtensionsProperies() { + return extentionsProperties; + } } diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index 37be3dc..260feb4 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -576,6 +576,7 @@ <module>asterix-experiments</module> <module>asterix-coverage</module> <module>asterix-active</module> + <module>asterix-extension</module> </modules> <repositories> -- To view, visit https://asterix-gerrit.ics.uci.edu/1017 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I280268495cc3aad00f898cba21f7299f7120ce5c Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: abdullah alamoudi <bamou...@gmail.com>