ARTEMIS-27 / ARTEMIS-340 Add JDBC Storage Manager
Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/64f74acd Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/64f74acd Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/64f74acd Branch: refs/heads/master Commit: 64f74acdbc0dbfd2c3d15eddb827fcea1e679518 Parents: 9dd9c02 Author: Martyn Taylor <mtay...@redhat.com> Authored: Thu Jan 7 10:12:07 2016 +0000 Committer: Clebert Suconic <clebertsuco...@apache.org> Committed: Wed Jan 13 09:38:40 2016 -0500 ---------------------------------------------------------------------- .gitignore | 1 + .../config/ActiveMQDefaultConfiguration.java | 36 + .../jdbc/store/journal/JDBCJournalImpl.java | 2 +- artemis-server/pom.xml | 5 + .../artemis/core/config/Configuration.java | 6 +- .../artemis/core/config/StoreConfiguration.java | 29 + .../core/config/impl/ConfigurationImpl.java | 16 +- .../storage/DatabaseStorageConfiguration.java | 58 + .../storage/FileStorageConfiguration.java | 58 + .../deployers/impl/FileConfigurationParser.java | 46 +- .../journal/AbstractJournalStorageManager.java | 1828 +++++++++++++ .../impl/journal/AddMessageRecord.java | 13 +- .../impl/journal/JDBCJournalStorageManager.java | 88 + .../impl/journal/JournalRecordIds.java | 1 - .../impl/journal/JournalStorageManager.java | 2542 +++--------------- .../wireformat/ReplicationStartSyncMessage.java | 12 +- .../wireformat/ReplicationSyncFileMessage.java | 10 +- .../core/replication/ReplicationEndpoint.java | 2 +- .../core/replication/ReplicationManager.java | 10 +- .../artemis/core/security/SecurityAuth.java | 2 +- .../core/server/impl/ActiveMQServerImpl.java | 10 +- .../resources/schema/artemis-configuration.xsd | 58 + .../impl/DatabaseStoreConfigurationTest.java | 50 + .../artemis/tests/util/ActiveMQTestBase.java | 18 + .../test/resources/database-store-config.xml | 30 + docs/user-manual/en/persistence.md | 58 +- pom.xml | 13 + tests/integration-tests/pom.xml | 5 + .../tests/integration/client/PagingTest.java | 2 +- .../store/journal/FakeEncodingSupportImpl.java | 46 + .../jdbc/store/journal/JDBCJournalTest.java | 127 + ...AddressSettingsConfigurationStorageTest.java | 22 +- .../DeleteMessagesOnStartupTest.java | 19 +- .../persistence/DuplicateCacheTest.java | 5 + ...nnectionFactoryConfigurationStorageTest.java | 5 + .../persistence/JMSStorageManagerTest.java | 5 + .../RolesConfigurationStorageTest.java | 5 + .../persistence/StorageManagerTestBase.java | 61 +- 38 files changed, 3079 insertions(+), 2225 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index c4b0866..820c732 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ ratReport.txt .settings .checkstyle .factorypath +**/derby.log # for native build CMakeCache.txt http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/config/ActiveMQDefaultConfiguration.java ---------------------------------------------------------------------- diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/config/ActiveMQDefaultConfiguration.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/config/ActiveMQDefaultConfiguration.java index 1fc861d..462e597 100644 --- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/config/ActiveMQDefaultConfiguration.java +++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/config/ActiveMQDefaultConfiguration.java @@ -396,6 +396,18 @@ public final class ActiveMQDefaultConfiguration { // How often the reaper will be run to check for timed out group bindings. Only valid for LOCAL handlers private static long DEFAULT_GROUPING_HANDLER_REAPER_PERIOD = 30000; + // Which store type to use, options are FILE or DATABASE, FILE is default. + private static String DEFAULT_STORE_TYPE = "FILE"; + + // Default database url. Derby database is used by default. + private static String DEFAULT_DATABASE_URL = "jdbc:derby:data/derby;create=true"; + + // Default message table name, used with Database storage type + private static String DEFAULT_MESSAGE_TABLE_NAME = "MESSAGES"; + + // Default bindings table name, used with Database storage type + private static String DEFAULT_BINDINGS_TABLE_NAME = "BINDINGS"; + /** * If true then the ActiveMQ Artemis Server will make use of any Protocol Managers that are in available on the classpath. If false then only the core protocol will be available, unless in Embedded mode where users can inject their own Protocol Managers. */ @@ -1052,4 +1064,28 @@ public final class ActiveMQDefaultConfiguration { return DEFAULT_GROUPING_HANDLER_REAPER_PERIOD; } + /** + * The default storage type. Options are FILE and DATABASE. + */ + public static String getDefaultStoreType() { + return DEFAULT_STORE_TYPE; + } + + /** + * The default database URL, used with DATABASE store type. + */ + public static String getDefaultDatabaseUrl() { + return DEFAULT_DATABASE_URL; + } + + /** + * The default Message Journal table name, used with DATABASE store. + */ + public static String getDefaultMessageTableName() { + return DEFAULT_MESSAGE_TABLE_NAME; + } + + public static String getDefaultBindingsTableName() { + return DEFAULT_BINDINGS_TABLE_NAME; + } } http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/journal/JDBCJournalImpl.java ---------------------------------------------------------------------- diff --git a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/journal/JDBCJournalImpl.java b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/journal/JDBCJournalImpl.java index db5a06d..7309c94 100644 --- a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/journal/JDBCJournalImpl.java +++ b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/journal/JDBCJournalImpl.java @@ -33,6 +33,7 @@ import java.util.Timer; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.activemq.artemis.core.io.SequentialFileFactory; import org.apache.activemq.artemis.core.journal.EncodingSupport; import org.apache.activemq.artemis.core.journal.IOCompletion; import org.apache.activemq.artemis.core.journal.Journal; @@ -40,7 +41,6 @@ import org.apache.activemq.artemis.core.journal.JournalLoadInformation; import org.apache.activemq.artemis.core.journal.LoaderCallback; import org.apache.activemq.artemis.core.journal.PreparedTransactionInfo; import org.apache.activemq.artemis.core.journal.RecordInfo; -import org.apache.activemq.artemis.core.journal.SequentialFileFactory; import org.apache.activemq.artemis.core.journal.TransactionFailureCallback; import org.apache.activemq.artemis.core.journal.impl.JournalFile; import org.apache.derby.jdbc.AutoloadedDriver; http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/artemis-server/pom.xml ---------------------------------------------------------------------- diff --git a/artemis-server/pom.xml b/artemis-server/pom.xml index ba89cd5..d4bf594 100644 --- a/artemis-server/pom.xml +++ b/artemis-server/pom.xml @@ -68,6 +68,11 @@ </dependency> <dependency> <groupId>org.apache.activemq</groupId> + <artifactId>artemis-jdbc-store</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.activemq</groupId> <artifactId>artemis-core-client</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/Configuration.java ---------------------------------------------------------------------- diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/Configuration.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/Configuration.java index 93ef487..96d5050 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/Configuration.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/Configuration.java @@ -271,8 +271,9 @@ public interface Configuration { /** * Add an acceptor to the config + * * @param name the name of the acceptor - * @param uri the URI of the acceptor + * @param uri the URI of the acceptor * @return this * @throws Exception in case of Parsing errors on the URI */ @@ -935,4 +936,7 @@ public interface Configuration { */ File getBrokerInstance(); + StoreConfiguration getStoreConfiguration(); + + Configuration setStoreConfiguration(StoreConfiguration storeConfiguration); } http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/StoreConfiguration.java ---------------------------------------------------------------------- diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/StoreConfiguration.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/StoreConfiguration.java new file mode 100644 index 0000000..d25cdd9 --- /dev/null +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/StoreConfiguration.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.core.config; + +import java.io.Serializable; + +public interface StoreConfiguration extends Serializable { + + public enum StoreType { + FILE, + DATABASE + } + + StoreType getStoreType(); +} http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImpl.java ---------------------------------------------------------------------- diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImpl.java index 9f35789..fc36809 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImpl.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/impl/ConfigurationImpl.java @@ -44,6 +44,7 @@ import org.apache.activemq.artemis.core.config.ConnectorServiceConfiguration; import org.apache.activemq.artemis.core.config.CoreQueueConfiguration; import org.apache.activemq.artemis.core.config.DivertConfiguration; import org.apache.activemq.artemis.core.config.HAPolicyConfiguration; +import org.apache.activemq.artemis.core.config.StoreConfiguration; import org.apache.activemq.artemis.core.config.ha.ReplicaPolicyConfiguration; import org.apache.activemq.artemis.core.config.ha.ReplicatedPolicyConfiguration; import org.apache.activemq.artemis.core.security.Role; @@ -228,6 +229,8 @@ public class ConfigurationImpl implements Configuration, Serializable { private HAPolicyConfiguration haPolicyConfiguration; + private StoreConfiguration storeConfiguration; + /** * Parent folder for all data folders. */ @@ -407,7 +410,6 @@ public class ConfigurationImpl implements Configuration, Serializable { return this; } - @Override public ConfigurationImpl addConnectorConfiguration(final String name, final String uri) throws Exception { @@ -422,7 +424,6 @@ public class ConfigurationImpl implements Configuration, Serializable { return this; } - @Override public ConfigurationImpl clearConnectorConfigurations() { connectorConfigs.clear(); @@ -1279,6 +1280,17 @@ public class ConfigurationImpl implements Configuration, Serializable { } @Override + public StoreConfiguration getStoreConfiguration() { + return storeConfiguration; + } + + @Override + public ConfigurationImpl setStoreConfiguration(StoreConfiguration storeConfiguration) { + this.storeConfiguration = storeConfiguration; + return this; + } + + @Override public int hashCode() { final int prime = 31; int result = 1; http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/DatabaseStorageConfiguration.java ---------------------------------------------------------------------- diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/DatabaseStorageConfiguration.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/DatabaseStorageConfiguration.java new file mode 100644 index 0000000..f284af0 --- /dev/null +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/DatabaseStorageConfiguration.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.core.config.storage; + +import org.apache.activemq.artemis.api.config.ActiveMQDefaultConfiguration; +import org.apache.activemq.artemis.core.config.StoreConfiguration; + +public class DatabaseStorageConfiguration implements StoreConfiguration { + + private String messageTableName = ActiveMQDefaultConfiguration.getDefaultMessageTableName(); + + private String bindingsTableName = ActiveMQDefaultConfiguration.getDefaultBindingsTableName(); + + private String jdbcConnectionUrl = ActiveMQDefaultConfiguration.getDefaultDatabaseUrl(); + + @Override + public StoreType getStoreType() { + return StoreType.DATABASE; + } + + public String getMessageTableName() { + return messageTableName; + } + + public void setMessageTableName(String messageTableName) { + this.messageTableName = messageTableName; + } + + public String getBindingsTableName() { + return bindingsTableName; + } + + public void setBindingsTableName(String bindingsTableName) { + this.bindingsTableName = bindingsTableName; + } + + public void setJdbcConnectionUrl(String jdbcConnectionUrl) { + this.jdbcConnectionUrl = jdbcConnectionUrl; + } + + public String getJdbcConnectionUrl() { + return jdbcConnectionUrl; + } +} http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/FileStorageConfiguration.java ---------------------------------------------------------------------- diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/FileStorageConfiguration.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/FileStorageConfiguration.java new file mode 100644 index 0000000..05888b7 --- /dev/null +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/FileStorageConfiguration.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.core.config.storage; + +import org.apache.activemq.artemis.api.config.ActiveMQDefaultConfiguration; +import org.apache.activemq.artemis.core.config.StoreConfiguration; + +public class FileStorageConfiguration implements StoreConfiguration { + + private String messageTableName = ActiveMQDefaultConfiguration.getDefaultMessageTableName(); + + private String bindingsTableName = ActiveMQDefaultConfiguration.getDefaultBindingsTableName(); + + private String jdbcConnectionUrl = ActiveMQDefaultConfiguration.getDefaultDatabaseUrl(); + + @Override + public StoreType getStoreType() { + return StoreType.DATABASE; + } + + public String getMessageTableName() { + return messageTableName; + } + + public void setMessageTableName(String messageTableName) { + this.messageTableName = messageTableName; + } + + public String getBindingsTableName() { + return bindingsTableName; + } + + public void setBindingsTableName(String bindingsTableName) { + this.bindingsTableName = bindingsTableName; + } + + public void setJdbcConnectionUrl(String jdbcConnectionUrl) { + this.jdbcConnectionUrl = jdbcConnectionUrl; + } + + public String getJdbcConnectionUrl() { + return jdbcConnectionUrl; + } +} http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/64f74acd/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java ---------------------------------------------------------------------- diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java index 019ebb3..8b11b3b 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java @@ -54,6 +54,8 @@ import org.apache.activemq.artemis.core.config.ha.SharedStoreMasterPolicyConfigu import org.apache.activemq.artemis.core.config.ha.SharedStoreSlavePolicyConfiguration; import org.apache.activemq.artemis.core.config.impl.ConfigurationImpl; import org.apache.activemq.artemis.core.config.impl.Validators; +import org.apache.activemq.artemis.core.config.storage.DatabaseStorageConfiguration; +import org.apache.activemq.artemis.core.config.storage.FileStorageConfiguration; import org.apache.activemq.artemis.core.io.aio.AIOSequentialFileFactory; import org.apache.activemq.artemis.core.security.Role; import org.apache.activemq.artemis.core.server.ActiveMQServerLogger; @@ -214,6 +216,12 @@ public final class FileConfigurationParser extends XMLConfigurationUtil { config.setHAPolicyConfiguration(new LiveOnlyPolicyConfiguration()); } + NodeList storeTypeNodes = e.getElementsByTagName("store"); + + if (storeTypeNodes.getLength() > 0) { + parseStoreConfiguration((Element) storeTypeNodes.item(0), config); + } + config.setResolveProtocols(getBoolean(e, "resolve-protocols", config.isResolveProtocols())); config.setPersistenceEnabled(getBoolean(e, "persistence-enabled", config.isPersistenceEnabled())); @@ -666,7 +674,7 @@ public final class FileConfigurationParser extends XMLConfigurationUtil { return securityMatch; } - private Pair<SecuritySettingPlugin,Map<String,String>> parseSecuritySettingPlugins(Node item) { + private Pair<SecuritySettingPlugin, Map<String, String>> parseSecuritySettingPlugins(Node item) { final String clazz = item.getAttributes().getNamedItem("class-name").getNodeValue(); final Map<String, String> settings = new HashMap<>(); NodeList children = item.getChildNodes(); @@ -905,6 +913,28 @@ public final class FileConfigurationParser extends XMLConfigurationUtil { HA_LIST.add("replication"); } + private static final ArrayList<String> STORE_TYPE_LIST = new ArrayList<>(); + + static { + STORE_TYPE_LIST.add("database-store"); + STORE_TYPE_LIST.add("file-store"); + } + + private void parseStoreConfiguration(final Element e, final Configuration mainConfig) { + for (String storeType : STORE_TYPE_LIST) { + NodeList storeNodeList = e.getElementsByTagName(storeType); + if (storeNodeList.getLength() > 0) { + Element storeNode = (Element) storeNodeList.item(0); + if (storeNode.getTagName().equals("database-store")) { + mainConfig.setStoreConfiguration(createDatabaseStoreConfig(storeNode)); + } + else if (storeNode.getTagName().equals("file-store")) { + mainConfig.setStoreConfiguration(createFileStoreConfig(storeNode)); + } + } + } + } + private void parseHAPolicyConfiguration(final Element e, final Configuration mainConfig) { for (String haType : HA_LIST) { NodeList haNodeList = e.getElementsByTagName(haType); @@ -1105,6 +1135,20 @@ public final class FileConfigurationParser extends XMLConfigurationUtil { return null; } + private DatabaseStorageConfiguration createDatabaseStoreConfig(Element storeNode) { + NodeList databaseStoreNode = storeNode.getElementsByTagName("database-store"); + + DatabaseStorageConfiguration conf = new DatabaseStorageConfiguration(); + conf.setBindingsTableName(getString(storeNode, "bindings-table-name", conf.getBindingsTableName(), Validators.NO_CHECK)); + conf.setMessageTableName(getString(storeNode, "message-table-name", conf.getMessageTableName(), Validators.NO_CHECK)); + conf.setJdbcConnectionUrl(getString(storeNode, "jdbc-connection-url", conf.getJdbcConnectionUrl(), Validators.NO_CHECK)); + return conf; + } + + private FileStorageConfiguration createFileStoreConfig(Element storeNode) { + return new FileStorageConfiguration(); + } + private void parseBroadcastGroupConfiguration(final Element e, final Configuration mainConfig) { String name = e.getAttribute("name");
