Repository: incubator-eagle Updated Branches: refs/heads/master 8b504792b -> 41cb7e1a0
[EAGLE-757] fix alert JdbcDatabaseHandler connection issue Author: wujinhu <wujinhu...@126.com> Closes #648 from wujinhu/EAGLE-757. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/41cb7e1a Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/41cb7e1a Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/41cb7e1a Branch: refs/heads/master Commit: 41cb7e1a06c6d2479d27d0d8bba160094289efba Parents: 8b50479 Author: wujinhu <wujinhu...@126.com> Authored: Mon Nov 14 16:09:05 2016 +0800 Committer: wujinhu <wujinhu...@126.com> Committed: Mon Nov 14 16:09:05 2016 +0800 ---------------------------------------------------------------------- .../eagle/alert/metadata/MetadataUtils.java | 21 ++++- .../metadata/impl/JdbcDatabaseHandler.java | 86 ++++++++++++++++---- .../metadata/impl/JdbcMetadataDaoImpl.java | 3 +- .../alert/metadata/impl/JdbcSchemaManager.java | 2 +- .../alert/metadata/impl/MetadataDaoFactory.java | 19 +++-- .../metadata/impl/MongoMetadataDaoImpl.java | 8 +- .../alert/resource/impl/JdbcImplTest.java | 5 +- .../alert/resource/impl/MongoImplTest.java | 5 +- .../src/test/resources/application-mongo.conf | 12 +-- .../src/test/resources/application-mysql.conf | 11 +-- .../src/test/resources/application.conf | 14 ++-- .../store/jdbc/JDBCDataSourceConfig.java | 23 ++++-- .../metadata/store/jdbc/JDBCMetadataStore.java | 2 + .../jdbc/provider/JDBCDataSourceProvider.java | 2 +- .../JDBCMetadataStoreConfigProvider.java | 13 ++- .../store/jdbc/JDBCDataSourceProviderTest.java | 2 +- .../src/test/resources/application.conf | 2 +- .../jpm/mr/running/parser/MRJobParser.java | 1 - .../eagle/jpm/util/JobNameNormalization.java | 2 +- .../service/JDBCSecurityMetadataDAO.java | 6 +- .../src/test/resources/application.conf | 2 +- .../src/main/resources/application.conf | 4 +- 22 files changed, 170 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/MetadataUtils.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/MetadataUtils.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/MetadataUtils.java index 841a0a0..3e03b57 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/MetadataUtils.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/MetadataUtils.java @@ -36,6 +36,15 @@ import java.sql.SQLException; public class MetadataUtils { private static final Logger LOG = LoggerFactory.getLogger(MetadataUtils.class); + public static final String META_DATA = "metadata"; + public static final String ALERT_META_DATA_DAO = "metadataDao"; + public static final String JDBC_USERNAME_PATH = "jdbc.username"; + public static final String JDBC_PASSWORD_PATH = "jdbc.password"; + public static final String JDBC_DRIVER_PATH = "jdbc.driverClassName"; + public static final String JDBC_DATABASE_PATH = "jdbc.database"; + public static final String JDBC_CONNECTION_PATH = "jdbc.connection"; + public static final String JDBC_CONNECTION_PROPERTIES_PATH = "jdbc.connectionProperties"; + public static final String MONGO_CONNECTION_PATH = "mongo.connection"; public static <T> String getKey(T t) { if (t instanceof StreamDefinition) { @@ -67,11 +76,17 @@ public class MetadataUtils { public static Connection getJdbcConnection(Config config) { Connection connection = null; - String conn = config.getString("connection"); try { - connection = DriverManager.getConnection(conn); + if (config.hasPath(JDBC_USERNAME_PATH)) { + connection = DriverManager.getConnection( + config.getString(JDBC_CONNECTION_PATH), + config.getString(JDBC_USERNAME_PATH), + config.getString(JDBC_PASSWORD_PATH)); + } else { + connection = DriverManager.getConnection(config.getString(JDBC_CONNECTION_PATH)); + } } catch (SQLException e) { - e.printStackTrace(); + LOG.error(e.getMessage(), e); } return connection; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcDatabaseHandler.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcDatabaseHandler.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcDatabaseHandler.java index 571cdda..550eb00 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcDatabaseHandler.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcDatabaseHandler.java @@ -18,6 +18,7 @@ package org.apache.eagle.alert.metadata.impl; +import org.apache.commons.dbcp.BasicDataSource; import org.apache.eagle.alert.metadata.MetadataUtils; import org.apache.eagle.alert.metadata.resource.OpResult; import com.fasterxml.jackson.core.JsonProcessingException; @@ -27,6 +28,7 @@ import com.typesafe.config.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.sql.DataSource; import java.io.IOException; import java.sql.*; import java.util.HashMap; @@ -51,7 +53,7 @@ public class JdbcDatabaseHandler { private Map<String, String> tblNameMap = new HashMap<>(); private static final ObjectMapper mapper = new ObjectMapper(); - private Connection connection = null; + private DataSource dataSource; static { mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @@ -61,11 +63,20 @@ public class JdbcDatabaseHandler { // "jdbc:mysql://dbhost/database?" + "user=sqluser&password=sqluserpw" this.tblNameMap = JdbcSchemaManager.tblNameMap; try { - Class.forName("com.mysql.jdbc.Driver"); JdbcSchemaManager.getInstance().init(config); - connection = MetadataUtils.getJdbcConnection(config); - } catch (ClassNotFoundException e) { - e.printStackTrace(); + BasicDataSource bDatasource = new BasicDataSource(); + bDatasource.setDriverClassName(config.getString(MetadataUtils.JDBC_DRIVER_PATH)); + if (config.hasPath(MetadataUtils.JDBC_USERNAME_PATH)) { + bDatasource.setUsername(config.getString(MetadataUtils.JDBC_USERNAME_PATH)); + bDatasource.setPassword(config.getString(MetadataUtils.JDBC_PASSWORD_PATH)); + } + bDatasource.setUrl(config.getString(MetadataUtils.JDBC_CONNECTION_PATH)); + if (config.hasPath(MetadataUtils.JDBC_CONNECTION_PROPERTIES_PATH)) { + bDatasource.setConnectionProperties(config.getString(MetadataUtils.JDBC_CONNECTION_PROPERTIES_PATH)); + } + this.dataSource = bDatasource; + } catch (Exception e) { + LOG.error(e.getMessage(), e); } } @@ -85,7 +96,9 @@ public class JdbcDatabaseHandler { Savepoint savepoint = null; String key = null; String value = null; + Connection connection = null; try { + connection = dataSource.getConnection(); statement = connection.prepareStatement(String.format(INSERT_STATEMENT, tb)); key = MetadataUtils.getKey(t); value = mapper.writeValueAsString(t); @@ -102,7 +115,7 @@ public class JdbcDatabaseHandler { connection.commit(); } catch (SQLException e) { LOG.error(e.getMessage(), e.getCause()); - if (e.getMessage().toLowerCase().contains("duplicate")) { + if (e.getMessage().toLowerCase().contains("duplicate") && connection != null) { LOG.info("Detected duplicated entity"); try { connection.rollback(savepoint); @@ -124,6 +137,13 @@ public class JdbcDatabaseHandler { LOG.error("Failed to close statement: {}", e.getMessage(), e.getCause()); } } + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + LOG.error("Failed to close statement: {}", e.getMessage(), e.getCause()); + } + } } return result; } @@ -131,7 +151,9 @@ public class JdbcDatabaseHandler { private <T> OpResult update(String tb, String key, String value) throws SQLException { OpResult result = new OpResult(); PreparedStatement statement = null; + Connection connection = null; try { + connection = dataSource.getConnection(); statement = connection.prepareStatement(String.format(UPDATE_STATEMENT, tb)); Clob clob = connection.createClob(); clob.setString(1, value); @@ -141,13 +163,20 @@ public class JdbcDatabaseHandler { int status = statement.executeUpdate(); LOG.info("update {} entities from table {}", status, tb); } catch (SQLException e) { - e.printStackTrace(); + LOG.error(e.getMessage(), e); result.code = OpResult.FAILURE; result.message = e.getMessage(); } finally { if (statement != null) { statement.close(); } + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + LOG.error("Failed to close statement: {}", e.getMessage(), e.getCause()); + } + } } return result; } @@ -182,7 +211,9 @@ public class JdbcDatabaseHandler { public <T> T executeSelectByIdStatement(Class<T> clz, String id) { String tb = getTableName(clz.getSimpleName()); List<T> result = new LinkedList<>(); + Connection connection = null; try { + connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(String.format(QUERY_CONDITION_STATEMENT, tb)); statement.setString(1, id); ResultSet rs = statement.executeQuery(); @@ -199,7 +230,15 @@ public class JdbcDatabaseHandler { rs.close(); statement.close(); } catch (SQLException e) { - e.printStackTrace(); + LOG.error(e.getMessage(), e); + } finally { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + LOG.error("Failed to close statement: {}", e.getMessage(), e.getCause()); + } + } } if (result.isEmpty()) { return null; @@ -211,7 +250,9 @@ public class JdbcDatabaseHandler { public <T> List<T> executeSelectStatement(Class<T> clz, String query) { String tb = getTableName(clz.getSimpleName()); List<T> result = new LinkedList<>(); + Connection connection = null; try { + connection = dataSource.getConnection(); Statement statement = connection.createStatement(); ResultSet rs = statement.executeQuery(query); while (rs.next()) { @@ -227,7 +268,15 @@ public class JdbcDatabaseHandler { rs.close(); statement.close(); } catch (SQLException e) { - e.printStackTrace(); + LOG.error(e.getMessage(), e); + } finally { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + LOG.error("Failed to close statement: {}", e.getMessage(), e.getCause()); + } + } } return result; } @@ -235,7 +284,9 @@ public class JdbcDatabaseHandler { public <T> OpResult remove(String clzName, String key) { String tb = getTableName(clzName); OpResult result = new OpResult(); + Connection connection = null; try { + connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(String.format(DELETE_STATEMENT, tb, key)); statement.setString(1, key); int status = statement.executeUpdate(); @@ -246,20 +297,21 @@ public class JdbcDatabaseHandler { } catch (SQLException e) { result.code = OpResult.FAILURE; result.message = e.getMessage(); - //e.printStackTrace(); + LOG.error(e.getMessage(), e); + } finally { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + LOG.error("Failed to close statement: {}", e.getMessage(), e.getCause()); + } + } } return result; } public void close() throws IOException { //JdbcSchemaManager.getInstance().shutdown(); - try { - if (this.connection != null) { - this.connection.close(); - } - } catch (SQLException e) { - LOG.warn(e.getMessage()); - } } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java index d748ed1..5e0a0e7 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java @@ -23,6 +23,7 @@ import org.apache.eagle.alert.coordination.model.internal.Topology; import org.apache.eagle.alert.engine.coordinator.*; import org.apache.eagle.alert.engine.model.AlertPublishEvent; import org.apache.eagle.alert.metadata.IMetadataDao; +import org.apache.eagle.alert.metadata.MetadataUtils; import org.apache.eagle.alert.metadata.resource.Models; import org.apache.eagle.alert.metadata.resource.OpResult; import com.google.inject.Inject; @@ -40,7 +41,7 @@ public class JdbcMetadataDaoImpl implements IMetadataDao { @Inject public JdbcMetadataDaoImpl(Config config) { - handler = new JdbcDatabaseHandler(config); + handler = new JdbcDatabaseHandler(config.getConfig(MetadataUtils.META_DATA)); } @Override http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcSchemaManager.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcSchemaManager.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcSchemaManager.java index 33d9a06..4568726 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcSchemaManager.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcSchemaManager.java @@ -84,7 +84,7 @@ public class JdbcSchemaManager { this.platform = PlatformFactory.createNewPlatformInstance("mysql"); connection = MetadataUtils.getJdbcConnection(config); - String dbName = config.getString("database"); + String dbName = config.getString(MetadataUtils.JDBC_DATABASE_PATH); this.database = platform.readModelFromDatabase(connection, dbName); LOG.info("Loaded " + database); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MetadataDaoFactory.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MetadataDaoFactory.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MetadataDaoFactory.java index b03fc61..6df99d9 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MetadataDaoFactory.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MetadataDaoFactory.java @@ -19,6 +19,7 @@ package org.apache.eagle.alert.metadata.impl; import org.apache.eagle.alert.metadata.IMetadataDao; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; +import org.apache.eagle.alert.metadata.MetadataUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,25 +38,25 @@ public class MetadataDaoFactory { private MetadataDaoFactory() { Config config = ConfigFactory.load(); - Config datastoreConfig = config.getConfig("datastore"); - if (datastoreConfig == null) { - LOG.warn("datastore is not configured, use in-memory store !!!"); - dao = new InMemMetadataDaoImpl(datastoreConfig); + Config metaDataConfig = config.getConfig(MetadataUtils.META_DATA); + if (metaDataConfig != null) { + LOG.warn("metadata is not configured, use in-memory store !!!"); + dao = new InMemMetadataDaoImpl(metaDataConfig); } else { - String clsName = datastoreConfig.getString("metadataDao"); + String clsName = metaDataConfig.getString(MetadataUtils.ALERT_META_DATA_DAO); Class<?> clz; try { clz = Thread.currentThread().getContextClassLoader().loadClass(clsName); if (IMetadataDao.class.isAssignableFrom(clz)) { Constructor<?> cotr = clz.getConstructor(Config.class); - LOG.info("metadada DAO loaded: " + clsName); - dao = (IMetadataDao) cotr.newInstance(datastoreConfig); + LOG.info("metadata.alertMetadataDao loaded: " + clsName); + dao = (IMetadataDao) cotr.newInstance(metaDataConfig); } else { - throw new Exception("metadataDao configuration need to be implementation of IMetadataDao! "); + throw new Exception("metadata.metadataDao configuration need to be implementation of IMetadataDao! "); } } catch (Exception e) { LOG.error("error when initialize the dao, fall back to in memory mode!", e); - dao = new InMemMetadataDaoImpl(datastoreConfig); + dao = new InMemMetadataDaoImpl(metaDataConfig); } } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MongoMetadataDaoImpl.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MongoMetadataDaoImpl.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MongoMetadataDaoImpl.java index 498f540..af2e231 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MongoMetadataDaoImpl.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/MongoMetadataDaoImpl.java @@ -64,6 +64,8 @@ public class MongoMetadataDaoImpl implements IMetadataDao { private static final ObjectMapper mapper = new ObjectMapper(); private static final int DEFAULT_CAPPED_MAX_SIZE = 500 * 1024 * 1024; private static final int DEFAULT_CAPPED_MAX_DOCUMENTS = 20000; + private static final String MANGO_CAPPED_MAX_SIZE = "mongo.cappedMaxSize"; + private static final String MANGO_CAPPED_MAX_DOCUMENTS = "mongo.cappedMaxDocuments"; static { mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @@ -97,9 +99,9 @@ public class MongoMetadataDaoImpl implements IMetadataDao { @Inject public MongoMetadataDaoImpl(Config config) { - this.connection = config.getString("connection"); - this.cappedMaxSize = config.hasPath("cappedMaxSize") ? config.getInt("cappedMaxSize") : DEFAULT_CAPPED_MAX_SIZE; - this.cappedMaxDocuments = config.hasPath("cappedMaxDocuments") ? config.getInt("cappedMaxDocuments") : DEFAULT_CAPPED_MAX_DOCUMENTS; + this.connection = config.getString(MetadataUtils.MONGO_CONNECTION_PATH); + this.cappedMaxSize = config.hasPath(MANGO_CAPPED_MAX_SIZE) ? config.getInt(MANGO_CAPPED_MAX_SIZE) : DEFAULT_CAPPED_MAX_SIZE; + this.cappedMaxDocuments = config.hasPath(MANGO_CAPPED_MAX_DOCUMENTS) ? config.getInt(MANGO_CAPPED_MAX_DOCUMENTS) : DEFAULT_CAPPED_MAX_DOCUMENTS; this.client = new MongoClient(new MongoClientURI(this.connection)); init(); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/JdbcImplTest.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/JdbcImplTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/JdbcImplTest.java index 8812f87..158c0c2 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/JdbcImplTest.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/JdbcImplTest.java @@ -29,6 +29,7 @@ import org.apache.eagle.alert.engine.coordinator.Publishment; import org.apache.eagle.alert.engine.coordinator.PublishmentType; import org.apache.eagle.alert.engine.coordinator.StreamingCluster; import org.apache.eagle.alert.metadata.IMetadataDao; +import org.apache.eagle.alert.metadata.MetadataUtils; import org.apache.eagle.alert.metadata.impl.JdbcMetadataDaoImpl; import org.apache.eagle.alert.metadata.resource.OpResult; import org.junit.*; @@ -47,8 +48,8 @@ public class JdbcImplTest { public static void setup() { System.setProperty("config.resource", "/application-mysql.conf"); ConfigFactory.invalidateCaches(); - Config config = ConfigFactory.load().getConfig("datastore"); - dao = new JdbcMetadataDaoImpl(config); + Config config = ConfigFactory.load(); + dao = new JdbcMetadataDaoImpl(config.getConfig(MetadataUtils.META_DATA)); } @AfterClass http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/MongoImplTest.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/MongoImplTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/MongoImplTest.java index 213943d..4328be3 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/MongoImplTest.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/java/org/apache/eagle/service/alert/resource/impl/MongoImplTest.java @@ -34,6 +34,7 @@ import org.apache.eagle.alert.coordination.model.internal.Topology; import org.apache.eagle.alert.engine.coordinator.*; import org.apache.eagle.alert.engine.model.AlertPublishEvent; import org.apache.eagle.alert.metadata.IMetadataDao; +import org.apache.eagle.alert.metadata.MetadataUtils; import org.apache.eagle.alert.metadata.impl.MongoMetadataDaoImpl; import org.apache.eagle.alert.metadata.resource.OpResult; import org.junit.AfterClass; @@ -72,8 +73,8 @@ public class MongoImplTest { System.setProperty("config.resource", "/application-mongo.conf"); ConfigFactory.invalidateCaches(); - Config config = ConfigFactory.load().getConfig("datastore"); - dao = new MongoMetadataDaoImpl(config); + Config config = ConfigFactory.load(); + dao = new MongoMetadataDaoImpl(config.getConfig(MetadataUtils.META_DATA)); } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mongo.conf ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mongo.conf b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mongo.conf index bdc632c..4c6fba2 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mongo.conf +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mongo.conf @@ -13,10 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -{ - "datastore": { - "metadataDao": "org.apache.eagle.alert.metadata.impl.InMemMetadataDaoImpl", - "connection": "mongodb://localhost:27017", - "cappedSize": 20000 + +metadata { + metadataDao = org.apache.eagle.alert.metadata.impl.InMemMetadataDaoImpl + mongo { + connection = "mongodb://localhost:27017" + cappedSize = 20000 } } + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mysql.conf ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mysql.conf b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mysql.conf index 55a7121..2a1aa2c 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mysql.conf +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application-mysql.conf @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -{ - "datastore": { - "metadataDao": "org.apache.eagle.alert.metadata.impl.JdbcMetadataDaoImpl", - "connection": "jdbc:mysql://localhost:3306/alert_metadata?user=root&password=&createDatabaseIfNotExist=true", - "database": "alert_metadata" +metadata { + metadataDao = org.apache.eagle.alert.metadata.impl.JdbcMetadataDaoImpl + jdbc { + connection = "jdbc:mysql://localhost:3306/alert_metadata?user=root&password=&createDatabaseIfNotExist=true" + database = "alert_metadata" + driverClassName = com.mysql.jdbc.Driver } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application.conf b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application.conf index cdb0b25..2304cd2 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application.conf +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/test/resources/application.conf @@ -14,11 +14,13 @@ # limitations under the License. -{ - "datastore": { - "metadataDao": "org.apache.eagle.alert.metadata.impl.InMemMetadataDaoImpl", - "connection": "localhost:27017", - "properties": { - } +metadata { + metadataDao = org.apache.eagle.alert.metadata.impl.InMemMetadataDaoImpl + jdbc { + url = "localhost:27017" + } + properties { + } } + http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceConfig.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceConfig.java b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceConfig.java index 6b629ac..e9d12c0 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceConfig.java +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceConfig.java @@ -44,7 +44,8 @@ public class JDBCDataSourceConfig { private String username; private String password; private String driverClassName = DEFAULT_DRIVER_CLASS; - private String url = DEFAULT_URL; + private String connection = DEFAULT_URL; + private String database; private String connectionProperties = DEFAULT_CONNECTION_PROPERTIES; public String getPassword() { @@ -71,12 +72,20 @@ public class JDBCDataSourceConfig { this.driverClassName = driverClassName; } - public String getUrl() { - return url; + public String getConnection() { + return connection; } - public void setUrl(String url) { - this.url = url; + public void setConnection(String connection) { + this.connection = connection; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; } public String getConnectionProperties() { @@ -89,7 +98,7 @@ public class JDBCDataSourceConfig { @Override public String toString() { - return String.format("%s { \n driverClassName=%s \n url=%s \n connectionProperties=%s \n username=%s \n password=*****\n}", - CONFIG_PREFIX, driverClassName, url, username, connectionProperties); + return String.format("%s { \n driverClassName=%s \n connection=%s \n database=%s \n connectionProperties=%s \n username=%s \n password=*****\n}", + CONFIG_PREFIX,driverClassName,connection, database, connectionProperties, username); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java index 98d4015..9a7e7ab 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/JDBCMetadataStore.java @@ -19,6 +19,7 @@ package org.apache.eagle.metadata.store.jdbc; import com.google.inject.Singleton; import org.apache.eagle.alert.metadata.IMetadataDao; +import org.apache.eagle.alert.metadata.MetadataUtils; import org.apache.eagle.alert.metadata.impl.JdbcMetadataDaoImpl; import org.apache.eagle.metadata.persistence.MetadataStore; import org.apache.eagle.metadata.service.ApplicationEntityService; @@ -33,6 +34,7 @@ import javax.sql.DataSource; public class JDBCMetadataStore extends MetadataStore { @Override protected void configure() { + System.setProperty(MetadataUtils.META_DATA + "." + MetadataUtils.ALERT_META_DATA_DAO, JdbcMetadataDaoImpl.class.getCanonicalName()); bind(IMetadataDao.class).to(JdbcMetadataDaoImpl.class).in(Singleton.class); bind(DataSource.class).toProvider(JDBCDataSourceProvider.class).in(Singleton.class); bind(JDBCDataSourceConfig.class).toProvider(JDBCMetadataStoreConfigProvider.class).in(Singleton.class); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java index 751c853..7495ef5 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCDataSourceProvider.java @@ -38,7 +38,7 @@ public class JDBCDataSourceProvider implements Provider<DataSource> { datasource.setDriverClassName(config.getDriverClassName()); datasource.setUsername(config.getUsername()); datasource.setPassword(config.getPassword()); - datasource.setUrl(config.getUrl()); + datasource.setUrl(config.getConnection()); datasource.setConnectionProperties(config.getConnectionProperties()); LOGGER.info("Register JDBCDataSourceShutdownHook"); Runtime.getRuntime().addShutdownHook(new Thread("JDBCDataSourceShutdownHook") { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCMetadataStoreConfigProvider.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCMetadataStoreConfigProvider.java b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCMetadataStoreConfigProvider.java index fdc81ff..21ba6fe 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCMetadataStoreConfigProvider.java +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/main/java/org/apache/eagle/metadata/store/jdbc/provider/JDBCMetadataStoreConfigProvider.java @@ -32,9 +32,14 @@ public class JDBCMetadataStoreConfigProvider implements Provider<JDBCDataSourceC @Override public JDBCDataSourceConfig get() { - JDBCDataSourceConfig dataSourceConfig = new ObjectMapper().convertValue( - config.getConfig(JDBCDataSourceConfig.CONFIG_PREFIX).root().unwrapped(), JDBCDataSourceConfig.class); - LOGGER.info("JDBC Configuration: {}", dataSourceConfig); - return dataSourceConfig; + try { + JDBCDataSourceConfig dataSourceConfig = new ObjectMapper().convertValue( + config.getConfig(JDBCDataSourceConfig.CONFIG_PREFIX).root().unwrapped(), JDBCDataSourceConfig.class); + LOGGER.info("JDBC Configuration: {}", dataSourceConfig); + return dataSourceConfig; + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + return null; + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceProviderTest.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceProviderTest.java b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceProviderTest.java index 1ae632d..3e33026 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceProviderTest.java +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/java/org/apache/eagle/metadata/store/jdbc/JDBCDataSourceProviderTest.java @@ -42,7 +42,7 @@ public class JDBCDataSourceProviderTest extends JDBCMetadataTestBase { @Test public void testDataSourceConfig() { - Assert.assertEquals("jdbc:h2:mem:test;INIT=RUNSCRIPT FROM './src/test/resources/init.sql'", dataSourceConfig.getUrl()); + Assert.assertEquals("jdbc:h2:mem:test;INIT=RUNSCRIPT FROM './src/test/resources/init.sql'", dataSourceConfig.getConnection()); Assert.assertEquals(null, dataSourceConfig.getUsername()); Assert.assertEquals(null, dataSourceConfig.getPassword()); Assert.assertEquals("encoding=UTF8;timeout=60", dataSourceConfig.getConnectionProperties()); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/resources/application.conf b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/resources/application.conf index dec8129..a834c34 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/resources/application.conf +++ b/eagle-core/eagle-metadata/eagle-metadata-jdbc/src/test/resources/application.conf @@ -20,7 +20,7 @@ username = null password = null driverClassName = "org.h2.Driver" - url = "jdbc:h2:mem:test;INIT=RUNSCRIPT FROM './src/test/resources/init.sql'" + connection = "jdbc:h2:mem:test;INIT=RUNSCRIPT FROM './src/test/resources/init.sql'" connectionProperties = "encoding=UTF8;timeout=60" } } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java index 7b90dfc..edf4bbb 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java @@ -565,7 +565,6 @@ public class MRJobParser implements Runnable { fetchMRRunningInfo(); } catch (Exception e) { LOG.warn("exception found when process application {}, {}", app.getId(), e); - e.printStackTrace(); } finally { for (String jobId : mrJobEntityMap.keySet()) { mrJobEntityCreationHandler.add(mrJobEntityMap.get(jobId)); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java index 0b38c67..0337dd5 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java @@ -62,7 +62,7 @@ public class JobNameNormalization { try { // load normalization rules String key = JOB_NAME_NORMALIZATION_RULES_KEY.toLowerCase(); - String value = conf.getString(key) != null ? conf.getString(key) : Constants.JOB_NAME_NORMALIZATION_RULES_KEY_DEFAULT; + String value = conf.hasPath(key) && conf.getString(key) != null ? conf.getString(key) : Constants.JOB_NAME_NORMALIZATION_RULES_KEY_DEFAULT; // multiple rules are concatenated with semicolon, i.e. ; String[] rules = value.split(MULTIPLE_RULE_DILIMITER); for (String rule : rules) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java index e9662e6..e089bf9 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java @@ -262,14 +262,14 @@ public class JDBCSecurityMetadataDAO implements ISecurityMetadataDAO { Connection connection; try { if (config.hasPath("metadata.jdbc.username")) { - connection = DriverManager.getConnection(config.getString("metadata.jdbc.url"), + connection = DriverManager.getConnection(config.getString("metadata.jdbc.connection"), config.getString("metadata.jdbc.username"), config.getString("metadata.jdbc.password")); } else { - connection = DriverManager.getConnection(config.getString("metadata.jdbc.url")); + connection = DriverManager.getConnection(config.getString("metadata.jdbc.connection")); } } catch (Exception e) { - LOG.error("error get connection for {}", config.getString("metadata.jdbc.url"), e); + LOG.error("error get connection for {}", config.getString("metadata.jdbc.connection"), e); throw e; } return connection; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-security/eagle-security-common/src/test/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/test/resources/application.conf b/eagle-security/eagle-security-common/src/test/resources/application.conf index 2b69406..4d041e4 100644 --- a/eagle-security/eagle-security-common/src/test/resources/application.conf +++ b/eagle-security/eagle-security-common/src/test/resources/application.conf @@ -71,7 +71,7 @@ "metadata":{ "jdbc":{ "driverClassName":"org.h2.Driver" - "url":"jdbc:h2:./eagle-test" + "connection":"jdbc:h2:./eagle-test" } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/41cb7e1a/eagle-server/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-server/src/main/resources/application.conf b/eagle-server/src/main/resources/application.conf index 557fa00..59df87b 100644 --- a/eagle-server/src/main/resources/application.conf +++ b/eagle-server/src/main/resources/application.conf @@ -71,11 +71,13 @@ storage { # --------------------------------------------- metadata { store = org.apache.eagle.metadata.service.memory.MemoryMetadataStore + metadataDao = org.apache.eagle.alert.metadata.impl.InMemMetadataDaoImpl jdbc { username = "root" password = "" driverClassName = com.mysql.jdbc.Driver - url = "jdbc:mysql://server.eagle.apache.org:3306/eagle" + database = "eagle" + connection = "jdbc:mysql://server.eagle.apache.org:3306/eagle?user=root" } }
