This is an automated email from the ASF dual-hosted git repository.
baiyangtx pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 604d1c140 [AMORO-3340]: Database based http session store (#3341)
604d1c140 is described below
commit 604d1c140d01f17b88f68ba3508b81e83ab26fe1
Author: baiyangtx <[email protected]>
AuthorDate: Fri Nov 29 16:57:18 2024 +0800
[AMORO-3340]: Database based http session store (#3341)
* database based http session store
* docs
* spotless
---
.../apache/amoro/server/AmoroManagementConf.java | 6 +
.../apache/amoro/server/AmoroServiceContainer.java | 12 +-
...FactoryProvider.java => DataSourceFactory.java} | 118 ++----------------
.../persistence/HttpSessionHandlerFactory.java | 90 ++++++++++++++
.../persistence/SqlSessionFactoryProvider.java | 132 +--------------------
.../converter/Object2ByteArrayConvert.java | 6 +-
.../src/main/resources/derby/ams-derby-init.sql | 16 +++
.../src/main/resources/mysql/ams-mysql-init.sql | 17 +++
amoro-ams/src/main/resources/mysql/upgrade.sql | 20 +++-
.../main/resources/postgres/ams-postgres-init.sql | 32 +++++
amoro-ams/src/main/resources/postgres/upgrade.sql | 36 +++++-
.../TestIcebergServerTableDescriptor.java | 6 +-
.../amoro/server/table/DerbyPersistence.java | 6 +-
dist/src/main/amoro-bin/conf/config.yaml | 1 +
docs/admin-guides/deployment.md | 1 +
15 files changed, 256 insertions(+), 243 deletions(-)
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
index c21a2ca1b..c76133774 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
@@ -256,6 +256,12 @@ public class AmoroManagementConf {
.defaultValue("token")
.withDescription("The authentication used by REST APIs, token
(default) or basic.");
+ public static final ConfigOption<Duration> HTTP_SERVER_SESSION_TIMEOUT =
+ ConfigOptions.key("http-server.session-timeout")
+ .durationType()
+ .defaultValue(Duration.ofDays(7))
+ .withDescription("Timeout for http session.");
+
public static final ConfigOption<Integer> OPTIMIZING_COMMIT_THREAD_COUNT =
ConfigOptions.key("self-optimizing.commit-thread-count")
.intType()
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
index fd35da72f..5c5ee9c46 100644
--- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
+++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroServiceContainer.java
@@ -35,6 +35,8 @@ import org.apache.amoro.server.dashboard.utils.AmsUtil;
import org.apache.amoro.server.dashboard.utils.CommonUtil;
import org.apache.amoro.server.manager.EventsManager;
import org.apache.amoro.server.manager.MetricManager;
+import org.apache.amoro.server.persistence.DataSourceFactory;
+import org.apache.amoro.server.persistence.HttpSessionHandlerFactory;
import org.apache.amoro.server.persistence.SqlSessionFactoryProvider;
import org.apache.amoro.server.resource.ContainerMetadata;
import org.apache.amoro.server.resource.OptimizerManager;
@@ -64,11 +66,12 @@ import org.apache.amoro.utils.IcebergThreadPools;
import org.apache.amoro.utils.JacksonUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.iceberg.SystemProperties;
-import org.eclipse.jetty.server.session.SessionHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
+import javax.sql.DataSource;
+
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -88,6 +91,7 @@ public class AmoroServiceContainer {
public static final String SERVER_CONFIG_FILENAME = "config.yaml";
private final HighAvailabilityContainer haContainer;
+ private DataSource dataSource;
private DefaultTableService tableService;
private DefaultOptimizingService optimizingService;
private TerminalManager terminalManager;
@@ -244,7 +248,8 @@ public class AmoroServiceContainer {
config -> {
config.addStaticFiles(dashboardServer.configStaticFiles());
config.addStaticFiles("/META-INF/resources/webjars",
Location.CLASSPATH);
- config.sessionHandler(SessionHandler::new);
+ config.sessionHandler(
+ () ->
HttpSessionHandlerFactory.createSessionHandler(dataSource, serviceConfig));
config.enableCorsForAllOrigins();
config.jsonMapper(JavalinJsonMapper.createDefaultJsonMapper());
config.showJavalinBanner = false;
@@ -439,7 +444,8 @@ public class AmoroServiceContainer {
expandedConfigurationMap.putAll(envConfig);
serviceConfig = Configurations.fromObjectMap(expandedConfigurationMap);
AmoroManagementConfValidator.validateConfig(serviceConfig);
- SqlSessionFactoryProvider.getInstance().init(serviceConfig);
+ dataSource = DataSourceFactory.createDataSource(serviceConfig);
+ SqlSessionFactoryProvider.getInstance().init(dataSource);
}
private Map<String, Object> initEnvConfig() {
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/persistence/SqlSessionFactoryProvider.java
b/amoro-ams/src/main/java/org/apache/amoro/server/persistence/DataSourceFactory.java
similarity index 56%
copy from
amoro-ams/src/main/java/org/apache/amoro/server/persistence/SqlSessionFactoryProvider.java
copy to
amoro-ams/src/main/java/org/apache/amoro/server/persistence/DataSourceFactory.java
index 14f4b8fdf..26d193215 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/persistence/SqlSessionFactoryProvider.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/persistence/DataSourceFactory.java
@@ -18,39 +18,16 @@
package org.apache.amoro.server.persistence;
-import static com.github.pagehelper.page.PageAutoDialect.registerDialectAlias;
-
-import com.github.pagehelper.PageInterceptor;
-import com.github.pagehelper.dialect.helper.MySqlDialect;
-import com.github.pagehelper.dialect.helper.PostgreSqlDialect;
-import com.github.pagehelper.dialect.helper.SqlServer2012Dialect;
import org.apache.amoro.config.Configurations;
import org.apache.amoro.server.AmoroManagementConf;
-import org.apache.amoro.server.persistence.mapper.ApiTokensMapper;
-import org.apache.amoro.server.persistence.mapper.CatalogMetaMapper;
-import org.apache.amoro.server.persistence.mapper.OptimizerMapper;
-import org.apache.amoro.server.persistence.mapper.OptimizingMapper;
-import org.apache.amoro.server.persistence.mapper.PlatformFileMapper;
-import org.apache.amoro.server.persistence.mapper.ResourceMapper;
-import org.apache.amoro.server.persistence.mapper.TableBlockerMapper;
-import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
-import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
import org.apache.commons.dbcp2.BasicDataSource;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.impl.BaseObjectPoolConfig;
import org.apache.ibatis.jdbc.ScriptRunner;
-import org.apache.ibatis.mapping.DatabaseIdProvider;
-import org.apache.ibatis.mapping.Environment;
-import org.apache.ibatis.mapping.VendorDatabaseIdProvider;
-import org.apache.ibatis.session.Configuration;
-import org.apache.ibatis.session.SqlSession;
-import org.apache.ibatis.session.SqlSessionFactory;
-import org.apache.ibatis.session.SqlSessionFactoryBuilder;
-import org.apache.ibatis.transaction.TransactionFactory;
-import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.sql.DataSource;
+
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
@@ -61,36 +38,21 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.ResultSet;
-import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
-import java.util.Properties;
-
-public class SqlSessionFactoryProvider {
- private static final Logger LOG =
LoggerFactory.getLogger(SqlSessionFactoryProvider.class);
+/** Factory to help create data source */
+public class DataSourceFactory {
+ private static final Logger LOG =
LoggerFactory.getLogger(DataSourceFactory.class);
private static final String DERBY_INIT_SQL_SCRIPT =
"derby/ams-derby-init.sql";
private static final String MYSQL_INIT_SQL_SCRIPT =
"mysql/ams-mysql-init.sql";
private static final String POSTGRES_INIT_SQL_SCRIPT =
"postgres/ams-postgres-init.sql";
- private static final SqlSessionFactoryProvider INSTANCE = new
SqlSessionFactoryProvider();
-
- public static SqlSessionFactoryProvider getInstance() {
- return INSTANCE;
- }
-
- private static String dbType;
-
- private volatile SqlSessionFactory sqlSessionFactory;
-
- public void init(Configurations config) throws SQLException {
-
- registerDialectAliases();
-
+ public static DataSource createDataSource(Configurations config) {
BasicDataSource dataSource = new BasicDataSource();
dataSource.setUrl(config.getString(AmoroManagementConf.DB_CONNECTION_URL));
dataSource.setDriverClassName(config.getString(AmoroManagementConf.DB_DRIVER_CLASS_NAME));
- dbType = config.getString(AmoroManagementConf.DB_TYPE);
+ String dbType = config.getString(AmoroManagementConf.DB_TYPE);
if (AmoroManagementConf.DB_TYPE_MYSQL.equals(dbType)
|| AmoroManagementConf.DB_TYPE_POSTGRES.equals(dbType)) {
dataSource.setUsername(config.getString(AmoroManagementConf.DB_USER_NAME));
@@ -113,53 +75,12 @@ public class SqlSessionFactoryProvider {
dataSource.setSoftMinEvictableIdleTimeMillis(
BaseObjectPoolConfig.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME.toMillis());
dataSource.setLifo(BaseObjectPoolConfig.DEFAULT_LIFO);
- TransactionFactory transactionFactory = new JdbcTransactionFactory();
- Environment environment = new Environment("develop", transactionFactory,
dataSource);
- Configuration configuration = new Configuration(environment);
- configuration.addMapper(TableMetaMapper.class);
- configuration.addMapper(OptimizingMapper.class);
- configuration.addMapper(CatalogMetaMapper.class);
- configuration.addMapper(OptimizerMapper.class);
- configuration.addMapper(ApiTokensMapper.class);
- configuration.addMapper(PlatformFileMapper.class);
- configuration.addMapper(ResourceMapper.class);
- configuration.addMapper(TableBlockerMapper.class);
-
- PageInterceptor interceptor = new PageInterceptor();
- Properties interceptorProperties = new Properties();
- interceptorProperties.setProperty("reasonable", "false");
- interceptor.setProperties(interceptorProperties);
- configuration.addInterceptor(interceptor);
-
- DatabaseIdProvider provider = new VendorDatabaseIdProvider();
- Properties properties = new Properties();
- properties.setProperty("MySQL", "mysql");
- properties.setProperty("PostgreSQL", "postgres");
- properties.setProperty("Derby", "derby");
- provider.setProperties(properties);
- configuration.setDatabaseId(provider.getDatabaseId(dataSource));
- if (sqlSessionFactory == null) {
- synchronized (this) {
- if (sqlSessionFactory == null) {
- sqlSessionFactory = new
SqlSessionFactoryBuilder().build(configuration);
- }
- }
- }
- createTablesIfNeed(config);
- }
-
- private void registerDialectAliases() {
- registerDialectAlias("postgres", PostgreSqlDialect.class);
- registerDialectAlias("mysql", MySqlDialect.class);
- registerDialectAlias("derby", SqlServer2012Dialect.class);
+ createTablesIfNeed(dataSource, config);
+ return dataSource;
}
- /**
- * create tables for database
- *
- * @param config
- */
- private void createTablesIfNeed(Configurations config) {
+ /** create tables for database */
+ private static void createTablesIfNeed(DataSource ds, Configurations config)
{
boolean initSchema =
config.getBoolean(AmoroManagementConf.DB_AUTO_CREATE_TABLES);
if (!initSchema) {
LOG.info("Skip auto create tables due to configuration");
@@ -169,8 +90,7 @@ public class SqlSessionFactoryProvider {
String query = "";
LOG.info("Start create tables, database type:{}", dbTypeConfig);
- try (SqlSession sqlSession = get().openSession(true);
- Connection connection = sqlSession.getConnection();
+ try (Connection connection = ds.getConnection();
Statement statement = connection.createStatement()) {
if (AmoroManagementConf.DB_TYPE_DERBY.equals(dbTypeConfig)) {
query = "SELECT 1 FROM SYS.SYSTABLES WHERE TABLENAME =
'CATALOG_METADATA'";
@@ -203,7 +123,7 @@ public class SqlSessionFactoryProvider {
}
}
- private URI getInitSqlScriptPath(String type) throws URISyntaxException {
+ private static URI getInitSqlScriptPath(String type) throws
URISyntaxException {
String scriptPath = null;
if (type.equals(AmoroManagementConf.DB_TYPE_MYSQL)) {
scriptPath = MYSQL_INIT_SQL_SCRIPT;
@@ -218,16 +138,4 @@ public class SqlSessionFactoryProvider {
}
return scriptUrl.toURI();
}
-
- public SqlSessionFactory get() {
- Preconditions.checkState(
- sqlSessionFactory != null, "Persistent configuration is not
initialized yet.");
- return sqlSessionFactory;
- }
-
- public static String getDbType() {
- Preconditions.checkState(
- StringUtils.isNotBlank(dbType), "Persistent configuration is not
initialized yet.");
- return dbType;
- }
}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/persistence/HttpSessionHandlerFactory.java
b/amoro-ams/src/main/java/org/apache/amoro/server/persistence/HttpSessionHandlerFactory.java
new file mode 100644
index 000000000..2ee650c52
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/persistence/HttpSessionHandlerFactory.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.persistence;
+
+import org.apache.amoro.config.Configurations;
+import org.apache.amoro.server.AmoroManagementConf;
+import org.eclipse.jetty.server.session.DatabaseAdaptor;
+import org.eclipse.jetty.server.session.DefaultSessionCache;
+import org.eclipse.jetty.server.session.JDBCSessionDataStore;
+import org.eclipse.jetty.server.session.SessionHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.SQLException;
+import java.time.Duration;
+
+public class HttpSessionHandlerFactory {
+ private static final Logger LOG =
LoggerFactory.getLogger(HttpSessionHandlerFactory.class);
+
+ public static SessionHandler createSessionHandler(
+ DataSource dataSource, Configurations configurations) {
+ DatabaseAdaptor adaptor = new DatabaseAdaptor();
+ adaptor.setDatasource(dataSource);
+
+ JDBCSessionDataStore dataStore = new JDBCSessionDataStore();
+ dataStore.setDatabaseAdaptor(adaptor);
+ dataStore.setSessionTableSchema(new AmoroSessionTableSchema(dataSource));
+
+ SessionHandler handler = new SessionHandler();
+ DefaultSessionCache cache = new DefaultSessionCache(handler);
+ cache.setSessionDataStore(dataStore);
+ // set session timeout
+ Duration sessionTimeout =
configurations.get(AmoroManagementConf.HTTP_SERVER_SESSION_TIMEOUT);
+ handler.setMaxInactiveInterval((int) sessionTimeout.getSeconds());
+ handler.setSessionCache(cache);
+ return handler;
+ }
+
+ private static class AmoroSessionTableSchema extends
JDBCSessionDataStore.SessionTableSchema {
+ private final DataSource dataSource;
+
+ public AmoroSessionTableSchema(DataSource dataSource) {
+ setTableName("http_session");
+ setIdColumn("session_id");
+ setContextPathColumn("context_path");
+ setVirtualHostColumn("virtual_host");
+ setLastNodeColumn("last_node");
+ setAccessTimeColumn("access_time");
+ setLastAccessTimeColumn("last_access_time");
+ setCreateTimeColumn("create_time");
+ setCookieTimeColumn("cookie_time");
+ setLastSavedTimeColumn("last_save_time");
+ setExpiryTimeColumn("expiry_time");
+ setMaxIntervalColumn("max_interval");
+ setMapColumn("data_store");
+ this.dataSource = dataSource;
+ }
+
+ @Override
+ public void prepareTables() throws SQLException {
+ LOG.info("Skip jetty prepare http session tables.");
+ try (Connection connection = dataSource.getConnection()) {
+ // make the id table
+ connection.setAutoCommit(true);
+ DatabaseMetaData metaData = connection.getMetaData();
+ _dbAdaptor.adaptTo(metaData);
+ }
+ }
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/persistence/SqlSessionFactoryProvider.java
b/amoro-ams/src/main/java/org/apache/amoro/server/persistence/SqlSessionFactoryProvider.java
index 14f4b8fdf..608472d16 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/persistence/SqlSessionFactoryProvider.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/persistence/SqlSessionFactoryProvider.java
@@ -24,8 +24,6 @@ import com.github.pagehelper.PageInterceptor;
import com.github.pagehelper.dialect.helper.MySqlDialect;
import com.github.pagehelper.dialect.helper.PostgreSqlDialect;
import com.github.pagehelper.dialect.helper.SqlServer2012Dialect;
-import org.apache.amoro.config.Configurations;
-import org.apache.amoro.server.AmoroManagementConf;
import org.apache.amoro.server.persistence.mapper.ApiTokensMapper;
import org.apache.amoro.server.persistence.mapper.CatalogMetaMapper;
import org.apache.amoro.server.persistence.mapper.OptimizerMapper;
@@ -35,84 +33,32 @@ import
org.apache.amoro.server.persistence.mapper.ResourceMapper;
import org.apache.amoro.server.persistence.mapper.TableBlockerMapper;
import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
-import org.apache.commons.dbcp2.BasicDataSource;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.pool2.impl.BaseObjectPoolConfig;
-import org.apache.ibatis.jdbc.ScriptRunner;
import org.apache.ibatis.mapping.DatabaseIdProvider;
import org.apache.ibatis.mapping.Environment;
import org.apache.ibatis.mapping.VendorDatabaseIdProvider;
import org.apache.ibatis.session.Configuration;
-import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;
import org.apache.ibatis.transaction.TransactionFactory;
import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.ResultSet;
+import javax.sql.DataSource;
+
import java.sql.SQLException;
-import java.sql.Statement;
-import java.time.Duration;
import java.util.Properties;
public class SqlSessionFactoryProvider {
- private static final Logger LOG =
LoggerFactory.getLogger(SqlSessionFactoryProvider.class);
-
- private static final String DERBY_INIT_SQL_SCRIPT =
"derby/ams-derby-init.sql";
- private static final String MYSQL_INIT_SQL_SCRIPT =
"mysql/ams-mysql-init.sql";
- private static final String POSTGRES_INIT_SQL_SCRIPT =
"postgres/ams-postgres-init.sql";
-
private static final SqlSessionFactoryProvider INSTANCE = new
SqlSessionFactoryProvider();
public static SqlSessionFactoryProvider getInstance() {
return INSTANCE;
}
- private static String dbType;
-
private volatile SqlSessionFactory sqlSessionFactory;
- public void init(Configurations config) throws SQLException {
-
+ public void init(DataSource dataSource) throws SQLException {
registerDialectAliases();
- BasicDataSource dataSource = new BasicDataSource();
- dataSource.setUrl(config.getString(AmoroManagementConf.DB_CONNECTION_URL));
-
dataSource.setDriverClassName(config.getString(AmoroManagementConf.DB_DRIVER_CLASS_NAME));
- dbType = config.getString(AmoroManagementConf.DB_TYPE);
- if (AmoroManagementConf.DB_TYPE_MYSQL.equals(dbType)
- || AmoroManagementConf.DB_TYPE_POSTGRES.equals(dbType)) {
-
dataSource.setUsername(config.getString(AmoroManagementConf.DB_USER_NAME));
-
dataSource.setPassword(config.getString(AmoroManagementConf.DB_PASSWORD));
- }
- dataSource.setDefaultAutoCommit(false);
-
dataSource.setMaxTotal(config.getInteger(AmoroManagementConf.DB_CONNECT_MAX_TOTAL));
-
dataSource.setMaxIdle(config.getInteger(AmoroManagementConf.DB_CONNECT_MAX_IDLE));
- dataSource.setMinIdle(0);
-
dataSource.setMaxWaitMillis(config.getLong(AmoroManagementConf.DB_CONNECT_MAX_WAIT_MILLIS));
- dataSource.setLogAbandoned(true);
- dataSource.setRemoveAbandonedOnBorrow(true);
- dataSource.setRemoveAbandonedTimeout(60);
- dataSource.setTimeBetweenEvictionRunsMillis(Duration.ofMillis(10 * 60 *
1000L).toMillis());
- dataSource.setTestOnBorrow(BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW);
- dataSource.setTestWhileIdle(BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE);
- dataSource.setMinEvictableIdleTimeMillis(1000);
-
dataSource.setNumTestsPerEvictionRun(BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN);
- dataSource.setTestOnReturn(BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN);
- dataSource.setSoftMinEvictableIdleTimeMillis(
- BaseObjectPoolConfig.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME.toMillis());
- dataSource.setLifo(BaseObjectPoolConfig.DEFAULT_LIFO);
TransactionFactory transactionFactory = new JdbcTransactionFactory();
Environment environment = new Environment("develop", transactionFactory,
dataSource);
Configuration configuration = new Configuration(environment);
@@ -145,7 +91,6 @@ public class SqlSessionFactoryProvider {
}
}
}
- createTablesIfNeed(config);
}
private void registerDialectAliases() {
@@ -154,80 +99,9 @@ public class SqlSessionFactoryProvider {
registerDialectAlias("derby", SqlServer2012Dialect.class);
}
- /**
- * create tables for database
- *
- * @param config
- */
- private void createTablesIfNeed(Configurations config) {
- boolean initSchema =
config.getBoolean(AmoroManagementConf.DB_AUTO_CREATE_TABLES);
- if (!initSchema) {
- LOG.info("Skip auto create tables due to configuration");
- return;
- }
- String dbTypeConfig = config.getString(AmoroManagementConf.DB_TYPE);
- String query = "";
- LOG.info("Start create tables, database type:{}", dbTypeConfig);
-
- try (SqlSession sqlSession = get().openSession(true);
- Connection connection = sqlSession.getConnection();
- Statement statement = connection.createStatement()) {
- if (AmoroManagementConf.DB_TYPE_DERBY.equals(dbTypeConfig)) {
- query = "SELECT 1 FROM SYS.SYSTABLES WHERE TABLENAME =
'CATALOG_METADATA'";
- } else if (AmoroManagementConf.DB_TYPE_MYSQL.equals(dbTypeConfig)) {
- query =
- String.format(
- "SELECT 1 FROM information_schema.tables WHERE table_schema =
'%s' AND table_name = '%s'",
- connection.getCatalog(), "catalog_metadata");
- } else if (AmoroManagementConf.DB_TYPE_POSTGRES.equals(dbTypeConfig)) {
- query =
- String.format(
- "SELECT 1 FROM information_schema.tables WHERE table_schema =
%s AND table_name = '%s'",
- "current_schema()", "catalog_metadata");
- }
- LOG.info("Start check table creation, using query: {}", query);
- try (ResultSet rs = statement.executeQuery(query)) {
- if (!rs.next()) {
- Path script = Paths.get(getInitSqlScriptPath(dbTypeConfig));
- LOG.info("Table not exists, start run create tables script file:{}",
script);
- ScriptRunner runner = new ScriptRunner(connection);
- runner.runScript(
- new InputStreamReader(Files.newInputStream(script),
StandardCharsets.UTF_8));
- LOG.info("Tables are created successfully");
- } else {
- LOG.info("Tables are created, skip auto create tables.");
- }
- }
- } catch (Exception e) {
- throw new IllegalStateException("Create tables failed", e);
- }
- }
-
- private URI getInitSqlScriptPath(String type) throws URISyntaxException {
- String scriptPath = null;
- if (type.equals(AmoroManagementConf.DB_TYPE_MYSQL)) {
- scriptPath = MYSQL_INIT_SQL_SCRIPT;
- } else if (type.equals(AmoroManagementConf.DB_TYPE_DERBY)) {
- scriptPath = DERBY_INIT_SQL_SCRIPT;
- } else if (type.equals(AmoroManagementConf.DB_TYPE_POSTGRES)) {
- scriptPath = POSTGRES_INIT_SQL_SCRIPT;
- }
- URL scriptUrl = ClassLoader.getSystemResource(scriptPath);
- if (scriptUrl == null) {
- throw new IllegalStateException("Cannot find init sql script:" +
scriptPath);
- }
- return scriptUrl.toURI();
- }
-
public SqlSessionFactory get() {
Preconditions.checkState(
sqlSessionFactory != null, "Persistent configuration is not
initialized yet.");
return sqlSessionFactory;
}
-
- public static String getDbType() {
- Preconditions.checkState(
- StringUtils.isNotBlank(dbType), "Persistent configuration is not
initialized yet.");
- return dbType;
- }
}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/persistence/converter/Object2ByteArrayConvert.java
b/amoro-ams/src/main/java/org/apache/amoro/server/persistence/converter/Object2ByteArrayConvert.java
index d6deaba06..d12fa40bd 100644
---
a/amoro-ams/src/main/java/org/apache/amoro/server/persistence/converter/Object2ByteArrayConvert.java
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/persistence/converter/Object2ByteArrayConvert.java
@@ -19,7 +19,6 @@
package org.apache.amoro.server.persistence.converter;
import org.apache.amoro.server.AmoroManagementConf;
-import org.apache.amoro.server.persistence.SqlSessionFactoryProvider;
import org.apache.amoro.server.utils.CompressUtil;
import org.apache.amoro.utils.SerializationUtil;
import org.apache.ibatis.type.JdbcType;
@@ -31,14 +30,17 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
+import java.util.Locale;
public class Object2ByteArrayConvert<T> implements TypeHandler<T> {
@Override
public void setParameter(PreparedStatement ps, int i, T parameter, JdbcType
jdbcType)
throws SQLException {
+ String dbName =
+
ps.getConnection().getMetaData().getDatabaseProductName().toLowerCase(Locale.ENGLISH);
if (parameter == null) {
- if
(SqlSessionFactoryProvider.getDbType().equals(AmoroManagementConf.DB_TYPE_POSTGRES))
{
+ if (dbName.startsWith(AmoroManagementConf.DB_TYPE_POSTGRES)) {
ps.setNull(i, Types.BINARY);
} else {
ps.setNull(i, Types.BLOB);
diff --git a/amoro-ams/src/main/resources/derby/ams-derby-init.sql
b/amoro-ams/src/main/resources/derby/ams-derby-init.sql
index ca874b317..0e4e507e3 100644
--- a/amoro-ams/src/main/resources/derby/ams-derby-init.sql
+++ b/amoro-ams/src/main/resources/derby/ams-derby-init.sql
@@ -205,3 +205,19 @@ CREATE TABLE table_blocker (
PRIMARY KEY (blocker_id),
CONSTRAINT prev_uq UNIQUE (catalog_name, db_name, table_name,
prev_blocker_id)
);
+
+CREATE TABLE http_session (
+ session_id VARCHAR(120) NOT NULL,
+ context_path VARCHAR(60),
+ virtual_host VARCHAR(60),
+ last_node VARCHAR(60),
+ access_time BIGINT,
+ last_access_time BIGINT,
+ create_time BIGINT,
+ cookie_time BIGINT,
+ last_save_time BIGINT,
+ expiry_time BIGINT,
+ max_interval BIGINT,
+ data_store BLOB,
+ PRIMARY KEY(session_id, context_path, virtual_host)
+);
\ No newline at end of file
diff --git a/amoro-ams/src/main/resources/mysql/ams-mysql-init.sql
b/amoro-ams/src/main/resources/mysql/ams-mysql-init.sql
index 3db61746b..b72fa6101 100644
--- a/amoro-ams/src/main/resources/mysql/ams-mysql-init.sql
+++ b/amoro-ams/src/main/resources/mysql/ams-mysql-init.sql
@@ -225,3 +225,20 @@ CREATE TABLE `table_blocker` (
PRIMARY KEY (`blocker_id`),
UNIQUE KEY `uq_prev` (`catalog_name`,`db_name`,`table_name`,
`prev_blocker_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers'
ROW_FORMAT=DYNAMIC;
+
+CREATE TABLE `http_session` (
+ `session_id` varchar(120) NOT NULL COMMENT 'HTTP Session ID',
+ `context_path` varchar(60) COMMENT 'Jetty Context path',
+ `virtual_host` varchar(60) COMMENT 'Jetty virtual host',
+ `last_node` varchar(60) COMMENT 'Last node',
+ `access_time` bigint(20) COMMENT 'Access time',
+ `last_access_time` bigint(20) COMMENT 'Last access time',
+ `create_time` bigint(20) COMMENT 'Create time',
+ `cookie_time` bigint(20) COMMENT 'Cookie time',
+ `last_save_time` bigint(20) COMMENT 'Last save time',
+ `expiry_time` bigint(20) COMMENT 'Expiry time',
+ `max_interval` bigint(20) COMMENT 'Max internal',
+ `data_store` blob COMMENT 'Session data store',
+ PRIMARY KEY(`session_id`, `context_path`, `virtual_host`),
+ KEY `idx_session_expiry` (`expiry_time`)
+) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Http session store'
ROW_FORMAT=DYNAMIC;
diff --git a/amoro-ams/src/main/resources/mysql/upgrade.sql
b/amoro-ams/src/main/resources/mysql/upgrade.sql
index d075e0f7c..f632c01c3 100644
--- a/amoro-ams/src/main/resources/mysql/upgrade.sql
+++ b/amoro-ams/src/main/resources/mysql/upgrade.sql
@@ -48,4 +48,22 @@ SELECT `table_id`,`catalog_name`, `db_name`, `table_name`,
`current_snapshot_id
WHEN `optimizing_status` = 'FULL_OPTIMIZING' THEN 100
END,
`optimizing_status_start_time`, `optimizing_process_id`,
`optimizer_group`, `table_config`, `optimizing_config`, `pending_input`,
`table_summary`
-FROM table_runtime_backup;
\ No newline at end of file
+FROM table_runtime_backup;
+
+-- database http session handler
+CREATE TABLE `http_session` (
+ `session_id` varchar(120) NOT NULL COMMENT 'HTTP Session ID',
+ `context_path` varchar(60) COMMENT 'Jetty Context path',
+ `virtual_host` varchar(60) COMMENT 'Jetty virtual host',
+ `last_node` varchar(60) COMMENT 'Last node',
+ `access_time` bigint(20) COMMENT 'Access time',
+ `last_access_time` bigint(20) COMMENT 'Last access time',
+ `create_time` bigint(20) COMMENT 'Create time',
+ `cookie_time` bigint(20) COMMENT 'Cookie time',
+ `last_save_time` bigint(20) COMMENT 'Last save time',
+ `expiry_time` bigint(20) COMMENT 'Expiry time',
+ `max_interval` bigint(20) COMMENT 'Max internal',
+ `data_store` blob COMMENT 'Session data store',
+ PRIMARY KEY(`session_id`, `context_path`, `virtual_host`),
+ KEY `idx_session_expiry` (`expiry_time`)
+) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Http session store'
ROW_FORMAT=DYNAMIC;
diff --git a/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql
b/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql
index a7d0ccf6c..c7722e802 100644
--- a/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql
+++ b/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql
@@ -371,3 +371,35 @@ COMMENT ON COLUMN table_blocker.create_time IS 'Blocker
create time';
COMMENT ON COLUMN table_blocker.expiration_time IS 'Blocker expiration time';
COMMENT ON COLUMN table_blocker.properties IS 'Blocker properties';
COMMENT ON COLUMN table_blocker.prev_blocker_id is 'prev blocker id when
created';
+
+
+CREATE TABLE http_session (
+ session_id VARCHAR(120) NOT NULL,
+ context_path VARCHAR(60),
+ virtual_host VARCHAR(60),
+ last_node VARCHAR(60),
+ access_time BIGINT,
+ last_access_time BIGINT,
+ create_time BIGINT,
+ cookie_time BIGINT,
+ last_save_time BIGINT,
+ expiry_time BIGINT,
+ max_interval BIGINT,
+ data_store BYTEA,
+ PRIMARY KEY (session_id, context_path, virtual_host)
+);
+CREATE INDEX idx_session_expiry ON http_session (expiry_time);
+
+COMMENT ON COLUMN http_session.session_id IS 'Http session id';
+COMMENT ON COLUMN http_session.context_path IS 'Jetty context path';
+COMMENT ON COLUMN http_session.virtual_host IS 'Jetty virtual host';
+COMMENT ON COLUMN http_session.last_node IS 'Last node';
+COMMENT ON COLUMN http_session.access_time IS 'Access time';
+COMMENT ON COLUMN http_session.last_access_time IS 'Last access time';
+COMMENT ON COLUMN http_session.create_time IS 'Create time';
+COMMENT ON COLUMN http_session.cookie_time IS 'Cookie time';
+COMMENT ON COLUMN http_session.last_save_time IS 'Last save time';
+COMMENT ON COLUMN http_session.expiry_time IS 'Expiry time';
+COMMENT ON COLUMN http_session.max_interval IS 'Max internal';
+COMMENT ON COLUMN http_session.data_store IS 'Session data store';
+COMMENT ON TABLE http_session IS 'Http session store';
\ No newline at end of file
diff --git a/amoro-ams/src/main/resources/postgres/upgrade.sql
b/amoro-ams/src/main/resources/postgres/upgrade.sql
index c517de3be..225cdbd49 100644
--- a/amoro-ams/src/main/resources/postgres/upgrade.sql
+++ b/amoro-ams/src/main/resources/postgres/upgrade.sql
@@ -50,4 +50,38 @@ SELECT table_id,catalog_name, db_name, table_name,
current_snapshot_id,current_
WHEN optimizing_status = 'FULL_OPTIMIZING' THEN 100
END,
optimizing_status_start_time, optimizing_process_id, optimizer_group,
table_config, optimizing_config, pending_input
-FROM table_runtime_backup;
\ No newline at end of file
+FROM table_runtime_backup;
+
+-- ADD HTTP SESSION TABLE
+
+
+CREATE TABLE http_session (
+ session_id VARCHAR(120) NOT NULL,
+ context_path VARCHAR(60),
+ virtual_host VARCHAR(60),
+ last_node VARCHAR(60),
+ access_time BIGINT,
+ last_access_time BIGINT,
+ create_time BIGINT,
+ cookie_time BIGINT,
+ last_save_time BIGINT,
+ expiry_time BIGINT,
+ max_interval BIGINT,
+ data_store BYTEA,
+ PRIMARY KEY (session_id, context_path, virtual_host)
+);
+CREATE INDEX idx_session_expiry ON http_session (expiry_time);
+
+COMMENT ON COLUMN http_session.session_id IS 'Http session id';
+COMMENT ON COLUMN http_session.context_path IS 'Jetty context path';
+COMMENT ON COLUMN http_session.virtual_host IS 'Jetty virtual host';
+COMMENT ON COLUMN http_session.last_node IS 'Last node';
+COMMENT ON COLUMN http_session.access_time IS 'Access time';
+COMMENT ON COLUMN http_session.last_access_time IS 'Last access time';
+COMMENT ON COLUMN http_session.create_time IS 'Create time';
+COMMENT ON COLUMN http_session.cookie_time IS 'Cookie time';
+COMMENT ON COLUMN http_session.last_save_time IS 'Last save time';
+COMMENT ON COLUMN http_session.expiry_time IS 'Expiry time';
+COMMENT ON COLUMN http_session.max_interval IS 'Max internal';
+COMMENT ON COLUMN http_session.data_store IS 'Session data store';
+COMMENT ON TABLE http_session IS 'Http session store';
\ No newline at end of file
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestIcebergServerTableDescriptor.java
b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestIcebergServerTableDescriptor.java
index b897b3610..0905e0561 100644
---
a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestIcebergServerTableDescriptor.java
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestIcebergServerTableDescriptor.java
@@ -33,6 +33,7 @@ import org.apache.amoro.optimizing.MetricsSummary;
import org.apache.amoro.optimizing.OptimizingType;
import org.apache.amoro.process.ProcessStatus;
import org.apache.amoro.server.AmoroManagementConf;
+import org.apache.amoro.server.persistence.DataSourceFactory;
import org.apache.amoro.server.persistence.SqlSessionFactoryProvider;
import org.apache.amoro.server.persistence.mapper.OptimizingMapper;
import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
@@ -55,6 +56,8 @@ import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.sql.DataSource;
+
import java.io.IOException;
import java.sql.Connection;
import java.sql.ResultSet;
@@ -328,7 +331,8 @@ public class TestIcebergServerTableDescriptor extends
TestServerTableDescriptor
configurations.set(AmoroManagementConf.DB_TYPE,
AmoroManagementConf.DB_TYPE_DERBY);
configurations.set(
AmoroManagementConf.DB_DRIVER_CLASS_NAME,
"org.apache.derby.jdbc.EmbeddedDriver");
- SqlSessionFactoryProvider.getInstance().init(configurations);
+ DataSource ds = DataSourceFactory.createDataSource(configurations);
+ SqlSessionFactoryProvider.getInstance().init(ds);
LOG.info("Initialized derby persistent with url: {}", derbyUrl);
Runtime.getRuntime()
.addShutdownHook(
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/table/DerbyPersistence.java
b/amoro-ams/src/test/java/org/apache/amoro/server/table/DerbyPersistence.java
index 96ca0b9ae..817984d58 100644
---
a/amoro-ams/src/test/java/org/apache/amoro/server/table/DerbyPersistence.java
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/table/DerbyPersistence.java
@@ -20,6 +20,7 @@ package org.apache.amoro.server.table;
import org.apache.amoro.config.Configurations;
import org.apache.amoro.server.AmoroManagementConf;
+import org.apache.amoro.server.persistence.DataSourceFactory;
import org.apache.amoro.server.persistence.SqlSessionFactoryProvider;
import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
import org.apache.ibatis.session.SqlSession;
@@ -28,6 +29,8 @@ import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.sql.DataSource;
+
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -50,7 +53,8 @@ public class DerbyPersistence extends ExternalResource {
configurations.set(AmoroManagementConf.DB_TYPE,
AmoroManagementConf.DB_TYPE_DERBY);
configurations.set(
AmoroManagementConf.DB_DRIVER_CLASS_NAME,
"org.apache.derby.jdbc.EmbeddedDriver");
- SqlSessionFactoryProvider.getInstance().init(configurations);
+ DataSource ds = DataSourceFactory.createDataSource(configurations);
+ SqlSessionFactoryProvider.getInstance().init(ds);
LOG.info("Initialized derby persistent with url: {}", derbyUrl);
Runtime.getRuntime()
.addShutdownHook(
diff --git a/dist/src/main/amoro-bin/conf/config.yaml
b/dist/src/main/amoro-bin/conf/config.yaml
index d887239bb..721d990da 100644
--- a/dist/src/main/amoro-bin/conf/config.yaml
+++ b/dist/src/main/amoro-bin/conf/config.yaml
@@ -32,6 +32,7 @@ ams:
bind-port: 1261
http-server:
+ session-timeout: 7d
bind-port: 1630
rest-auth-type: token
diff --git a/docs/admin-guides/deployment.md b/docs/admin-guides/deployment.md
index 3d18cb7b5..32ebe9fbc 100644
--- a/docs/admin-guides/deployment.md
+++ b/docs/admin-guides/deployment.md
@@ -90,6 +90,7 @@ ams:
bind-port: 1261 #The port for accessing AMS optimizing service.
http-server:
+ session-timeout: 7d #Re-login after 7days
bind-port: 1630 #The port for accessing AMS Dashboard.
```
