This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit fb7f07c680b95e6999aff3ff02ccad7889ae7993
Author: He Xu <34228960+188x...@users.noreply.github.com>
AuthorDate: Sun Oct 16 20:10:59 2022 +0800
Revert Fix QueryHistory Clean
---
.../metadata/query/JdbcQueryHistoryStore.java | 73 +++++++---------------
.../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 44 +++++--------
.../metadata/query/RDBMSQueryHistoryDaoTest.java | 68 --------------------
3 files changed, 38 insertions(+), 147 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
index 094ad722b2..f5175e2619 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
@@ -154,7 +154,7 @@ public class JdbcQueryHistoryStore {
insertQhRealProviderList.forEach(qhRealizationMapper::insert);
session.commit();
- if (!queryMetricsList.isEmpty()) {
+ if (queryMetricsList.size() > 0) {
log.info("Insert {} query history into database takes {} ms",
queryMetricsList.size(),
System.currentTimeMillis() - startTime);
}
@@ -183,7 +183,7 @@ public class JdbcQueryHistoryStore {
return mapper.selectDaily(qhTableName, startTime, endTime);
}
}
-
+
public List<QueryHistory>
queryQueryHistoriesSubmitters(QueryHistoryRequest request, int size) {
try (SqlSession session = sqlSessionFactory.openSession()) {
QueryHistoryMapper mapper =
session.getMapper(QueryHistoryMapper.class);
@@ -198,23 +198,22 @@ public class JdbcQueryHistoryStore {
SelectStatementProvider statementProvider =
selectDistinct(queryHistoryRealizationTable.queryId)
.from(queryHistoryRealizationTable).where(queryHistoryRealizationTable.model,
isIn(modelIds))
.build().render(RenderingStrategies.MYBATIS3);
- return
mapper.selectMany(statementProvider).stream().map(QueryHistory::getQueryId)
- .collect(Collectors.toList());
+ return
mapper.selectMany(statementProvider).stream().map(QueryHistory::getQueryId).collect(Collectors.toList());
}
}
public List<QueryStatistics>
queryQueryHistoriesModelIds(QueryHistoryRequest request, int size) {
try (SqlSession session = sqlSessionFactory.openSession()) {
QueryStatisticsMapper mapper =
session.getMapper(QueryStatisticsMapper.class);
- SelectStatementProvider statementProvider1 =
selectDistinct(queryHistoryTable.engineType)
-
.from(queryHistoryTable).where(queryHistoryTable.engineType,
isNotEqualTo("NATIVE"))
- .and(queryHistoryTable.projectName,
isEqualTo(request.getProject())).build()
- .render(RenderingStrategies.MYBATIS3);
+ SelectStatementProvider statementProvider1 =
selectDistinct(queryHistoryTable.engineType).from(queryHistoryTable)
+ .where(queryHistoryTable.engineType,
isNotEqualTo("NATIVE"))
+ .and(queryHistoryTable.projectName,
isEqualTo(request.getProject()))
+ .build().render(RenderingStrategies.MYBATIS3);
List<QueryStatistics> engineTypes =
mapper.selectMany(statementProvider1);
- SelectStatementProvider statementProvider2 =
selectDistinct(queryHistoryRealizationTable.model)
- .from(queryHistoryRealizationTable)
- .where(queryHistoryRealizationTable.projectName,
isEqualTo(request.getProject())).limit(size)
+ SelectStatementProvider statementProvider2 =
selectDistinct(queryHistoryRealizationTable.model).from(queryHistoryRealizationTable)
+ .where(queryHistoryRealizationTable.projectName,
isEqualTo(request.getProject()))
+ .limit(size)
.build().render(RenderingStrategies.MYBATIS3);
List<QueryStatistics> modelIds =
mapper.selectMany(statementProvider2);
engineTypes.addAll(modelIds);
@@ -222,36 +221,16 @@ public class JdbcQueryHistoryStore {
}
}
- public long getMaxId() {
- try (SqlSession session = sqlSessionFactory.openSession()) {
- QueryHistoryMapper mapper =
session.getMapper(QueryHistoryMapper.class);
- SelectStatementProvider statementProvider =
select(max(queryHistoryTable.id))
- .from(queryHistoryTable)
- .build().render(RenderingStrategies.MYBATIS3);
- Long maxId = mapper.selectAsLong(statementProvider);
- return maxId == null ? 0 : maxId;
- }
- }
-
- public QueryHistory queryOldestQueryHistory(long retainMinId) {
+ public QueryHistory queryOldestQueryHistory(long maxSize) {
try (SqlSession session = sqlSessionFactory.openSession()) {
QueryHistoryMapper mapper =
session.getMapper(QueryHistoryMapper.class);
SelectStatementProvider statementProvider =
select(getSelectFields(queryHistoryTable))
- .from(queryHistoryTable)
- .where(queryHistoryTable.id, isEqualTo(retainMinId))
- .build().render(RenderingStrategies.MYBATIS3);
- return mapper.selectOne(statementProvider);
- }
- }
-
- public long getProjectCount(String project) {
- try (SqlSession session = sqlSessionFactory.openSession()) {
- QueryHistoryMapper mapper =
session.getMapper(QueryHistoryMapper.class);
- SelectStatementProvider statementProvider =
select(count(queryHistoryTable.id)) //
.from(queryHistoryTable) //
- .where(queryHistoryTable.projectName, isEqualTo(project))
//
+ .orderBy(queryHistoryTable.id.descending()) //
+ .limit(1) //
+ .offset(maxSize - 1) //
.build().render(RenderingStrategies.MYBATIS3);
- return mapper.selectAsLong(statementProvider);
+ return mapper.selectOne(statementProvider);
}
}
@@ -260,12 +239,11 @@ public class JdbcQueryHistoryStore {
QueryHistoryMapper mapper =
session.getMapper(QueryHistoryMapper.class);
SelectStatementProvider statementProvider =
select(getSelectFields(queryHistoryTable)) //
.from(queryHistoryTable) //
- .where(queryHistoryTable.id, //
- isEqualTo(select(queryHistoryTable.id) //
- .from(queryHistoryTable) //
- .where(queryHistoryTable.projectName,
isEqualTo(project)) //
-
.orderBy(queryHistoryTable.id.descending()).limit(1).offset(maxSize - 1)) //
- ).build().render(RenderingStrategies.MYBATIS3);
+ .where(queryHistoryTable.projectName, isEqualTo(project))
//
+ .orderBy(queryHistoryTable.id.descending()) //
+ .limit(1) //
+ .offset(maxSize - 1) //
+ .build().render(RenderingStrategies.MYBATIS3);
return mapper.selectOne(statementProvider);
}
}
@@ -520,7 +498,7 @@ public class JdbcQueryHistoryStore {
idToQHInfoList.forEach(pair ->
providers.add(changeQHInfoProvider(pair.getFirst(), pair.getSecond())));
providers.forEach(mapper::update);
session.commit();
- if (!idToQHInfoList.isEmpty()) {
+ if (idToQHInfoList.size() > 0) {
log.info("Update {} query history info takes {} ms",
idToQHInfoList.size(),
System.currentTimeMillis() - start);
}
@@ -642,8 +620,7 @@ public class JdbcQueryHistoryStore {
if (request.isSubmitterExactlyMatch()) {
filterSql = filterSql.and(queryHistoryTable.querySubmitter,
isIn(request.getFilterSubmitter()));
} else if (request.getFilterSubmitter().size() == 1) {
- filterSql = filterSql.and(queryHistoryTable.querySubmitter,
- isLikeCaseInsensitive("%" +
request.getFilterSubmitter().get(0) + "%"));
+ filterSql = filterSql.and(queryHistoryTable.querySubmitter,
isLikeCaseInsensitive("%" + request.getFilterSubmitter().get(0) + "%"));
}
}
@@ -667,12 +644,10 @@ public class JdbcQueryHistoryStore {
}
} else if (selectAllModels) {
// Process CONSTANTS, HIVE, RDBMS and all model
- filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
- or(queryHistoryTable.indexHit, isEqualTo(true)));
+ filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations), or(queryHistoryTable.indexHit, isEqualTo(true)));
} else if (request.getFilterModelIds() != null &&
!request.getFilterModelIds().isEmpty()) {
// Process CONSTANTS, HIVE, RDBMS and model1, model2, model3...
- filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
- or(queryHistoryTable.queryId,
+ filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations), or(queryHistoryTable.queryId,
isIn(selectDistinct(queryHistoryRealizationTable.queryId).from(queryHistoryRealizationTable)
.where(queryHistoryRealizationTable.model,
isIn(request.getFilterModelIds())))));
} else {
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
index 31990f27d4..73124496de 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
@@ -45,8 +45,8 @@ public class RDBMSQueryHistoryDAO implements QueryHistoryDAO {
private static final Logger logger =
LoggerFactory.getLogger(RDBMSQueryHistoryDAO.class);
@Setter
private String queryMetricMeasurement;
- private final String realizationMetricMeasurement;
- private final JdbcQueryHistoryStore jdbcQueryHisStore;
+ private String realizationMetricMeasurement;
+ private JdbcQueryHistoryStore jdbcQueryHisStore;
public static final String WEEK = "week";
public static final String DAY = "day";
@@ -105,21 +105,13 @@ public class RDBMSQueryHistoryDAO implements
QueryHistoryDAO {
}
public void deleteQueryHistoriesIfMaxSizeReached() {
- int globalMaxSize =
KylinConfig.getInstanceFromEnv().getQueryHistoryMaxSize();
- long globalMaxId = jdbcQueryHisStore.getMaxId();
- long retainMinId = globalMaxId - globalMaxSize + 1;
- logger.info("Clean QueryHistory Global MaxId: {}, MaxSize: {},
RetainMinId: {}", globalMaxId, globalMaxSize, retainMinId);
- if (retainMinId <= 1) {
- // no need to delete
- return;
- }
- QueryHistory queryHistory =
jdbcQueryHisStore.queryOldestQueryHistory(retainMinId);
- if (Objects.isNull(queryHistory)) {
- return;
+ QueryHistory queryHistory = jdbcQueryHisStore
+
.queryOldestQueryHistory(KylinConfig.getInstanceFromEnv().getQueryHistoryMaxSize());
+ if (Objects.nonNull(queryHistory)) {
+ long time = queryHistory.getQueryTime();
+ jdbcQueryHisStore.deleteQueryHistory(time);
+ jdbcQueryHisStore.deleteQueryHistoryRealization(time);
}
- long time = queryHistory.getQueryTime();
- jdbcQueryHisStore.deleteQueryHistory(time);
- jdbcQueryHisStore.deleteQueryHistoryRealization(time);
}
public QueryHistory getByQueryId(String queryId) {
@@ -127,21 +119,13 @@ public class RDBMSQueryHistoryDAO implements
QueryHistoryDAO {
}
public void deleteQueryHistoriesIfProjectMaxSizeReached(String project) {
- int projectMaxSize =
KylinConfig.getInstanceFromEnv().getQueryHistoryProjectMaxSize();
- long projectCount = jdbcQueryHisStore.getProjectCount(project);
- logger.info("Clean QueryHistory Project: {}, Count: {}, MaxSize: {}",
project, projectCount, projectMaxSize);
- if (projectCount <= projectMaxSize) {
- // no need to delete
- return;
- }
-
- QueryHistory queryHistory =
jdbcQueryHisStore.queryOldestQueryHistory(projectMaxSize, project);
- if (Objects.isNull(queryHistory)) {
- return;
+ QueryHistory queryHistory = jdbcQueryHisStore
+
.queryOldestQueryHistory(KylinConfig.getInstanceFromEnv().getQueryHistoryProjectMaxSize(),
project);
+ if (Objects.nonNull(queryHistory)) {
+ long time = queryHistory.getQueryTime();
+ jdbcQueryHisStore.deleteQueryHistory(time, project);
+ jdbcQueryHisStore.deleteQueryHistoryRealization(time, project);
}
- long time = queryHistory.getQueryTime();
- jdbcQueryHisStore.deleteQueryHistory(time, project);
- jdbcQueryHisStore.deleteQueryHistoryRealization(time, project);
}
public void deleteQueryHistoriesIfRetainTimeReached() {
diff --git
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
index 70baeed914..18443017dc 100644
---
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
+++
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
@@ -23,7 +23,6 @@ import static
org.apache.kylin.metadata.query.RDBMSQueryHistoryDAO.fillZeroForQu
import java.util.List;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.TimeUtil;
import org.apache.kylin.junit.TimeZoneTestRunner;
@@ -766,71 +765,4 @@ public class RDBMSQueryHistoryDaoTest extends
NLocalFileMetadataTestCase {
queryMetrics.setQueryHistoryInfo(queryHistoryInfo);
return queryMetrics;
}
-
- @Test
- public void testDeleteQueryHistoryMaxSizeForGlobal() {
- KylinConfig config = getTestConfig();
- config.setProperty("kylin.query.queryhistory.max-size", "2");
- Assert.assertEquals(2, config.getQueryHistoryMaxSize());
-
- // before delete
- Assert.assertEquals(0, queryHistoryDAO.getAllQueryHistories().size());
-
- // delete empty
- queryHistoryDAO.deleteQueryHistoriesIfMaxSizeReached();
- Assert.assertEquals(0, queryHistoryDAO.getAllQueryHistories().size());
-
- // insert
- queryHistoryDAO.insert(createQueryMetrics(1580311512000L, 1L, true,
PROJECT, true));
- queryHistoryDAO.insert(createQueryMetrics(1580397912000L, 2L, false,
PROJECT, true));
- Assert.assertEquals(2, queryHistoryDAO.getAllQueryHistories().size());
-
- // delete equals max size
- queryHistoryDAO.deleteQueryHistoriesIfMaxSizeReached();
- Assert.assertEquals(2, queryHistoryDAO.getAllQueryHistories().size());
-
- // delete > max size
- queryHistoryDAO.insert(createQueryMetrics(1580397912000L, 2L, false,
PROJECT, true));
- Assert.assertEquals(3, queryHistoryDAO.getAllQueryHistories().size());
- queryHistoryDAO.deleteQueryHistoriesIfMaxSizeReached();
- Assert.assertEquals(2, queryHistoryDAO.getAllQueryHistories().size());
-
- }
-
- @Test
- public void testDeleteQueryHistoryMaxSizeForProject() {
- String otherProject = "other_project";
- KylinConfig config = getTestConfig();
- config.setProperty("kylin.query.queryhistory.project-max-size", "2");
- Assert.assertEquals(2, config.getQueryHistoryProjectMaxSize());
-
- // before delete
- Assert.assertEquals(0, queryHistoryDAO.getAllQueryHistories().size());
-
- // delete empty
- queryHistoryDAO.deleteQueryHistoriesIfProjectMaxSizeReached(PROJECT);
- Assert.assertEquals(0, queryHistoryDAO.getAllQueryHistories().size());
-
- // insert
- queryHistoryDAO.insert(createQueryMetrics(1580311512000L, 1L, true,
PROJECT, true));
- queryHistoryDAO.insert(createQueryMetrics(1580397912000L, 2L, false,
PROJECT, true));
- queryHistoryDAO.insert(createQueryMetrics(1580311512000L, 1L, true,
otherProject, true));
- queryHistoryDAO.insert(createQueryMetrics(1580397912000L, 2L, false,
otherProject, true));
- Assert.assertEquals(4, queryHistoryDAO.getAllQueryHistories().size());
-
- // delete equals max size
- queryHistoryDAO.deleteQueryHistoriesIfProjectMaxSizeReached(PROJECT);
-
queryHistoryDAO.deleteQueryHistoriesIfProjectMaxSizeReached(otherProject);
- Assert.assertEquals(4, queryHistoryDAO.getAllQueryHistories().size());
-
- // delete > max size
- queryHistoryDAO.insert(createQueryMetrics(1580397912000L, 2L, false,
PROJECT, true));
- queryHistoryDAO.insert(createQueryMetrics(1580397912000L, 2L, false,
otherProject, true));
- Assert.assertEquals(6, queryHistoryDAO.getAllQueryHistories().size());
- queryHistoryDAO.deleteQueryHistoriesIfProjectMaxSizeReached(PROJECT);
-
queryHistoryDAO.deleteQueryHistoriesIfProjectMaxSizeReached(otherProject);
- Assert.assertEquals(4, queryHistoryDAO.getAllQueryHistories().size());
-
- }
-
}
