This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8cfe32cf4f61d218439e57e134ccc3413aa98f89
Author: kyotoYaho <nju_y...@apache.org>
AuthorDate: Wed Dec 19 10:29:38 2018 +0800
KYLIN-3540 move queryService of CubeController to CubeService
---
.../kylin/rest/controller/CubeController.java | 71 ++---------------
.../org/apache/kylin/rest/service/CubeService.java | 88 ++++++++++++++++++----
2 files changed, 78 insertions(+), 81 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index dcaa259..11753ec 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -56,8 +56,6 @@ import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metrics.MetricsManager;
-import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -70,7 +68,6 @@ import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
import org.apache.kylin.rest.request.LookupSnapshotBuildRequest;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
@@ -80,7 +77,6 @@ import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.rest.service.ProjectService;
-import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.util.ValidateUtil;
import org.apache.kylin.source.kafka.util.KafkaClient;
import org.slf4j.Logger;
@@ -124,10 +120,6 @@ public class CubeController extends BasicController {
@Qualifier("projectService")
private ProjectService projectService;
- @Autowired
- @Qualifier("queryService")
- private QueryService queryService;
-
@RequestMapping(value = "/validate/{cubeName}", method =
RequestMethod.GET, produces = { "application/json" })
@ResponseBody
public EnvelopeResponse<Boolean> validateModelName(@PathVariable String
cubeName) {
@@ -873,7 +865,7 @@ public class CubeController extends BasicController {
Map<Long, Long> queryMatchMap = null;
try {
hitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
- queryMatchMap = getCuboidQueryMatchCount(cubeName);
+ queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName);
} catch (Exception e) {
logger.warn("Fail to query on system cube due to " + e);
}
@@ -899,7 +891,7 @@ public class CubeController extends BasicController {
// Get cuboid target info for displaying heat map of cuboid hit
Map<Long, Long> displayHitFrequencyMap =
getTargetCuboidHitFrequency(cubeName);
// Get exactly matched cuboid query count
- Map<Long, Long> queryMatchMap = getCuboidQueryMatchCount(cubeName);
+ Map<Long, Long> queryMatchMap =
cubeService.getCuboidQueryMatchCount(cubeName);
Set<Long> currentCuboidSet =
cube.getCuboidScheduler().getAllCuboidIds();
return cubeService.getCuboidTreeResponse(cuboidScheduler,
recommendCuboidStatsMap, displayHitFrequencyMap,
@@ -909,69 +901,16 @@ public class CubeController extends BasicController {
private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws
IOException {
// Get cuboid source info
Map<Long, Long> optimizeHitFrequencyMap =
getSourceCuboidHitFrequency(cube.getName());
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap =
getCuboidRollingUpCount(cube.getName());
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap =
cubeService.getCuboidRollingUpStats(cube.getName());
return cubeService.getRecommendCuboidStatistics(cube,
optimizeHitFrequencyMap, rollingUpCountSourceMap);
}
private Map<Long, Long> getSourceCuboidHitFrequency(String cubeName) {
- return getCuboidHitFrequency(cubeName, true);
+ return cubeService.getCuboidHitFrequency(cubeName, true);
}
private Map<Long, Long> getTargetCuboidHitFrequency(String cubeName) {
- return getCuboidHitFrequency(cubeName, false);
- }
-
- private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean
isCuboidSource) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- if (!isCuboidSource) {
- cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
- }
- String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
- String table = cubeService.getMetricsManager()
-
.getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ") " //
- + "from " + table//
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "' " //
- + "group by " + cuboidColumn;
- sqlRequest.setSql(sql);
- List<List<String>> orgHitFrequency =
queryService.doQueryWithCache(sqlRequest).getResults();
- return cubeService.formatQueryCount(orgHitFrequency);
- }
-
- private Map<Long, Map<Long, Long>> getCuboidRollingUpCount(String
cubeName) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
- String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
- String table = cubeService.getMetricsManager()
-
.getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum("
+ aggCount + ")/count(*) " //
- + "from " + table //
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "' " //
- + "group by " + cuboidSource + ", " + cuboidTarget;
- sqlRequest.setSql(sql);
- List<List<String>> orgRollingUpCount =
queryService.doQueryWithCache(sqlRequest).getResults();
- return cubeService.formatRollingUpCount(orgRollingUpCount);
- }
-
- private Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
- String table = cubeService.getMetricsManager()
-
.getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ") " //
- + "from " + table //
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "' and "
- + QueryCubePropertyEnum.IF_MATCH.toString() + " = true " //
- + "group by " + cuboidSource;
- sqlRequest.setSql(sql);
- List<List<String>> orgMatchHitFrequency =
queryService.doQueryWithCache(sqlRequest).getResults();
- return cubeService.formatQueryCount(orgMatchHitFrequency);
+ return cubeService.getCuboidHitFrequency(cubeName, false);
}
/**
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index e541c37..e24e9de 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -65,6 +65,8 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.metrics.MetricsManager;
+import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
@@ -72,6 +74,7 @@ import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
+import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
@@ -121,6 +124,10 @@ public class CubeService extends BasicService implements
InitializingBean {
private ModelService modelService;
@Autowired
+ @Qualifier("queryService")
+ private QueryService queryService;
+
+ @Autowired
private AclEvaluate aclEvaluate;
@Autowired
@@ -472,7 +479,8 @@ public class CubeService extends BasicService implements
InitializingBean {
hr = new HBaseResponse();
CubeInstance cube =
CubeManager.getInstance(getConfig()).getCube(cubeName);
- if (cube.getStorageType() == IStorageAware.ID_HBASE ||
cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE) {
+ if (cube.getStorageType() == IStorageAware.ID_HBASE
+ || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE) {
try {
logger.debug("Loading HTable info " + cubeName + ", " +
tableName);
@@ -547,9 +555,9 @@ public class CubeService extends BasicService implements
InitializingBean {
}
public boolean isOrphonSegment(CubeInstance cube, String segId) {
- List<JobInstance> jobInstances =
jobService.searchJobsByCubeName(cube.getName(),
- cube.getProject(), Lists.newArrayList(JobStatusEnum.NEW,
JobStatusEnum.PENDING, JobStatusEnum.RUNNING,
- JobStatusEnum.ERROR, JobStatusEnum.STOPPED),
+ List<JobInstance> jobInstances = jobService.searchJobsByCubeName(
+ cube.getName(), cube.getProject(),
Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING,
+ JobStatusEnum.RUNNING, JobStatusEnum.ERROR,
JobStatusEnum.STOPPED),
JobTimeFilterEnum.ALL, JobService.JobSearchMode.CUBING_ONLY);
for (JobInstance jobInstance : jobInstances) {
// if there are segment related jobs, can not delete this segment.
@@ -886,6 +894,12 @@ public class CubeService extends BasicService implements
InitializingBean {
}
/** cube planner services */
+ public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube,
Map<Long, Long> hitFrequencyMap,
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws
IOException {
+ aclEvaluate.checkProjectAdminPermission(cube.getProject());
+ return CuboidRecommenderUtil.getRecommendCuboidList(cube,
hitFrequencyMap, rollingUpCountSourceMap);
+ }
+
public Map<Long, Long> formatQueryCount(List<List<String>> orgQueryCount) {
Map<Long, Long> formattedQueryCount = Maps.newLinkedHashMap();
for (List<String> hit : orgQueryCount) {
@@ -894,20 +908,64 @@ public class CubeService extends BasicService implements
InitializingBean {
return formattedQueryCount;
}
- public Map<Long, Map<Long, Long>> formatRollingUpCount(List<List<String>>
orgRollingUpCount) {
- Map<Long, Map<Long, Long>> formattedRollingUpCount =
Maps.newLinkedHashMap();
+ public Map<Long, Map<Long, Long>> formatRollingUpStats(List<List<String>>
orgRollingUpCount) {
+ Map<Long, Map<Long, Long>> formattedRollingUpStats =
Maps.newLinkedHashMap();
for (List<String> rollingUp : orgRollingUpCount) {
Map<Long, Long> childMap = Maps.newLinkedHashMap();
childMap.put(Long.parseLong(rollingUp.get(1)), (long)
Double.parseDouble(rollingUp.get(2)));
- formattedRollingUpCount.put(Long.parseLong(rollingUp.get(0)),
childMap);
- }
- return formattedRollingUpCount;
- }
-
- public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube,
Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws
IOException {
- aclEvaluate.checkProjectAdminPermission(cube.getProject());
- return CuboidRecommenderUtil.getRecommendCuboidList(cube,
hitFrequencyMap, rollingUpCountSourceMap);
+ formattedRollingUpStats.put(Long.parseLong(rollingUp.get(0)),
childMap);
+ }
+ return formattedRollingUpStats;
+ }
+
+ public Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean
isCuboidSource) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ if (!isCuboidSource) {
+ cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ }
+ String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+ String table =
getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ")" //
+ + " from " + table//
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "' " //
+ + " group by " + cuboidColumn;
+ sqlRequest.setSql(sql);
+ List<List<String>> orgHitFrequency =
queryService.doQueryWithCache(sqlRequest).getResults();
+ return formatQueryCount(orgHitFrequency);
+ }
+
+ public Map<Long, Map<Long, Long>> getCuboidRollingUpStats(String cubeName)
{
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
+ String table =
getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum("
+ aggCount + ")/count(*)" //
+ + " from " + table //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "' " //
+ + " group by " + cuboidSource + ", " + cuboidTarget;
+ sqlRequest.setSql(sql);
+ List<List<String>> orgRollingUpCount =
queryService.doQueryWithCache(sqlRequest).getResults();
+ return formatRollingUpStats(orgRollingUpCount);
+ }
+
+ public Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+ String table =
getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ")" //
+ + " from " + table //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" +
cubeName + "'" //
+ + " and " + QueryCubePropertyEnum.IF_MATCH.toString() + " =
true" //
+ + " group by " + cuboidSource;
+ sqlRequest.setSql(sql);
+ List<List<String>> orgMatchHitFrequency =
queryService.doQueryWithCache(sqlRequest).getResults();
+ return formatQueryCount(orgMatchHitFrequency);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
