This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 99d3c9be4e3de0dcf7f40e9747d7a84667af2f9a
Author: Zhong, Yanghong <nju_y...@apache.org>
AuthorDate: Thu May 28 13:51:22 2020 +0800
KYLIN-4532 Introduce cube planner optimization benefit score
---
.../org/apache/kylin/common/KylinConfigBase.java | 6 +-
.../kylin/common/persistence/ResourceStore.java | 1 +
.../java/org/apache/kylin/cube/CubeInstance.java | 22 ++
.../java/org/apache/kylin/cube/CubeManager.java | 50 ++++
.../java/org/apache/kylin/cube/CubeUpdate.java | 18 ++
.../cube/cuboid/algorithm/CuboidRecommender.java | 2 +-
.../cube/cuboid/algorithm/CuboidStatsUtil.java | 82 +++++-
.../cube/cuboid/algorithm/OptimizationBenefit.java | 289 +++++++++++++++++++++
.../cube/cuboid/algorithm/RecommendResult.java | 58 +++++
.../engine/mr/common/CuboidRecommenderUtil.java | 34 ++-
.../kylin/rest/controller/CubeController.java | 75 ++++--
.../java/org/apache/kylin/rest/msg/Message.java | 5 +
.../rest/response/CuboidRecommendResponse.java | 42 +++
.../org/apache/kylin/rest/service/CubeService.java | 3 +-
webapp/app/js/services/cubes.js | 2 +-
15 files changed, 658 insertions(+), 31 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 93cdcf9..416800b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -840,7 +840,7 @@ public abstract class KylinConfigBase implements
Serializable {
return
Double.parseDouble(getOptional("kylin.cube.cubeplanner.bpus-min-benefit-ratio",
"0.01"));
}
- public int getCubePlannerAgreedyAlgorithmAutoThreshold() {
+ public int getCubePlannerGreedyAlgorithmAutoThreshold() {
return
Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-greedy",
"8"));
}
@@ -848,6 +848,10 @@ public abstract class KylinConfigBase implements
Serializable {
return
Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-genetic",
"23"));
}
+ public double getCubePlannerOptimizationSpaceBenefitRatio() {
+ return
Double.parseDouble(getOptional("kylin.cube.cubeplanner.optimization-space-benefit-ratio",
"0.1"));
+ }
+
/**
* get assigned server array, which a empty string array in default
* @return
diff --git
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 453fd01..b933ae3 100644
---
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -80,6 +80,7 @@ abstract public class ResourceStore {
public static final String DRAFT_RESOURCE_ROOT = "/draft";
public static final String USER_ROOT = "/user";
public static final String EXT_SNAPSHOT_RESOURCE_ROOT =
"/ext_table_snapshot";
+ public static final String CUBE_SCORE_ROOT = "/cube_score";
public static final String METASTORE_UUID_TAG = "/UUID";
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 454816d..0d3f39d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -130,6 +130,12 @@ public class CubeInstance extends RootPersistentEntity
implements IRealization,
@JsonProperty("snapshots")
private Map<String, String> snapshots = Maps.newHashMap();
+ @JsonProperty("score")
+ private double score = -1;
+
+ @JsonProperty("score_hint")
+ private String scoreHint;
+
// cuboid scheduler lazy built
transient private CuboidScheduler cuboidScheduler;
@@ -735,6 +741,22 @@ public class CubeInstance extends RootPersistentEntity
implements IRealization,
getSnapshots().put(table, snapshotResPath);
}
+ public void setScore(double score) {
+ this.score = score;
+ }
+
+ public double getScore() {
+ return score;
+ }
+
+ public String getScoreHint() {
+ return scoreHint;
+ }
+
+ public void setScoreHint(String scoreHint) {
+ this.scoreHint = scoreHint;
+ }
+
public static CubeInstance getCopyOf(CubeInstance other) {
CubeInstance ret = new CubeInstance();
ret.setName(other.getName());
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 6e9452d..a61a409 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -45,6 +45,7 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.cuboid.algorithm.OptimizationBenefit;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDescTiretreeGlobalDomainDictUtil;
import org.apache.kylin.cube.model.SnapshotTableDesc;
@@ -305,6 +306,24 @@ public class CubeManager implements IRealizationProvider {
}
}
+ private void updateCubeScore(CubeInstance cube, double score, String
scoreHint) throws IOException {
+ if (cube.getScore() == score) {
+ if (cube.getScoreHint() == null) {
+ if (scoreHint == null)
+ return;
+ } else if (cube.getScoreHint().equals(scoreHint)) {
+ return;
+ }
+ }
+ try (AutoLock lock = cubeMapLock.lockForWrite()) {
+ cube = cube.latestCopyForWrite(); // get a latest copy
+ CubeUpdate update = new CubeUpdate(cube);
+ update.setScore(score);
+ update.setScoreHint(scoreHint);
+ updateCube(update);
+ }
+ }
+
public CubeInstance updateCubeStatus(CubeInstance cube,
RealizationStatusEnum newStatus) throws IOException {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
cube = cube.latestCopyForWrite(); // get a latest copy
@@ -454,6 +473,14 @@ public class CubeManager implements IRealizationProvider {
if (update.getCuboidLastOptimized() >= 0) {
cube.setCuboidLastOptimized(update.getCuboidLastOptimized());
}
+
+ if (update.getScore() >= -1) {
+ cube.setScore(update.getScore());
+ }
+
+ if (update.getScoreHint() != null) {
+ cube.setScoreHint(update.getScoreHint());
+ }
}
private void processToUpdateSegments(CubeUpdate update,
Segments<CubeSegment> newSegs) {
@@ -586,6 +613,29 @@ public class CubeManager implements IRealizationProvider {
return snapshotResPath;
}
+ public OptimizationBenefit loadCubeOptimizationBenefit(final CubeInstance
cubeInstance) throws IOException {
+ logger.info("Going to load optimization benefit for cube {}",
cubeInstance.getName());
+ return OptimizationBenefit.loadFromStore(getStore(), cubeInstance);
+ }
+
+ public void saveCubeOptimizationBenefit(CubeInstance cubeInstance,
OptimizationBenefit benefit) throws IOException {
+ saveCubeOptimizationBenefit(cubeInstance, benefit, true);
+ }
+
+ public void saveCubeOptimizationBenefit(CubeInstance cubeInstance,
OptimizationBenefit benefit, boolean ifUpdate)
+ throws IOException {
+ logger.info("Going to update optimization benefit for cube {}",
cubeInstance.getName());
+ OptimizationBenefit.saveToStore(getStore(), cubeInstance, benefit);
+ if (ifUpdate) {
+ updateCubeScore(cubeInstance, benefit.getScore(),
benefit.getScoreHint());
+ }
+ }
+
+ private void removeCubeScoreStats(CubeInstance cubeInstance) throws
IOException {
+ logger.info("Going to remove optimization benefit for cube {}",
cubeInstance.getName());
+ OptimizationBenefit.removeFromStore(getStore(), cubeInstance);
+ }
+
@VisibleForTesting
/*private*/ String generateStorageLocation() {
String namePrefix = config.getHBaseTableNamePrefix();
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
index 33a3251..28c9bca 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
@@ -39,6 +39,8 @@ public class CubeUpdate {
private Map<String, String> updateTableSnapshotPath = null;
private long createTimeUTC = -1;
private long cuboidLastOptimized = -1;
+ private double score = -2;
+ private String scoreHint;
public CubeUpdate(CubeInstance cubeInstance) {
setCubeInstance(cubeInstance);
@@ -151,4 +153,20 @@ public class CubeUpdate {
public void setCuboidLastOptimized(long cuboidLastOptimized) {
this.cuboidLastOptimized = cuboidLastOptimized;
}
+
+ public double getScore() {
+ return score;
+ }
+
+ public void setScore(double score) {
+ this.score = score;
+ }
+
+ public String getScoreHint() {
+ return scoreHint;
+ }
+
+ public void setScoreHint(String scoreHint) {
+ this.scoreHint = scoreHint;
+ }
}
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
index dfea903..4fbdc21 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
@@ -118,7 +118,7 @@ public class CuboidRecommender {
*/
public Map<Long, Long> getRecommendCuboidList(CuboidStats cuboidStats,
KylinConfig kylinConf,
boolean ifForceRecommend) {
- long threshold1 = 1L <<
kylinConf.getCubePlannerAgreedyAlgorithmAutoThreshold() - 1;
+ long threshold1 = 1L <<
kylinConf.getCubePlannerGreedyAlgorithmAutoThreshold() - 1;
long threshold2 = 1L <<
kylinConf.getCubePlannerGeneticAlgorithmAutoThreshold() - 1;
if (threshold1 >= threshold2) {
logger.error("Invalid Cube Planner Algorithm configuration");
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
index 88b3db1..8beeb75 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
@@ -28,12 +28,17 @@ import java.util.SortedSet;
import java.util.TreeSet;
import org.apache.kylin.common.util.Pair;
-
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.shaded.com.google.common.collect.Lists;
import org.apache.kylin.shaded.com.google.common.collect.Maps;
+import org.apache.kylin.shaded.com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CuboidStatsUtil {
+ private static final Logger logger =
LoggerFactory.getLogger(CuboidStatsUtil.class);
+
private CuboidStatsUtil() {
throw new IllegalStateException("Class CuboidStatsUtil is an utility
class !");
}
@@ -348,6 +353,81 @@ public class CuboidStatsUtil {
return (cuboidToCheck & parentCuboid) == cuboidToCheck;
}
+ /**
+ * The formula for calculating the benefit of cube optimization according
to recommended cuboids is as follows:
+ * query benefit + space benefit
+ * | query benefit = (rollupBenefit - rollupCost) / rollupInputCount
+ * | space benefit = (curTotalSize - recomTotalSize) / spaceLimit
+ * @return the benefit of cube optimization according to recommended
cuboids
+ */
+ public static OptimizationBenefit calculateOptimizationBenefit(Map<Long,
Long> statistics,
+ Map<Long, Double> cuboidsSize, CuboidScheduler curCuboidScheduler,
CuboidScheduler recomCuboidScheduler,
+ Map<Long, Double> cuboidHitProbabilityMap, double spaceLimit,
double spaceBenefitRatio) {
+ if (cuboidHitProbabilityMap == null ||
cuboidHitProbabilityMap.isEmpty()) {
+ return OptimizationBenefit.ZERO;
+ }
+
+ Set<Long> currentCuboids = curCuboidScheduler.getAllCuboidIds();
+ Set<Long> recommendCuboids = recomCuboidScheduler.getAllCuboidIds();
+
+ double rollupBenefit = 0.0;
+ {// compute the benefits of added cuboids in recommendCuboids
+ Set<Long> recommendCuboidsOnly = Sets.newHashSet(recommendCuboids);
+ recommendCuboidsOnly.removeAll(currentCuboids);
+ logger.info("Added cuboids {}", recommendCuboidsOnly);
+ for (Long cuboid : recommendCuboidsOnly) {
+ Long pCuboid = curCuboidScheduler.findBestMatchCuboid(cuboid);
+ long rollupCount = statistics.get(pCuboid) -
statistics.get(cuboid);
+ rollupBenefit += rollupCount *
cuboidHitProbabilityMap.get(cuboid);
+ }
+ }
+
+ double rollupCost = 0.0;
+ {// compute the cost of removed cuboids in currentCuboids
+ Set<Long> currentCuboidsOnly = Sets.newHashSet(currentCuboids);
+ currentCuboidsOnly.removeAll(recommendCuboids);
+ logger.info("Removed cuboids {}", currentCuboidsOnly);
+ for (Long cuboid : currentCuboidsOnly) {
+ Long pCuboid =
recomCuboidScheduler.findBestMatchCuboid(cuboid);
+ long rollupCount = statistics.get(pCuboid) -
statistics.get(cuboid);
+ rollupCost += rollupCount *
cuboidHitProbabilityMap.get(cuboid);
+ }
+ }
+
+ double rollupInputCount = 0.0;
+ {// compute the total rollup input count
+ for (Long cuboid : currentCuboids) {
+ Long rowCount = statistics.get(cuboid);
+ rollupInputCount += rowCount *
cuboidHitProbabilityMap.get(cuboid);
+ }
+
+ Set<Long> recommendCuboidsOnly = Sets.newHashSet(recommendCuboids);
+ recommendCuboidsOnly.removeAll(currentCuboids);
+ for (Long cuboid : recommendCuboidsOnly) {
+ Long pCuboid = curCuboidScheduler.findBestMatchCuboid(cuboid);
+ Long rowCount = statistics.get(pCuboid);
+ rollupInputCount += rowCount *
cuboidHitProbabilityMap.get(cuboid);
+ }
+ }
+
+ double curTotalSize = 0.0;
+ {// compute the total size of current cuboids
+ for (Long cuboid : currentCuboids) {
+ curTotalSize += cuboidsSize.get(cuboid);
+ }
+ }
+
+ double recomTotalSize = 0.0;
+ {// compute the total size of recommended cuboids
+ for (Long cuboid : recommendCuboids) {
+ recomTotalSize += cuboidsSize.get(cuboid);
+ }
+ }
+
+ return new OptimizationBenefit(rollupBenefit, rollupCost,
rollupInputCount, curTotalSize, recomTotalSize,
+ spaceLimit, spaceBenefitRatio);
+ }
+
private static double calculateRollupRatio(Pair<Long, Long> rollupStats) {
double rollupInputCount = (double) rollupStats.getFirst() +
rollupStats.getSecond();
return rollupInputCount == 0 ? 0 : 1.0 * rollupStats.getFirst() /
rollupInputCount;
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/OptimizationBenefit.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/OptimizationBenefit.java
new file mode 100644
index 0000000..aa6ce01
--- /dev/null
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/OptimizationBenefit.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@SuppressWarnings("serial")
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE,
getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility =
JsonAutoDetect.Visibility.NONE, setterVisibility =
JsonAutoDetect.Visibility.NONE)
+public class OptimizationBenefit extends RootPersistentEntity {
+
+ private static final Logger logger =
LoggerFactory.getLogger(OptimizationBenefit.class);
+
+ public static final String SCORE_TYPE = "opt_benefit";
+
+ public static final Serializer<OptimizationBenefit> CUBE_SCORE_SERIALIZER
= new JsonSerializer<>(
+ OptimizationBenefit.class);
+
+ public static OptimizationBenefit loadFromStore(ResourceStore store,
CubeInstance cubeInstance) throws IOException {
+ return store.getResource(getScorePath(cubeInstance),
CUBE_SCORE_SERIALIZER);
+ }
+
+ public static void saveToStore(ResourceStore store, CubeInstance
cubeInstance, OptimizationBenefit obj)
+ throws IOException {
+ store.putBigResource(getScorePath(cubeInstance), obj,
System.currentTimeMillis(), CUBE_SCORE_SERIALIZER);
+ }
+
+ public static void removeFromStore(ResourceStore store, CubeInstance
cubeInstance) throws IOException {
+ store.deleteResource(getScorePath(cubeInstance));
+ }
+
+ private static String getScorePath(CubeInstance cubeInstance) {
+ return ResourceStore.CUBE_SCORE_ROOT + "/" + SCORE_TYPE + "/" +
cubeInstance.getName()
+ + MetadataConstants.FILE_SURFIX;
+ }
+
+ public static final OptimizationBenefit ZERO = new OptimizationBenefit();
+
+ // queryBenefit = (rollupBenefit - rollupCost) / totalRollupInputCount
+ @JsonProperty("query_benefit")
+ private double queryBenefit;
+
+ @JsonProperty("rollup_benefit")
+ private double rollupBenefit;
+
+ @JsonProperty("rollup_cost")
+ private double rollupCost;
+
+ @JsonProperty("rollup_input_count")
+ private double rollupInputCount;
+
+ // spaceBenefit = (curTotalSize - recomTotalSize) / spaceLimit
+ @JsonProperty("space_benefit")
+ private double spaceBenefit;
+
+ @JsonProperty("recommend_total_size")
+ private double recomTotalSize;
+
+ @JsonProperty("current_total_size")
+ private double curTotalSize;
+
+ @JsonProperty("space_limit_size")
+ private double spaceLimit;
+
+ // total benefit = query benefit + k * space benefit;
+ @JsonProperty("space_benefit_ratio")
+ private double k;
+
+ @JsonProperty("total_benefit")
+ private double totalBenefit;
+
+ public OptimizationBenefit() {
+ this.rollupBenefit = 0;
+ this.rollupCost = 0;
+ this.rollupInputCount = 0;
+ this.queryBenefit = 0;
+
+ this.recomTotalSize = 0;
+ this.curTotalSize = 0;
+ this.spaceLimit = 0;
+ this.spaceBenefit = 0;
+
+ this.k = 0;
+ this.totalBenefit = 0;
+ }
+
+ public OptimizationBenefit(double rollupBenefit, double rollupCost, double
rollupInputCount, double curTotalSize,
+ double recomTotalSize, double spaceLimit) {
+ this(rollupBenefit, rollupCost, rollupInputCount, curTotalSize,
recomTotalSize, spaceLimit, 0.1);
+ }
+
+ public OptimizationBenefit(double rollupBenefit, double rollupCost, double
rollupInputCount, double curTotalSize,
+ double recomTotalSize, double spaceLimit, double
spaceBenefitRatio) {
+ updateRandomUuid();
+
+ this.rollupBenefit = rollupBenefit;
+ this.rollupCost = rollupCost;
+ this.rollupInputCount = rollupInputCount;
+ this.queryBenefit = rollupInputCount == 0 ? 0 : (rollupBenefit -
rollupCost) / rollupInputCount;
+
+ this.curTotalSize = curTotalSize;
+ this.recomTotalSize = recomTotalSize;
+ this.spaceLimit = spaceLimit;
+
+ this.spaceBenefit = spaceLimit == 0 ? 0.0 : (curTotalSize -
recomTotalSize) / spaceLimit;
+
+ this.k = spaceBenefitRatio;
+ this.totalBenefit = queryBenefit + k * spaceBenefit;
+ }
+
+ public double getTotalBenefit() {
+ return totalBenefit;
+ }
+
+ public void setTotalBenefit(double totalBenefit) {
+ this.totalBenefit = totalBenefit;
+ }
+
+ public double getK() {
+ return k;
+ }
+
+ public void setK(double k) {
+ this.k = k;
+ }
+
+ public double getSpaceLimit() {
+ return spaceLimit;
+ }
+
+ public void setSpaceLimit(double spaceLimit) {
+ this.spaceLimit = spaceLimit;
+ }
+
+ public double getCurTotalSize() {
+ return curTotalSize;
+ }
+
+ public void setCurTotalSize(double curTotalSize) {
+ this.curTotalSize = curTotalSize;
+ }
+
+ public double getSpaceBenefit() {
+ return spaceBenefit;
+ }
+
+ public void setSpaceBenefit(double spaceBenefit) {
+ this.spaceBenefit = spaceBenefit;
+ }
+
+ public double getRecomTotalSize() {
+ return recomTotalSize;
+ }
+
+ public void setRecomTotalSize(double recomTotalSize) {
+ this.recomTotalSize = recomTotalSize;
+ }
+
+ public double getRollupInputCount() {
+ return rollupInputCount;
+ }
+
+ public void setRollupInputCount(double rollupInputCount) {
+ this.rollupInputCount = rollupInputCount;
+ }
+
+ public double getRollupBenefit() {
+ return rollupBenefit;
+ }
+
+ public void setRollupBenefit(double rollupBenefit) {
+ this.rollupBenefit = rollupBenefit;
+ }
+
+ public double getRollupCost() {
+ return rollupCost;
+ }
+
+ public void setRollupCost(double rollupCost) {
+ this.rollupCost = rollupCost;
+ }
+
+ public double getQueryBenefit() {
+ return queryBenefit;
+ }
+
+ public void setQueryBenefit(double queryBenefit) {
+ this.queryBenefit = queryBenefit;
+ }
+
+ @JsonProperty("score")
+ public double getScore() {
+ if (isZero()) {
+ return -1;
+ }
+ return (int) (100 * (totalBenefit <= 0 ? 1 : 1 / (1 + totalBenefit)));
+ }
+
+ public String getScoreHint() {
+ if (isZero()) {
+ return "";
+ }
+ String queryDesc = queryBenefit >= 0 ? "speed up " : "slow down ";
+ String spaceDesc = spaceBenefit >= 0 ? "reduce " : "increase ";
+
+ String queryRatio = String.format(Locale.ROOT, "%.0f",
Math.abs(queryBenefit) * 100);
+ String spaceRatio = String.format(Locale.ROOT, "%.0f",
Math.abs(spaceBenefit) * 100);
+ String spaceSize = String.format(Locale.ROOT, "%.0f",
Math.abs(curTotalSize - recomTotalSize));
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("Benefit you will get after optimization: \n");
+ sb.append("- query latency:
").append(queryDesc).append(queryRatio).append("%\n");
+ sb.append("- storage usage:
").append(spaceDesc).append(spaceRatio).append("% with size ").append(spaceSize)
+ .append(" MB\n");
+ return sb.toString();
+ }
+
+ public void printDetails() {
+ logger.info("benefit = queryBenefitRatio + spaceBenefitRatio = {}",
totalBenefit);
+ logger.info(" queryBenefit: {}", queryBenefit);
+ logger.info(" spaceBenefit: {}", spaceBenefit);
+ logger.info(" rollupBenefit: {}", rollupBenefit);
+ logger.info(" rollupCost: {}", rollupCost);
+ logger.info(" rollupInputCount: {}", rollupInputCount);
+ logger.info(" curTotalSize: {}", curTotalSize);
+ logger.info(" recomTotalSize: {}", recomTotalSize);
+ logger.info(" spaceLimit: {}", spaceLimit);
+ logger.info(" spaceBenefitRatio: {}", k);
+ }
+
+ public boolean isZero() {
+ return this.equals(ZERO);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ if (!super.equals(o))
+ return false;
+ OptimizationBenefit that = (OptimizationBenefit) o;
+ return Double.compare(that.queryBenefit, queryBenefit) == 0
+ && Double.compare(that.rollupBenefit, rollupBenefit) == 0
+ && Double.compare(that.rollupCost, rollupCost) == 0
+ && Double.compare(that.rollupInputCount, rollupInputCount) == 0
+ && Double.compare(that.spaceBenefit, spaceBenefit) == 0
+ && Double.compare(that.recomTotalSize, recomTotalSize) == 0
+ && Double.compare(that.curTotalSize, curTotalSize) == 0
+ && Double.compare(that.spaceLimit, spaceLimit) == 0 &&
Double.compare(that.k, k) == 0
+ && Double.compare(that.totalBenefit, totalBenefit) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), queryBenefit, rollupBenefit,
rollupCost, rollupInputCount, spaceBenefit,
+ recomTotalSize, curTotalSize, spaceLimit, k, totalBenefit);
+ }
+}
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/RecommendResult.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/RecommendResult.java
new file mode 100644
index 0000000..a491b69f
--- /dev/null
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/RecommendResult.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.Map;
+
+public class RecommendResult {
+
+ public static class Builder {
+ private Map<Long, Long> recommendCuboids;
+ private OptimizationBenefit optBenefit;
+
+ public Builder(Map<Long, Long> recommendCuboids) {
+ this.recommendCuboids = recommendCuboids;
+ }
+
+ public Builder setOptimizationBenefitModel(OptimizationBenefit
optBenefit) {
+ this.optBenefit = optBenefit;
+ return this;
+ }
+
+ public RecommendResult build() {
+ return new RecommendResult(recommendCuboids, optBenefit);
+ }
+ }
+
+ private Map<Long, Long> recommendCuboids;
+ private OptimizationBenefit optBenefit;
+
+ public RecommendResult(Map<Long, Long> recommendCuboids,
OptimizationBenefit optBenefit) {
+ this.recommendCuboids = recommendCuboids;
+ this.optBenefit = optBenefit;
+ }
+
+ public Map<Long, Long> getRecommendCuboids() {
+ return recommendCuboids;
+ }
+
+ public OptimizationBenefit getOptimizationBenefit() {
+ return optBenefit;
+ }
+}
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
index 6d9b748..ab958aa 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
@@ -27,8 +27,13 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
import org.apache.kylin.cube.cuboid.algorithm.CuboidRecommender;
import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStatsUtil;
+import org.apache.kylin.cube.cuboid.algorithm.OptimizationBenefit;
+import org.apache.kylin.cube.cuboid.algorithm.RecommendResult;
+import org.apache.kylin.shaded.com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,7 +72,7 @@ public class CuboidRecommenderUtil {
}
/** Trigger cube planner phase two for optimization */
- public static Map<Long, Long> getRecommendCuboidList(CubeInstance cube,
Map<Long, Long> hitFrequencyMap,
+ public static RecommendResult getRecommendCuboidList(CubeInstance cube,
Map<Long, Long> hitFrequencyMap,
Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap)
throws IOException {
CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
@@ -84,6 +89,7 @@ public class CuboidRecommenderUtil {
String key = cube.getName();
double queryUncertaintyRatio =
config.getCubePlannerQueryUncertaintyRatio();
double bpusMinBenefitRatio =
config.getCubePlannerBPUSMinBenefitRatio();
+ double expansionRatio = config.getCubePlannerExpansionRateThreshold();
CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid,
statsPair.getFirst(),
statsPair.getSecond()) {
@Override
@@ -100,7 +106,31 @@ public class CuboidRecommenderUtil {
.setHitFrequencyMap(hitFrequencyMap) //
.setRollingUpCountSourceMap(rollingUpCountSourceMap) //
.build();
- return
CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, config);
+ Map<Long, Long> recomCuboidStatsMap =
CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats,
+ config);
+
+ OptimizationBenefit optBftModel;
+ if (hitFrequencyMap != null && !hitFrequencyMap.isEmpty()) {
+ Map<Long, Long> allStats = cuboidStats.getStatistics();
+ Map<Long, Double> allSizes =
CuboidStatsReaderUtil.readCuboidSizeFromCube(allStats, cube);
+
+ CuboidScheduler recomCuboidScheduler = new
TreeCuboidScheduler(cube.getDescriptor(),
+ Lists.newArrayList(recomCuboidStatsMap.keySet()),
+ new
TreeCuboidScheduler.CuboidCostComparator(recomCuboidStatsMap));
+
+ Map<Long, Double> cuboidHitProbabilityMap =
CuboidStatsUtil.calculateCuboidHitProbability(allStats.keySet(),
+ hitFrequencyMap, baseCuboid, queryUncertaintyRatio);
+
+ double spaceLimit = expansionRatio * allSizes.get(baseCuboid);
+ double spaceBenefitRatio =
config.getCubePlannerOptimizationSpaceBenefitRatio();
+
+ optBftModel =
CuboidStatsUtil.calculateOptimizationBenefit(allStats, allSizes,
cuboidScheduler,
+ recomCuboidScheduler, cuboidHitProbabilityMap, spaceLimit,
spaceBenefitRatio);
+ logger.info("The benefit for optimizing cube {} is {}", cube,
optBftModel.getTotalBenefit());
+ } else {
+ optBftModel = OptimizationBenefit.ZERO;
+ }
+ return new
RecommendResult.Builder(recomCuboidStatsMap).setOptimizationBenefitModel(optBftModel).build();
}
/** For future segment level recommend */
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index d237002..e937920 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -39,6 +39,8 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
+import org.apache.kylin.cube.cuboid.algorithm.OptimizationBenefit;
+import org.apache.kylin.cube.cuboid.algorithm.RecommendResult;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
@@ -71,6 +73,7 @@ import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
import org.apache.kylin.rest.request.LookupSnapshotBuildRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
+import org.apache.kylin.rest.response.CuboidRecommendResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.GeneralResponse;
@@ -862,26 +865,34 @@ public class CubeController extends BasicController {
checkCubeExists(cubeName);
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
- Map<Long, Long> cuboidList = getRecommendCuboidList(cube);
- List<Set<String>> dimensionSetList = Lists.newLinkedList();
+ Message msg = MsgPicker.getMsg();
+ String errorMsg = String.format(Locale.ROOT,
msg.getFAIL_RECOMMEND_CUBOID(), cubeName);
+
+ RecommendResult recommRet = getRecommendCuboidList(cube);
+ if (recommRet == null) {
+ logger.warn(errorMsg);
+ throw new InternalErrorException(errorMsg);
+ }
+ Map<Long, Long> cuboidList = recommRet.getRecommendCuboids();
if (cuboidList == null || cuboidList.isEmpty()) {
- logger.info("Cannot get recommended cuboid list for cube " +
cubeName);
- } else {
- if (cuboidList.size() < top) {
- logger.info("Require " + top + " recommended cuboids, but only
" + cuboidList.size() + " is found.");
- }
- Iterator<Long> cuboidIterator = cuboidList.keySet().iterator();
- RowKeyColDesc[] rowKeyColDescList =
cube.getDescriptor().getRowkey().getRowKeyColumns();
-
- while (top-- > 0 && cuboidIterator.hasNext()) {
- Set<String> dimensionSet = Sets.newHashSet();
- dimensionSetList.add(dimensionSet);
- long cuboid = cuboidIterator.next();
- for (int i = 0; i < rowKeyColDescList.length; i++) {
- if ((cuboid & (1L << rowKeyColDescList[i].getBitIndex()))
> 0) {
- dimensionSet.add(rowKeyColDescList[i].getColumn());
- }
+ logger.warn(errorMsg);
+ throw new InternalErrorException(errorMsg);
+ }
+ if (cuboidList.size() < top) {
+ logger.info("Only recommend " + cuboidList.size() + " cuboids less
than topn " + top);
+ }
+ Iterator<Long> cuboidIterator = cuboidList.keySet().iterator();
+ RowKeyColDesc[] rowKeyColDescList =
cube.getDescriptor().getRowkey().getRowKeyColumns();
+
+ List<Set<String>> dimensionSetList = Lists.newLinkedList();
+ while (top-- > 0 && cuboidIterator.hasNext()) {
+ Set<String> dimensionSet = Sets.newHashSet();
+ dimensionSetList.add(dimensionSet);
+ long cuboid = cuboidIterator.next();
+ for (int i = 0; i < rowKeyColDescList.length; i++) {
+ if ((cuboid & (1L << rowKeyColDescList[i].getBitIndex())) > 0)
{
+ dimensionSet.add(rowKeyColDescList[i].getColumn());
}
}
}
@@ -923,13 +934,27 @@ public class CubeController extends BasicController {
@RequestMapping(value = "/{cubeName}/cuboids/recommend", method =
RequestMethod.GET)
@ResponseBody
- public CuboidTreeResponse getRecommendCuboids(@PathVariable String
cubeName) throws IOException {
+ public CuboidRecommendResponse getRecommendCuboids(@PathVariable String
cubeName) throws IOException {
checkCubeExists(cubeName);
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
- Map<Long, Long> recommendCuboidStatsMap = getRecommendCuboidList(cube);
+
+ Message msg = MsgPicker.getMsg();
+ String errorMsg = String.format(Locale.ROOT,
msg.getFAIL_RECOMMEND_CUBOID(), cubeName);
+
+ RecommendResult recommRet = getRecommendCuboidList(cube);
+ if (recommRet == null) {
+ logger.warn(errorMsg);
+ return new CuboidRecommendResponse(new CuboidTreeResponse(),
OptimizationBenefit.ZERO);
+ }
+ Map<Long, Long> recommendCuboidStatsMap =
recommRet.getRecommendCuboids();
if (recommendCuboidStatsMap == null ||
recommendCuboidStatsMap.isEmpty()) {
- return new CuboidTreeResponse();
+ logger.warn(errorMsg);
+ return new CuboidRecommendResponse(new CuboidTreeResponse(),
OptimizationBenefit.ZERO);
}
+
+ //update cube score
+ cubeService.getCubeManager().saveCubeOptimizationBenefit(cube,
recommRet.getOptimizationBenefit());
+
CuboidScheduler cuboidScheduler = new
TreeCuboidScheduler(cube.getDescriptor(),
Lists.newArrayList(recommendCuboidStatsMap.keySet()),
new
TreeCuboidScheduler.CuboidCostComparator(recommendCuboidStatsMap));
@@ -940,11 +965,13 @@ public class CubeController extends BasicController {
Map<Long, Long> queryMatchMap =
cubeService.getCuboidQueryMatchCount(cubeName);
Set<Long> currentCuboidSet =
cube.getCuboidScheduler().getAllCuboidIds();
- return cubeService.getCuboidTreeResponse(cuboidScheduler,
recommendCuboidStatsMap, displayHitFrequencyMap,
- queryMatchMap, currentCuboidSet);
+
+ CuboidTreeResponse cuboidTree =
cubeService.getCuboidTreeResponse(cuboidScheduler, recommendCuboidStatsMap,
+ displayHitFrequencyMap, queryMatchMap, currentCuboidSet);
+ return new CuboidRecommendResponse(cuboidTree,
recommRet.getOptimizationBenefit());
}
- private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws
IOException {
+ private RecommendResult getRecommendCuboidList(CubeInstance cube) throws
IOException {
// Get cuboid source info
Map<Long, Long> optimizeHitFrequencyMap =
getSourceCuboidHitFrequency(cube.getName());
Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap =
cubeService
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
index ea23761..a06d897 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
@@ -173,6 +173,11 @@ public class Message {
return "Rebuild snapshot of hive view '%s' is not supported, please
refresh segment of the cube";
}
+ // Cube Planner
+ public String getFAIL_RECOMMEND_CUBOID() {
+ return "Cannot get recommend cuboid list for cube '%s'.";
+ }
+
// Model
public String getINVALID_MODEL_DEFINITION() {
return "The data model definition is invalid.";
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/response/CuboidRecommendResponse.java
b/server-base/src/main/java/org/apache/kylin/rest/response/CuboidRecommendResponse.java
new file mode 100644
index 0000000..58bbd8c
--- /dev/null
+++
b/server-base/src/main/java/org/apache/kylin/rest/response/CuboidRecommendResponse.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.response;
+
+import java.io.Serializable;
+
+import org.apache.kylin.cube.cuboid.algorithm.OptimizationBenefit;
+
+public class CuboidRecommendResponse implements Serializable {
+
+ private CuboidTreeResponse cuboidTree;
+ private OptimizationBenefit optBenefit;
+
+ public CuboidRecommendResponse(CuboidTreeResponse cuboidTree,
OptimizationBenefit optBenefit) {
+ this.cuboidTree = cuboidTree;
+ this.optBenefit = optBenefit;
+ }
+
+ public CuboidTreeResponse getCuboidTree() {
+ return cuboidTree;
+ }
+
+ public OptimizationBenefit getOptBenefit() {
+ return optBenefit;
+ }
+}
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index f11f31a..f332b4c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -43,6 +43,7 @@ import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidCLI;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.algorithm.RecommendResult;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
@@ -1019,7 +1020,7 @@ public class CubeService extends BasicService implements
InitializingBean {
}
/** cube planner services */
- public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube,
Map<Long, Long> hitFrequencyMap,
+ public RecommendResult getRecommendCuboidStatistics(CubeInstance cube,
Map<Long, Long> hitFrequencyMap,
Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap)
throws IOException {
aclEvaluate.checkProjectAdminPermission(cube.getProject());
return CuboidRecommenderUtil.getRecommendCuboidList(cube,
hitFrequencyMap, rollingUpCountSourceMap);
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 522d043..13de034 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -74,7 +74,7 @@ KylinApp.factory('CubeService', ['$resource', function
($resource, config) {
isArray: false,
interceptor: {
response: function(response) {
- return transformCuboidsResponse(response.data);
+ return transformCuboidsResponse(response.data.cuboidTree);
}
}
},
