APACHE-KYLIN-2723: change java Serializationutils to json serializer and fix same class name & cannot cast issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/167ab463 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/167ab463 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/167ab463 Branch: refs/heads/yaho-cube-planner Commit: 167ab463b9b76091a9b08c426f811d1f0d4e3ec1 Parents: 4ef7962 Author: Zhong <nju_y...@apache.org> Authored: Tue Aug 15 00:49:53 2017 +0800 Committer: Zhong <nju_y...@apache.org> Committed: Tue Aug 15 00:49:53 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/QueryContext.java | 118 ++++++++++++++++++- .../org/apache/kylin/common/util/JsonUtil.java | 17 +++ .../apache/kylin/rest/response/SQLResponse.java | 15 ++- .../tool/metrics/systemcube/SCCreator.java | 11 +- .../tool/metrics/systemcube/SCCreatorTest.java | 10 +- tool/src/test/resources/SCSinkTools.json | 15 +-- 6 files changed, 149 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/167ab463/core-common/src/main/java/org/apache/kylin/common/QueryContext.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java index 09fbd13..7aa3bd8 100644 --- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java +++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java @@ -221,6 +221,38 @@ public class QueryContext { return skippedRows; } + public void setRealizationName(String realizationName) { + this.realizationName = realizationName; + } + + public void setRpcServer(String rpcServer) { + this.rpcServer = rpcServer; + } + + public void setCallTimeMs(long callTimeMs) { + this.callTimeMs = callTimeMs; + } + + public void setSkippedRows(long skippedRows) { + this.skippedRows = skippedRows; + } + + public void setScannedRows(long scannedRows) { + this.scannedRows = scannedRows; + } + + public void setReturnedRows(long returnedRows) { + this.returnedRows = returnedRows; + } + + public void setAggregatedRows(long aggregatedRows) { + this.aggregatedRows = aggregatedRows; + } + + public void setScannedBytes(long scannedBytes) { + this.scannedBytes = scannedBytes; + } + public long getScannedRows() { return scannedRows; } @@ -289,6 +321,62 @@ public class QueryContext { this.storageScannedBytes += scanBytes; } + public void setCubeName(String cubeName) { + this.cubeName = cubeName; + } + + public void setSegmentName(String segmentName) { + this.segmentName = segmentName; + } + + public void setSourceCuboidId(long sourceCuboidId) { + this.sourceCuboidId = sourceCuboidId; + } + + public void setTargetCuboidId(long targetCuboidId) { + this.targetCuboidId = targetCuboidId; + } + + public void setFilterMask(long filterMask) { + this.filterMask = filterMask; + } + + public void setIfSuccess(boolean ifSuccess) { + this.ifSuccess = ifSuccess; + } + + public void setCallCount(long callCount) { + this.callCount = callCount; + } + + public void setCallTimeSum(long callTimeSum) { + this.callTimeSum = callTimeSum; + } + + public void setCallTimeMax(long callTimeMax) { + this.callTimeMax = callTimeMax; + } + + public void setStorageSkippedRows(long storageSkippedRows) { + this.storageSkippedRows = storageSkippedRows; + } + + public void setStorageScannedRows(long storageScannedRows) { + this.storageScannedRows = storageScannedRows; + } + + public void setStorageReturnedRows(long storageReturnedRows) { + this.storageReturnedRows = storageReturnedRows; + } + + public void setStorageAggregatedRows(long storageAggregatedRows) { + this.storageAggregatedRows = storageAggregatedRows; + } + + public void setStorageScannedBytes(long storageScannedBytes) { + this.storageScannedBytes = storageScannedBytes; + } + public String getCubeName() { return cubeName; } @@ -355,11 +443,14 @@ public class QueryContext { public static class CubeSegmentStatisticsResult implements Serializable { protected static final long serialVersionUID = 1L; - private final String queryType; - private final Map<String, Map<String, CubeSegmentStatistics>> cubeSegmentStatisticsMap; + private String queryType; + private Map<String, Map<String, CubeSegmentStatistics>> cubeSegmentStatisticsMap; private String realization; private int realizationType; + public CubeSegmentStatisticsResult() { + } + public CubeSegmentStatisticsResult(String queryType, Map<String, Map<String, CubeSegmentStatistics>> cubeSegmentStatisticsMap) { this.queryType = queryType; @@ -382,8 +473,18 @@ public class QueryContext { this.realizationType = realizationType; } + public void setQueryType(String queryType) { + this.queryType = queryType; + } + + public void setCubeSegmentStatisticsMap( + Map<String, Map<String, CubeSegmentStatistics>> cubeSegmentStatisticsMap) { + this.cubeSegmentStatisticsMap = cubeSegmentStatisticsMap; + } + public String getQueryType() { return queryType; + } public Map<String, Map<String, CubeSegmentStatistics>> getCubeSegmentStatisticsMap() { @@ -401,8 +502,8 @@ public class QueryContext { public static class QueryStatisticsResult implements Serializable { protected static final long serialVersionUID = 1L; - private final List<RPCStatistics> rpcStatisticsList; - private final List<CubeSegmentStatisticsResult> cubeSegmentStatisticsResultList; + private List<RPCStatistics> rpcStatisticsList; + private List<CubeSegmentStatisticsResult> cubeSegmentStatisticsResultList; public QueryStatisticsResult() { rpcStatisticsList = Lists.newArrayList(); @@ -415,6 +516,15 @@ public class QueryContext { this.cubeSegmentStatisticsResultList = cubeSegmentStatisticsResultList; } + public void setRpcStatisticsList(List<RPCStatistics> rpcStatisticsList) { + this.rpcStatisticsList = rpcStatisticsList; + } + + public void setCubeSegmentStatisticsResultList( + List<CubeSegmentStatisticsResult> cubeSegmentStatisticsResultList) { + this.cubeSegmentStatisticsResultList = cubeSegmentStatisticsResultList; + } + public List<RPCStatistics> getRpcStatisticsList() { return rpcStatisticsList; } http://git-wip-us.apache.org/repos/asf/kylin/blob/167ab463/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java index 4f3086a..f874b16 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java @@ -41,10 +41,12 @@ public class JsonUtil { // reuse the object mapper to save memory footprint private static final ObjectMapper mapper = new ObjectMapper(); private static final ObjectMapper indentMapper = new ObjectMapper(); + private static final ObjectMapper typeMapper = new ObjectMapper(); static { mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true); + typeMapper.enableDefaultTyping(); } public static <T> T readValue(File src, Class<T> valueType) @@ -82,6 +84,14 @@ public class JsonUtil { return mapper.readTree(content); } + public static <T> T readValueWithTyping(InputStream src, Class<T> valueType) throws IOException { + return typeMapper.readValue(src, valueType); + } + + public static <T> T readValueWithTyping(byte[] src, Class<T> valueType) throws IOException { + return typeMapper.readValue(src, valueType); + } + public static void writeValueIndent(OutputStream out, Object value) throws IOException, JsonGenerationException, JsonMappingException { indentMapper.writeValue(out, value); @@ -104,4 +114,11 @@ public class JsonUtil { return indentMapper.writeValueAsString(value); } + public static void writeValueWithTyping(OutputStream out, Object value) throws IOException { + typeMapper.writeValue(out, value); + } + + public static byte[] writeValueWithTypingAsBytes(Object value) throws IOException { + return typeMapper.writeValueAsBytes(value); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/167ab463/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java index bca52bc..4897008 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java +++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java @@ -21,8 +21,8 @@ package org.apache.kylin.rest.response; import java.io.Serializable; import java.util.List; -import org.apache.commons.lang.SerializationUtils; import org.apache.kylin.common.QueryContext; +import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.metadata.querymeta.SelectedColumnMeta; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -208,7 +208,7 @@ public class SQLResponse implements Serializable { public QueryContext.QueryStatisticsResult getQueryStatistics() { if (queryStatistics != null) { try { - return (QueryContext.QueryStatisticsResult) SerializationUtils.deserialize(queryStatistics); + return JsonUtil.readValueWithTyping(queryStatistics, QueryContext.QueryStatisticsResult.class); } catch (Exception e) { // Exception may happen due to System.out.println("Error while deserialize queryStatistics due to " + e); } @@ -217,7 +217,14 @@ public class SQLResponse implements Serializable { } public void setQueryStatistics(QueryContext.QueryStatisticsResult queryStatisticsResult) { - this.queryStatistics = queryStatisticsResult == null ? null - : SerializationUtils.serialize(queryStatisticsResult); + if (queryStatisticsResult != null) { + try { + this.queryStatistics = JsonUtil.writeValueWithTypingAsBytes(queryStatisticsResult); + return; + } catch (Exception e) { + System.out.println("Error while serialize queryStatistics due to " + e); + } + } + this.queryStatistics = null; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/167ab463/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/SCCreator.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/SCCreator.java b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/SCCreator.java index fb2fbb5..40dec87 100644 --- a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/SCCreator.java +++ b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/SCCreator.java @@ -38,6 +38,7 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.persistence.Serializer; import org.apache.kylin.common.util.AbstractApplication; +import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeInstance; @@ -52,7 +53,6 @@ import org.apache.kylin.tool.metrics.systemcube.util.HiveSinkTool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -80,12 +80,7 @@ public class SCCreator extends AbstractApplication { private final KylinConfig config; - private final ObjectMapper mapper; - public SCCreator() { - mapper = new ObjectMapper(); - mapper.enableDefaultTyping(); - config = KylinConfig.getInstanceFromEnv(); options = new Options(); @@ -122,8 +117,8 @@ public class SCCreator extends AbstractApplication { output += "/"; } - Set<SinkTool> sourceToolSet = mapper - .readValue(new BufferedInputStream(new FileInputStream(new File(inputConfig))), HashSet.class); + Set<SinkTool> sourceToolSet = JsonUtil.readValueWithTyping( + new BufferedInputStream(new FileInputStream(new File(inputConfig))), HashSet.class); run(owner, output, sourceToolSet); } http://git-wip-us.apache.org/repos/asf/kylin/blob/167ab463/tool/src/test/java/org/apache/kylin/tool/metrics/systemcube/SCCreatorTest.java ---------------------------------------------------------------------- diff --git a/tool/src/test/java/org/apache/kylin/tool/metrics/systemcube/SCCreatorTest.java b/tool/src/test/java/org/apache/kylin/tool/metrics/systemcube/SCCreatorTest.java index 6718723..771b13a 100644 --- a/tool/src/test/java/org/apache/kylin/tool/metrics/systemcube/SCCreatorTest.java +++ b/tool/src/test/java/org/apache/kylin/tool/metrics/systemcube/SCCreatorTest.java @@ -26,6 +26,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.tool.metrics.systemcube.util.HiveSinkTool; @@ -33,7 +34,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -69,9 +69,7 @@ public class SCCreatorTest extends LocalFileMetadataTestCase { try (BufferedOutputStream os = new BufferedOutputStream( new FileOutputStream("src/test/resources/SCSinkTools.json"))) { - ObjectMapper mapper = new ObjectMapper(); - mapper.enableDefaultTyping(); - mapper.writeValue(os, Sets.newHashSet(hiveSinkTool)); + JsonUtil.writeValueWithTyping(os, Sets.newHashSet(hiveSinkTool)); } } @@ -79,9 +77,7 @@ public class SCCreatorTest extends LocalFileMetadataTestCase { public void testReadSinkToolsJson() throws Exception { try (BufferedInputStream is = new BufferedInputStream( new FileInputStream("src/main/resources/SCSinkTools.json"))) { - ObjectMapper mapper = new ObjectMapper(); - mapper.enableDefaultTyping(); - Set<HiveSinkTool> sinkToolSet = mapper.readValue(is, HashSet.class); + Set<HiveSinkTool> sinkToolSet = JsonUtil.readValueWithTyping(is, HashSet.class); for (HiveSinkTool entry : sinkToolSet) { System.out.println(entry.getCubeDescOverrideProperties()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/167ab463/tool/src/test/resources/SCSinkTools.json ---------------------------------------------------------------------- diff --git a/tool/src/test/resources/SCSinkTools.json b/tool/src/test/resources/SCSinkTools.json index 15a715f..310d982 100644 --- a/tool/src/test/resources/SCSinkTools.json +++ b/tool/src/test/resources/SCSinkTools.json @@ -1,14 +1 @@ -[ - [ - "org.apache.kylin.tool.metrics.systemcube.util.HiveSinkTool", - { - "storageType": 2, - "cubeDescOverrideProperties": [ - "java.util.HashMap", - { - "kylin.cube.algorithm": "INMEM" - } - ] - } - ] -] \ No newline at end of file +[["org.apache.kylin.tool.metrics.systemcube.util.HiveSinkTool",{"storageType":2,"cubeDescOverrideProperties":["java.util.HashMap",{"kylin.cube.algorithm":"INMEM"}]}]] \ No newline at end of file
