Repository: kylin Updated Branches: refs/heads/spark-it 36980c38b -> 387fa1458
Make integration test to building cube with spark Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/387fa145 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/387fa145 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/387fa145 Branch: refs/heads/spark-it Commit: 387fa1458e950fb588aa38c006ba7fa514dd7f23 Parents: 36980c3 Author: shaofengshi <[email protected]> Authored: Fri Jan 20 16:03:55 2017 +0800 Committer: shaofengshi <[email protected]> Committed: Fri Jan 20 16:03:55 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 6 ++++++ .../measure/bitmap/RoaringBitmapCounter.java | 3 ++- .../bitmap/RoaringBitmapCounterFactory.java | 3 ++- .../measure/percentile/PercentileCounter.java | 22 +++++++++++++++++++- .../apache/kylin/engine/spark/SparkCubing.java | 5 ++++- kylin-it/pom.xml | 19 +++++++++++++++++ 6 files changed, 54 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 35f5f7a..1734bee 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -68,6 +68,12 @@ abstract public class KylinConfigBase implements Serializable { return sparkHome; } + sparkHome = System.getProperty("SPARK_HOME"); + if (StringUtils.isNotEmpty(sparkHome)) { + logger.info("SPARK_HOME was set to " + sparkHome); + return sparkHome; + } + return getKylinHome() + File.separator + "spark"; } http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java index cd07d20..3d6d3e4 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounter.java @@ -24,6 +24,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap; import java.io.DataOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.Iterator; @@ -31,7 +32,7 @@ import java.util.Iterator; /** * A {@link BitmapCounter} based on roaring bitmap. */ -public class RoaringBitmapCounter implements BitmapCounter { +public class RoaringBitmapCounter implements BitmapCounter, Serializable { private ImmutableRoaringBitmap bitmap; http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java index a71df95..822afa2 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/RoaringBitmapCounterFactory.java @@ -21,9 +21,10 @@ package org.apache.kylin.measure.bitmap; import org.roaringbitmap.buffer.MutableRoaringBitmap; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; -public class RoaringBitmapCounterFactory implements BitmapCounterFactory { +public class RoaringBitmapCounterFactory implements BitmapCounterFactory, Serializable { public static final BitmapCounterFactory INSTANCE = new RoaringBitmapCounterFactory(); private RoaringBitmapCounterFactory() {} http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java index bf505cf..18d3bba 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java @@ -18,6 +18,9 @@ package org.apache.kylin.measure.percentile; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.Serializable; import java.nio.ByteBuffer; @@ -30,7 +33,7 @@ public class PercentileCounter implements Serializable { double compression; double quantileRatio; - TDigest registers; + transient TDigest registers; public PercentileCounter(double compression) { this(compression, INVALID_QUANTILE_RATIO); @@ -94,4 +97,21 @@ public class PercentileCounter implements Serializable { public void clear() { reInitRegisters(); } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + registers.compress(); + int bound = registers.byteSize(); + ByteBuffer buf = ByteBuffer.allocate(bound); + registers.asSmallBytes(buf); + out.writeInt(bound); + out.write(buf.array()); + } + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + int bound = in.readInt(); + ByteBuffer buf = ByteBuffer.allocate(bound); + in.read(buf.array()); + registers = AVLTreeDigest.fromBytes(buf); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java index 0437a80..1a6ec70 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java @@ -630,7 +630,10 @@ public class SparkCubing extends AbstractApplication { kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class); kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class); kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class); - kyroClasses.add(org.apache.kylin.measure.topn.Counter.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileCounter.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class); try { kyroClasses.add(Class.forName("com.google.common.collect.EmptyImmutableList")); http://git-wip-us.apache.org/repos/asf/kylin/blob/387fa145/kylin-it/pom.xml ---------------------------------------------------------------------- diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index 69e7909..5ea11a9 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -241,6 +241,25 @@ <artifactId>kafka_2.10</artifactId> <scope>provided</scope> </dependency> + + <!-- Spark dependency --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_2.10</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-sql_2.10</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-hive_2.10</artifactId> + <scope>provided</scope> + </dependency> </dependencies>
