Repository: kylin Updated Branches: refs/heads/KYLIN-2428 ae3dff85c -> e911ef473
KYLIN-2428 clean guava Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e911ef47 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e911ef47 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e911ef47 Branch: refs/heads/KYLIN-2428 Commit: e911ef4730b6fcdf7405216b30500a5e0e2ff570 Parents: ae3dff8 Author: Billy Liu <billy...@apache.org> Authored: Mon Feb 13 11:49:19 2017 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Mon Feb 13 11:49:19 2017 +0800 ---------------------------------------------------------------------- .../kylin/engine/spark/KylinKryoRegistrator.java | 10 +++------- .../kylin/engine/spark/SparkCubingByLayer.java | 6 +++--- .../apache/kylin/engine/spark/SparkExecutable.java | 15 ++++++++------- .../engine/spark/cube/ListBackedCuboidWriter.java | 4 +--- .../kylin/engine/spark/util/IteratorUtils.java | 4 +--- 5 files changed, 16 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java index 322e835..f5f4f36 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java @@ -19,15 +19,11 @@ package org.apache.kylin.engine.spark; import com.esotericsoftware.kryo.Kryo; -import com.google.common.collect.Sets; -import com.google.common.hash.Hashing; -//import org.apache.kylin.measure.MeasureIngester; import org.apache.spark.serializer.KryoRegistrator; -//import org.reflections.Reflections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -//import java.io.Serializable; +import java.util.LinkedHashSet; import java.util.Set; /** @@ -39,7 +35,7 @@ public class KylinKryoRegistrator implements KryoRegistrator { @Override public void registerClasses(Kryo kryo) { - Set<Class> kyroClasses = Sets.newLinkedHashSet(); + Set<Class> kyroClasses = new LinkedHashSet(); kyroClasses.add(byte[].class); kyroClasses.add(int[].class); kyroClasses.add(byte[][].class); @@ -74,7 +70,7 @@ public class KylinKryoRegistrator implements KryoRegistrator { kyroClasses.add(org.apache.spark.sql.types.DateType$.class); kyroClasses.add(org.apache.spark.sql.types.Metadata.class); kyroClasses.add(org.apache.spark.sql.types.StringType$.class); - kyroClasses.add(Hashing.murmur3_128().getClass()); +// kyroClasses.add(Hashing.murmur3_128().getClass()); kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class); kyroClasses.add(org.apache.spark.sql.types.Decimal.class); kyroClasses.add(scala.math.BigDecimal.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java index 317d2bd..9329c08 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java @@ -17,7 +17,6 @@ */ package org.apache.kylin.engine.spark; -import com.google.common.collect.Lists; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; @@ -72,6 +71,7 @@ import java.io.File; import java.io.FileFilter; import java.io.Serializable; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -333,7 +333,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa } } - private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = Lists.newArrayListWithCapacity(0); + private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = new ArrayList(0); class CuboidFlatMap implements PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> { @@ -370,7 +370,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa return EMTPY_ITERATOR; } - List<Tuple2<ByteArray, Object[]>> tuples = Lists.newArrayListWithCapacity(myChildren.size()); + List<Tuple2<ByteArray, Object[]>> tuples = new ArrayList(myChildren.size()); for (Long child : myChildren) { Cuboid childCuboid = Cuboid.findById(cubeDesc, child); Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers()); http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java index 4ab846f..015b480 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java @@ -20,8 +20,7 @@ package org.apache.kylin.engine.spark; import java.io.File; import java.util.Map; -import jodd.util.StringUtil; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.common.util.Logger; @@ -31,8 +30,6 @@ import org.apache.kylin.job.execution.ExecutableContext; import org.apache.kylin.job.execution.ExecuteResult; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; - /** */ public class SparkExecutable extends AbstractExecutable { @@ -74,12 +71,16 @@ public class SparkExecutable extends AbstractExecutable { @Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final KylinConfig config = context.getConfig(); - Preconditions.checkNotNull(config.getSparkHome()); - Preconditions.checkNotNull(config.getKylinJobJarPath()); + if (config.getSparkHome() == null) { + throw new NullPointerException(); + } + if (config.getKylinJobJarPath() == null) { + throw new NullPointerException(); + } String jars = this.getParam(JARS); String hadoopConf = "/etc/hadoop/conf"; - if (StringUtil.isNotEmpty(config.getHadoopConfDir())) { + if (StringUtils.isNotEmpty(config.getHadoopConfDir())) { hadoopConf = config.getHadoopConfDir(); } else { String hiveConf = ClassLoader.getSystemClassLoader().getResource("hive-site.xml").getFile().toString(); http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java index 531670f..a2740bf 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java @@ -23,8 +23,6 @@ import java.util.ArrayList; import org.apache.kylin.engine.spark.SparkCuboidWriter; import org.apache.kylin.gridtable.GTRecord; -import com.google.common.collect.Lists; - import scala.Tuple2; /** @@ -35,7 +33,7 @@ public class ListBackedCuboidWriter implements SparkCuboidWriter { private final TupleConverter tupleConverter; public ListBackedCuboidWriter(TupleConverter tupleConverter) { - this.result = Lists.newArrayList(); + this.result = new ArrayList(); this.tupleConverter = tupleConverter; } http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java index 13464a0..a8a4d28 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java @@ -24,8 +24,6 @@ import java.util.NoSuchElementException; import org.apache.spark.api.java.function.Function; -import com.google.common.collect.Lists; - import scala.Tuple2; /** @@ -47,7 +45,7 @@ public class IteratorUtils { if (!hasNext()) { throw new NoSuchElementException(); } - final LinkedList<V> values = Lists.newLinkedList(); + final LinkedList<V> values = new LinkedList(); K currentKey = current._1(); values.add(current._2()); while (input.hasNext()) {