minor refactoring
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/762992d0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/762992d0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/762992d0 Branch: refs/heads/stream_m1 Commit: 762992d0ac981549f91587a73144e2d554662425 Parents: 11e20ad Author: Hongbin Ma <mahong...@apache.org> Authored: Mon Jun 20 11:43:54 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Mon Jun 20 11:44:06 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/util/SparkEntry.java | 42 +++++++++++++++++++ .../apache/kylin/engine/spark/SparkCubing.java | 1 + .../apache/kylin/engine/spark/SparkEntry.java | 43 -------------------- .../kylin/engine/spark/SparkExecutable.java | 2 +- 4 files changed, 44 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/762992d0/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java new file mode 100644 index 0000000..fd324a2 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.common.util; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.base.Preconditions; + +/** + */ +public final class SparkEntry { + + public static void main(String[] args) throws Exception { + System.out.println("SparkEntry args:" + StringUtils.join(args, " ")); + Preconditions.checkArgument(args.length >= 2, "-className is required"); + Preconditions.checkArgument(args[0].equals("-className"), "-className is required"); + final String className = args[1]; + final Object o = Class.<AbstractApplication> forName(className).newInstance(); + Preconditions.checkArgument(o instanceof AbstractApplication, className + " is not a subClass of AbstractSparkApplication"); + String[] appArgs = new String[args.length - 2]; + for (int i = 2; i < args.length; i++) { + appArgs[i - 2] = args[i]; + } + AbstractApplication abstractApplication = (AbstractApplication) o; + abstractApplication.execute(appArgs); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/762992d0/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java index c56892b..2c7a8fa 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java @@ -610,6 +610,7 @@ public class SparkCubing extends AbstractApplication { } }); + final Map<Long, HyperLogLogPlusCounter> samplingResult = sampling(rowJavaRDD, cubeName); final byte[][] splitKeys = createHTable(cubeName, segmentId, samplingResult); http://git-wip-us.apache.org/repos/asf/kylin/blob/762992d0/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkEntry.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkEntry.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkEntry.java deleted file mode 100644 index 35d223a..0000000 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkEntry.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -package org.apache.kylin.engine.spark; - -import org.apache.commons.lang3.StringUtils; - -import com.google.common.base.Preconditions; -import org.apache.kylin.common.util.AbstractApplication; - -/** - */ -public final class SparkEntry { - - public static void main(String[] args) throws Exception { - System.out.println("SparkEntry args:" + StringUtils.join(args, " ")); - Preconditions.checkArgument(args.length >= 2, "-className is required"); - Preconditions.checkArgument(args[0].equals("-className"), "-className is required"); - final String className = args[1]; - final Object o = Class.<AbstractApplication> forName(className).newInstance(); - Preconditions.checkArgument(o instanceof AbstractApplication, className + " is not a subClass of AbstractSparkApplication"); - String[] appArgs = new String[args.length - 2]; - for (int i = 2; i < args.length; i++) { - appArgs[i - 2] = args[i]; - } - AbstractApplication abstractApplication = (AbstractApplication) o; - abstractApplication.execute(appArgs); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/762992d0/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java index 7aa4250..fd37cfa 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java @@ -64,7 +64,7 @@ public class SparkExecutable extends AbstractExecutable { Preconditions.checkNotNull(config.getSparkHome()); Preconditions.checkNotNull(config.getSparkMaster()); try { - String cmd = String.format("%s/bin/spark-submit --class \"org.apache.kylin.engine.spark.SparkEntry\" --master %s %s %s", + String cmd = String.format("%s/bin/spark-submit --class \"org.apache.kylin.common.util.SparkEntry\" --master %s %s %s", config.getSparkHome(), config.getSparkMaster(), config.getKylinSparkJobJarPath(),
