Repository: beam Updated Branches: refs/heads/DSL_SQL df5859d44 -> 9bcae5326
[BEAM-2440] BeamSql: reduce visibility Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fc5906e1 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fc5906e1 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fc5906e1 Branch: refs/heads/DSL_SQL Commit: fc5906e1e56c323ce520f93e08a024f65b2e4021 Parents: df5859d Author: James Xu <xumingmi...@gmail.com> Authored: Thu Jun 15 10:30:47 2017 +0800 Committer: James Xu <xumingmi...@gmail.com> Committed: Fri Jun 16 09:42:06 2017 +0800 ---------------------------------------------------------------------- .../beam/dsls/sql/example/BeamSqlExample.java | 2 +- .../dsls/sql/planner/BeamPipelineCreator.java | 2 +- .../beam/dsls/sql/planner/BeamSqlRelUtils.java | 74 -------------------- .../planner/UnsupportedOperatorsVisitor.java | 2 +- .../beam/dsls/sql/rel/BeamAggregationRel.java | 1 - .../apache/beam/dsls/sql/rel/BeamFilterRel.java | 1 - .../apache/beam/dsls/sql/rel/BeamIOSinkRel.java | 1 - .../beam/dsls/sql/rel/BeamIOSourceRel.java | 1 - .../beam/dsls/sql/rel/BeamProjectRel.java | 2 - .../apache/beam/dsls/sql/rel/BeamRelNode.java | 5 +- .../dsls/sql/rel/BeamSetOperatorRelBase.java | 1 - .../apache/beam/dsls/sql/rel/BeamSortRel.java | 1 - .../beam/dsls/sql/rel/BeamSqlRelUtils.java | 73 +++++++++++++++++++ .../apache/beam/dsls/sql/rel/BeamValuesRel.java | 1 - 14 files changed, 78 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java index 8ba785b..6bb1617 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/example/BeamSqlExample.java @@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory; * This is a quick example, which uses Beam SQL DSL to create a data pipeline. * */ -public class BeamSqlExample { +class BeamSqlExample { private static final Logger LOG = LoggerFactory.getLogger(BeamSqlExample.class); public static void main(String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java index abdc66c..38e039f 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamPipelineCreator.java @@ -27,7 +27,7 @@ import org.apache.beam.sdk.Pipeline; * pipeline. * */ -public class BeamPipelineCreator { +class BeamPipelineCreator { private Map<String, BaseBeamTable> sourceTables; private Pipeline pipeline; http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRelUtils.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRelUtils.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRelUtils.java deleted file mode 100644 index d9b6e17..0000000 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/BeamSqlRelUtils.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.dsls.sql.planner; - -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.beam.dsls.sql.rel.BeamRelNode; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.volcano.RelSubset; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.sql.SqlExplainLevel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utilities for {@code BeamRelNode}. - */ -public class BeamSqlRelUtils { - private static final Logger LOG = LoggerFactory.getLogger(BeamSqlRelUtils.class); - - private static final AtomicInteger sequence = new AtomicInteger(0); - private static final AtomicInteger classSequence = new AtomicInteger(0); - - public static String getStageName(BeamRelNode relNode) { - return relNode.getClass().getSimpleName().toUpperCase() + "_" + relNode.getId() + "_" - + sequence.getAndIncrement(); - } - - public static String getClassName(BeamRelNode relNode) { - return "Generated_" + relNode.getClass().getSimpleName().toUpperCase() + "_" + relNode.getId() - + "_" + classSequence.getAndIncrement(); - } - - public static BeamRelNode getBeamRelInput(RelNode input) { - if (input instanceof RelSubset) { - // go with known best input - input = ((RelSubset) input).getBest(); - } - return (BeamRelNode) input; - } - - public static String explain(final RelNode rel) { - return explain(rel, SqlExplainLevel.EXPPLAN_ATTRIBUTES); - } - - public static String explain(final RelNode rel, SqlExplainLevel detailLevel) { - String explain = ""; - try { - explain = RelOptUtil.toString(rel); - } catch (StackOverflowError e) { - LOG.error("StackOverflowError occurred while extracting plan. " - + "Please report it to the dev@ mailing list."); - LOG.error("RelNode " + rel + " ExplainLevel " + detailLevel, e); - LOG.error("Forcing plan to empty string and continue... " - + "SQL Runner may not working properly after."); - } - return explain; - } -} http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java index 9dfa21d..4a71024 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/planner/UnsupportedOperatorsVisitor.java @@ -23,6 +23,6 @@ import org.apache.calcite.sql.util.SqlShuttle; * Unsupported operation to visit a RelNode. * */ -public class UnsupportedOperatorsVisitor extends SqlShuttle { +class UnsupportedOperatorsVisitor extends SqlShuttle { } http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java index 828dcec..595563d 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamAggregationRel.java @@ -19,7 +19,6 @@ package org.apache.beam.dsls.sql.rel; import java.util.List; -import org.apache.beam.dsls.sql.planner.BeamSqlRelUtils; import org.apache.beam.dsls.sql.schema.BeamSqlRecordType; import org.apache.beam.dsls.sql.schema.BeamSqlRow; import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder; http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java index dc13646..40fe05c 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamFilterRel.java @@ -19,7 +19,6 @@ package org.apache.beam.dsls.sql.rel; import org.apache.beam.dsls.sql.interpreter.BeamSqlExpressionExecutor; import org.apache.beam.dsls.sql.interpreter.BeamSqlFnExecutor; -import org.apache.beam.dsls.sql.planner.BeamSqlRelUtils; import org.apache.beam.dsls.sql.schema.BeamSqlRow; import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder; import org.apache.beam.dsls.sql.transform.BeamSqlFilterFn; http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java index 76a7cb8..88fff63 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSinkRel.java @@ -20,7 +20,6 @@ package org.apache.beam.dsls.sql.rel; import com.google.common.base.Joiner; import java.util.List; import org.apache.beam.dsls.sql.BeamSqlEnv; -import org.apache.beam.dsls.sql.planner.BeamSqlRelUtils; import org.apache.beam.dsls.sql.schema.BaseBeamTable; import org.apache.beam.dsls.sql.schema.BeamSqlRow; import org.apache.beam.sdk.values.PCollection; http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java index 3fdeb28..ed2bf12 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamIOSourceRel.java @@ -19,7 +19,6 @@ package org.apache.beam.dsls.sql.rel; import com.google.common.base.Joiner; import org.apache.beam.dsls.sql.BeamSqlEnv; -import org.apache.beam.dsls.sql.planner.BeamSqlRelUtils; import org.apache.beam.dsls.sql.schema.BaseBeamTable; import org.apache.beam.dsls.sql.schema.BeamSqlRow; import org.apache.beam.sdk.values.PCollection; http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java index 937a834..e6331c6 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamProjectRel.java @@ -18,10 +18,8 @@ package org.apache.beam.dsls.sql.rel; import java.util.List; - import org.apache.beam.dsls.sql.interpreter.BeamSqlExpressionExecutor; import org.apache.beam.dsls.sql.interpreter.BeamSqlFnExecutor; -import org.apache.beam.dsls.sql.planner.BeamSqlRelUtils; import org.apache.beam.dsls.sql.schema.BeamSqlRow; import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder; import org.apache.beam.dsls.sql.transform.BeamSqlProjectFn; http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java index 80d1f39..0233ccf 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamRelNode.java @@ -17,7 +17,6 @@ */ package org.apache.beam.dsls.sql.rel; -import org.apache.beam.dsls.sql.planner.BeamPipelineCreator; import org.apache.beam.dsls.sql.schema.BeamSqlRow; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollectionTuple; @@ -25,13 +24,13 @@ import org.apache.calcite.rel.RelNode; /** * A new method {@link #buildBeamPipeline(PCollectionTuple)} is added, it's - * called by {@link BeamPipelineCreator}. + * called by {@code BeamPipelineCreator}. */ public interface BeamRelNode extends RelNode { /** * A {@link BeamRelNode} is a recursive structure, the - * {@link BeamPipelineCreator} visits it with a DFS(Depth-First-Search) + * {@code BeamPipelineCreator} visits it with a DFS(Depth-First-Search) * algorithm. */ PCollection<BeamSqlRow> buildBeamPipeline(PCollectionTuple inputPCollections) throws Exception; http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSetOperatorRelBase.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSetOperatorRelBase.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSetOperatorRelBase.java index 271e98f..3d41e3a 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSetOperatorRelBase.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSetOperatorRelBase.java @@ -21,7 +21,6 @@ package org.apache.beam.dsls.sql.rel; import java.io.Serializable; import java.util.List; -import org.apache.beam.dsls.sql.planner.BeamSqlRelUtils; import org.apache.beam.dsls.sql.schema.BeamSqlRow; import org.apache.beam.dsls.sql.transform.BeamSetOperatorsTransforms; import org.apache.beam.sdk.transforms.MapElements; http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java index 7632e6a..6c7be0b 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSortRel.java @@ -24,7 +24,6 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import org.apache.beam.dsls.sql.planner.BeamSqlRelUtils; import org.apache.beam.dsls.sql.schema.BeamSqlRow; import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder; import org.apache.beam.dsls.sql.utils.CalciteUtils; http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSqlRelUtils.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSqlRelUtils.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSqlRelUtils.java new file mode 100644 index 0000000..9f1f703 --- /dev/null +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamSqlRelUtils.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.dsls.sql.rel; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.sql.SqlExplainLevel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utilities for {@code BeamRelNode}. + */ +class BeamSqlRelUtils { + private static final Logger LOG = LoggerFactory.getLogger(BeamSqlRelUtils.class); + + private static final AtomicInteger sequence = new AtomicInteger(0); + private static final AtomicInteger classSequence = new AtomicInteger(0); + + public static String getStageName(BeamRelNode relNode) { + return relNode.getClass().getSimpleName().toUpperCase() + "_" + relNode.getId() + "_" + + sequence.getAndIncrement(); + } + + public static String getClassName(BeamRelNode relNode) { + return "Generated_" + relNode.getClass().getSimpleName().toUpperCase() + "_" + relNode.getId() + + "_" + classSequence.getAndIncrement(); + } + + public static BeamRelNode getBeamRelInput(RelNode input) { + if (input instanceof RelSubset) { + // go with known best input + input = ((RelSubset) input).getBest(); + } + return (BeamRelNode) input; + } + + public static String explain(final RelNode rel) { + return explain(rel, SqlExplainLevel.EXPPLAN_ATTRIBUTES); + } + + public static String explain(final RelNode rel, SqlExplainLevel detailLevel) { + String explain = ""; + try { + explain = RelOptUtil.toString(rel); + } catch (StackOverflowError e) { + LOG.error("StackOverflowError occurred while extracting plan. " + + "Please report it to the dev@ mailing list."); + LOG.error("RelNode " + rel + " ExplainLevel " + detailLevel, e); + LOG.error("Forcing plan to empty string and continue... " + + "SQL Runner may not working properly after."); + } + return explain; + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/fc5906e1/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamValuesRel.java ---------------------------------------------------------------------- diff --git a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamValuesRel.java b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamValuesRel.java index 61d9713..ce75768 100644 --- a/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamValuesRel.java +++ b/dsls/sql/src/main/java/org/apache/beam/dsls/sql/rel/BeamValuesRel.java @@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.List; -import org.apache.beam.dsls.sql.planner.BeamSqlRelUtils; import org.apache.beam.dsls.sql.schema.BeamSqlRecordType; import org.apache.beam.dsls.sql.schema.BeamSqlRow; import org.apache.beam.dsls.sql.schema.BeamSqlRowCoder;
