[40/50] [abbrv] flink git commit: [hotfix] [client] Unwrap SerializedThrowable in client log statements when possible
[hotfix] [client] Unwrap SerializedThrowable in client log statements when
possible
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c9623be2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c9623be2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c9623be2
Branch: refs/heads/table-retraction
Commit: c9623be2ac1ce44c1d04eeda336723b92e2fadf9
Parents: 91b2d6f
Author: Stephan Ewen
Authored: Tue May 2 22:10:40 2017 +0200
Committer: Stephan Ewen
Committed: Tue May 2 22:49:46 2017 +0200
--
.../java/org/apache/flink/runtime/client/JobClientActor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/c9623be2/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
index 368a2b6..a0bf97d 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobClientActor.java
@@ -34,6 +34,7 @@ import org.apache.flink.runtime.messages.JobClientMessages;
import org.apache.flink.runtime.messages.JobClientMessages.JobManagerActorRef;
import
org.apache.flink.runtime.messages.JobClientMessages.JobManagerLeaderAddress;
import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.util.SerializedThrowable;
import org.apache.flink.util.Preconditions;
import scala.concurrent.duration.FiniteDuration;
@@ -287,7 +288,8 @@ public abstract class JobClientActor extends
FlinkUntypedActor implements Leader
System.out.println(message.toString());
}
} else {
- LOG.info(message.toString(), message.error());
+ Throwable error =
SerializedThrowable.get(message.error(), getClass().getClassLoader());
+ LOG.info(message.toString(), error);
if (sysoutUpdates) {
System.out.println(message.toString());
message.error().printStackTrace(System.out);
[14/50] [abbrv] flink git commit: [FLINK-5892] Add new StateAssignmentOperationV2
[FLINK-5892] Add new StateAssignmentOperationV2
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8045faba
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8045faba
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8045faba
Branch: refs/heads/table-retraction
Commit: 8045fabac736cc8c6b48fda8328cf91f329dc3bf
Parents: 591841f
Author: guowei.mgw
Authored: Mon Apr 24 11:47:47 2017 +0200
Committer: zentol
Committed: Fri Apr 28 20:09:11 2017 +0200
--
.../checkpoint/StateAssignmentOperationV2.java | 458 +++
1 file changed, 458 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/8045faba/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
new file mode 100644
index 000..83c188c
--- /dev/null
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
@@ -0,0 +1,458 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.runtime.checkpoint;
+
+//import com.google.common.collect.Lists;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.state.ChainedStateHandle;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KeyGroupsStateHandle;
+import org.apache.flink.runtime.state.KeyedStateHandle;
+import org.apache.flink.runtime.state.OperatorStateHandle;
+import org.apache.flink.runtime.state.StreamStateHandle;
+import org.apache.flink.runtime.state.TaskStateHandles;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class encapsulates the operation of assigning restored state when
restoring from a checkpoint that works on the
+ * granularity of operators. This is the case for checkpoints that were
created either with a Flink version >= 1.3 or
+ * 1.2 if the savepoint only contains {@link SubtaskState}s for which the
length of contained
+ * {@link ChainedStateHandle}s is equal to 1.
+ */
+public class StateAssignmentOperationV2 {
+
+ private final Logger logger;
+ private final Map tasks;
+ private final Map taskStates;
+ private final boolean allowNonRestoredState;
+
+ public StateAssignmentOperationV2(
+ Logger logger,
+ Map tasks,
+ Map taskStates,
+ boolean allowNonRestoredState) {
+
+ this.logger = Preconditions.checkNotNull(logger);
+ this.tasks = Preconditions.checkNotNull(tasks);
+ this.taskStates = Preconditions.checkNotNull(taskStates);
+ this.allowNonRestoredState = allowNonRestoredState;
+ }
+
+ public boolean assignStates() throws Exception {
+ Map localStates = new
HashMap<>(taskStates);
+ Map localTasks = this.tasks;
+
+ Set allOperatorIDs = new HashSet<>();
+ for (ExecutionJobVertex executionJobVertex : tasks.values()) {
+
//allOperatorIDs.addAll(Lists.newArrayList(executionJobVertex.getOperatorIDs()));
+ }
+ for (Map.Entry taskGroupStateEntry :
taskStates.entrySet()) {
+ TaskState taskState = taskGroupStateEntry.getValue();
+ //find operat
[48/50] [abbrv] flink git commit: [FLINK-6093] [table] Implement and turn on retraction for table sinks.
[FLINK-6093] [table] Implement and turn on retraction for table sinks.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/917f7249
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/917f7249
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/917f7249
Branch: refs/heads/table-retraction
Commit: 917f7249cef687d68a233a2f0349dcdcbc187e61
Parents: 856485b
Author: Hequn Cheng
Authored: Thu Apr 27 23:03:44 2017 +0800
Committer: Fabian Hueske
Committed: Wed May 3 11:38:15 2017 +0200
--
.../flink/table/api/BatchTableEnvironment.scala | 16 +--
.../table/api/StreamTableEnvironment.scala | 118 +--
.../apache/flink/table/api/TableConfig.scala| 17 +++
.../flink/table/api/TableEnvironment.scala | 12 --
.../datastream/DataStreamRetractionRules.scala | 5 +-
.../runtime/CRowInputTupleOutputMapRunner.scala | 63 ++
.../apache/flink/table/sinks/CsvTableSink.scala | 98 ++-
.../flink/table/sinks/StreamRetractSink.scala | 35 ++
.../flink/table/TableEnvironmentTest.scala | 62 +-
.../scala/batch/TableEnvironmentITCase.scala| 19 ---
.../api/scala/stream/RetractionITCase.scala | 9 +-
.../api/scala/stream/TableSinkITCase.scala | 33 +-
.../api/scala/stream/utils/StreamITCase.scala | 8 --
13 files changed, 360 insertions(+), 135 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/917f7249/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
index 34f5018..d8d93b0 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
@@ -26,12 +26,11 @@ import org.apache.calcite.rel.RelNode
import org.apache.calcite.rel.`type`.RelDataType
import org.apache.calcite.sql2rel.RelDecorrelator
import org.apache.calcite.tools.RuleSet
-import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction}
+import org.apache.flink.api.common.functions.MapFunction
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.io.DiscardingOutputFormat
import org.apache.flink.api.java.typeutils.GenericTypeInfo
import org.apache.flink.api.java.{DataSet, ExecutionEnvironment}
-import org.apache.flink.configuration.Configuration
import org.apache.flink.table.explain.PlanJsonParser
import org.apache.flink.table.expressions.Expression
import org.apache.flink.table.plan.nodes.FlinkConventions
@@ -39,7 +38,6 @@ import org.apache.flink.table.plan.nodes.dataset.DataSetRel
import org.apache.flink.table.plan.rules.FlinkRuleSets
import org.apache.flink.table.plan.schema.{DataSetTable, TableSourceTable}
import org.apache.flink.table.runtime.MapRunner
-import org.apache.flink.table.runtime.types.CRow
import org.apache.flink.table.sinks.{BatchTableSink, TableSink}
import org.apache.flink.table.sources.{BatchTableSource, TableSource}
import org.apache.flink.types.Row
@@ -150,18 +148,6 @@ abstract class BatchTableEnvironment(
if (requestedTypeInfo.getTypeClass == classOf[Row]) {
// Row to Row, no conversion needed
None
-} else if (requestedTypeInfo.getTypeClass == classOf[CRow]) {
- // Row to CRow, only needs to be wrapped
- Some(
-new RichMapFunction[Row, CRow] {
- private var outCRow: CRow = _
- override def open(parameters: Configuration): Unit = outCRow = new
CRow(null, true)
- override def map(value: Row): CRow = {
-outCRow.row = value
-outCRow
- }
-}.asInstanceOf[MapFunction[IN, OUT]]
- )
} else {
// some type that is neither Row or CRow
http://git-wip-us.apache.org/repos/asf/flink/blob/917f7249/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index e2bccf3..31ae78b 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -27,19 +27,20 @@ import org.apache.calcite.rel.`type`.
[03/50] [abbrv] flink git commit: [FLINK-6293] [tests] Harden JobManagerITCase
[FLINK-6293] [tests] Harden JobManagerITCase One of the unit tests in JobManagerITCase starts a MiniCluster and sends a LeaderSessionMessage to the JobManager without waiting until the JobManager has gained leadership. This can lead to a dropped TriggerSavepoint message which will cause the test to deadlock. This PR fixes the problem by explicitly waiting for the JobManager to become the leader. This closes #3796. Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f3da8f69 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f3da8f69 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f3da8f69 Branch: refs/heads/table-retraction Commit: f3da8f69e99be49068ab4ea3abc5e1c4eba7bf32 Parents: 7c35dc0 Author: Till Rohrmann Authored: Fri Apr 28 10:04:57 2017 +0200 Committer: Till Rohrmann Committed: Fri Apr 28 15:25:44 2017 +0200 -- .../org/apache/flink/runtime/jobmanager/JobManagerITCase.scala | 5 + 1 file changed, 5 insertions(+) -- http://git-wip-us.apache.org/repos/asf/flink/blob/f3da8f69/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala -- diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala index ce8517e..5fb9ddf 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala @@ -765,6 +765,11 @@ class JobManagerITCase(_system: ActorSystem) val jobManager = flinkCluster .getLeaderGateway(deadline.timeLeft) + // we have to make sure that the job manager knows also that he is the leader + // in case of standalone leader retrieval this can happen after the getLeaderGateway call + val leaderFuture = jobManager.ask(NotifyWhenLeader, timeout.duration) + Await.ready(leaderFuture, timeout.duration) + val jobId = new JobID() // Trigger savepoint for non-existing job
[23/50] [abbrv] flink git commit: [FLINK-6112] [table] Support Calcite 1.12's new numerical functions
[FLINK-6112] [table] Support Calcite 1.12's new numerical functions
This closes #3718.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a06b3222
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a06b3222
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a06b3222
Branch: refs/heads/table-retraction
Commit: a06b3222bb97150f643c22fe1a1054e3239c1c2d
Parents: d27d3dd
Author: mtunique
Authored: Fri Apr 7 18:04:03 2017 +0800
Committer: twalthr
Committed: Sun Apr 30 15:12:46 2017 +0200
--
docs/dev/table_api.md | 396 ++
.../flink/table/api/scala/expressionDsl.scala | 68
.../table/codegen/calls/BuiltInMethods.scala| 46 +++
.../table/codegen/calls/ConstantCallGen.scala | 38 ++
.../table/codegen/calls/FunctionGenerator.scala | 161
.../table/expressions/ExpressionParser.scala| 11 +-
.../table/expressions/mathExpressions.scala | 158
.../flink/table/validate/FunctionCatalog.scala | 24 ++
.../table/expressions/ScalarFunctionsTest.scala | 402 ++-
.../table/expressions/SqlExpressionTest.scala | 12 +
10 files changed, 1312 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/a06b3222/docs/dev/table_api.md
--
diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 4b9e565..6d52a78 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1977,6 +1977,127 @@ BOOLEAN.isNotFalse
+
+
+{% highlight java %}
+NUMERIC.sin()
+{% endhighlight %}
+
+
+Calculates the sine of a given number.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.cos()
+{% endhighlight %}
+
+
+Calculates the cosine of a given number.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.tan()
+{% endhighlight %}
+
+
+Calculates the tangent of a given number.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.cot()
+{% endhighlight %}
+
+
+Calculates the tangent of a given number.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.asin()
+{% endhighlight %}
+
+
+Calculates the tangent of a given number.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.acos()
+{% endhighlight %}
+
+
+Calculates the tangent of a given number.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.atan()
+{% endhighlight %}
+
+
+Calculates the tangent of a given number.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.degrees()
+{% endhighlight %}
+
+
+Converts numeric from radians to degrees.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.radians()
+{% endhighlight %}
+
+
+Converts numeric from degrees to radians.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.sign()
+{% endhighlight %}
+
+
+Calculates the signum of a given number.
+
+
+
+
+
+{% highlight java %}
+NUMERIC.round(numeric2)
+{% endhighlight %}
+
+
+Rounds the given number to the other value places right to the
decimal point.
+
+
+
@@ -2156,6 +2277,17 @@ NUMERIC.floor()
+
+
+{% highlight java %}
+pi()
+{% endhighlight %}
+
+
+ Returns pi.
+
+
+
@@ -3175,6 +3307,138 @@ NUMERIC.floor()
Calculates the largest integer less than or equal to a given
number.
+
+
+
+{% highlight scala %}
+NUMERIC.sin()
+{% endhighlight %}
+
+
+Calculates the sine of a given number.
+
+
+
+
+
+{% highlight scala %}
+NUMERIC.cos()
+{% endhighlight %}
+
+
+Calculates the cosine of a given number.
+
+
+
+
+
+{% highlight scala %}
+NUMERIC.tan()
+{% endhighlight %}
+
+
+Calculates the tangent of a given number.
+
+
+
+
+
+{% highlight scala %}
+NUMERIC.cot()
+{% endhighlight %}
+
+
+Calculates the tangent of a given number.
+
+
+
+
+
+{% highlight scala %}
+NUMERIC.asin()
+{% endhighlight %}
+
+
+Calculates the tangent of a given number.
+
+
+
+
+
+{% highlight scala %}
+NUMERIC.acos()
+{% endhighlight %}
+
+
+Calculates the tangent of a given number.
+
+
+
+
+
+{% highlight scala %}
+NUMERIC.atan()
+{% endhighlight %}
+
+
[08/50] [abbrv] flink git commit: [FLINK-5810] [flip-6] Multiple small cleanups across Resource Manager related code
[FLINK-5810] [flip-6] Multiple small cleanups across Resource Manager related
code
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/759f46ea
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/759f46ea
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/759f46ea
Branch: refs/heads/table-retraction
Commit: 759f46ea6716e2f9da98002ed881be2fe6d7ab39
Parents: 59aefb5
Author: Stephan Ewen
Authored: Thu Mar 16 18:53:21 2017 +0100
Committer: Till Rohrmann
Committed: Fri Apr 28 15:28:25 2017 +0200
--
.../types/ResourceIDRetrievable.java| 4 +++-
.../runtime/clusterframework/types/SlotID.java | 20 ++--
.../runtime/concurrent/ScheduledExecutor.java | 2 +-
.../flink/runtime/messages/Acknowledge.java | 3 ++-
.../registration/TaskExecutorConnection.java| 7 ---
.../slotmanager/PendingSlotRequest.java | 10 +-
.../taskexecutor/TaskExecutorGateway.java | 2 --
.../slotmanager/SlotProtocolTest.java | 16
8 files changed, 33 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/759f46ea/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceIDRetrievable.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceIDRetrievable.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceIDRetrievable.java
index b45d53c..e65840d 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceIDRetrievable.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceIDRetrievable.java
@@ -22,6 +22,8 @@ package org.apache.flink.runtime.clusterframework.types;
*/
public interface ResourceIDRetrievable {
+ /**
+* Gets the ResourceID of the object.
+*/
ResourceID getResourceID();
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/759f46ea/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
index d6409b6..83c28b4 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
@@ -24,7 +24,11 @@ import static
org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
- * Unique identifier for a slot which located in TaskManager.
+ * Unique identifier for a slot on a TaskManager. This ID is constant across
the
+ * life time of the TaskManager.
+ *
+ * In contrast, the {@link AllocationID} represents the a slot allocation
and changes
+ * every time the slot is allocated by a JobManager.
*/
public class SlotID implements ResourceIDRetrievable, Serializable {
@@ -66,10 +70,7 @@ public class SlotID implements ResourceIDRetrievable,
Serializable {
SlotID slotID = (SlotID) o;
- if (slotNumber != slotID.slotNumber) {
- return false;
- }
- return resourceId.equals(slotID.resourceId);
+ return slotNumber == slotID.slotNumber &&
resourceId.equals(slotID.resourceId);
}
@Override
@@ -83,13 +84,4 @@ public class SlotID implements ResourceIDRetrievable,
Serializable {
public String toString() {
return resourceId + "_" + slotNumber;
}
-
- /**
-* Generate a random slot id.
-*
-* @return A random slot id.
-*/
- public static SlotID generate() {
- return new SlotID(ResourceID.generate(), 0);
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/759f46ea/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/ScheduledExecutor.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/ScheduledExecutor.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/ScheduledExecutor.java
index c1b47e2..d09cfc3 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/ScheduledExecutor.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/ScheduledExecutor.java
@@ -82,7 +82,7 @@ public interface ScheduledExecutor extends Executor {
* @param delay the time
[20/50] [abbrv] flink git commit: [FLINK-6059] [table] Reject GenericType when converting DataSet or DataStream to Table.
[FLINK-6059] [table] Reject GenericType when converting DataSet or
DataStream to Table.
This closes #3546.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c8eb55f1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c8eb55f1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c8eb55f1
Branch: refs/heads/table-retraction
Commit: c8eb55f17d64722bb600c1083a478ab99e53f4ec
Parents: 2c68085
Author: Fabian Hueske
Authored: Wed Mar 15 13:24:42 2017 +0100
Committer: Fabian Hueske
Committed: Sat Apr 29 00:44:55 2017 +0200
--
.../flink/table/api/TableEnvironment.scala | 13 +++-
.../api/java/batch/TableEnvironmentITCase.java | 31
.../flink/table/TableEnvironmentTest.scala | 18 ++--
.../scala/batch/TableEnvironmentITCase.scala| 30 +++
4 files changed, 88 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/c8eb55f1/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 16c40fe..bd974b0 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -565,7 +565,14 @@ abstract class TableEnvironment(val config: TableConfig) {
*/
protected[flink] def getFieldInfo[A](inputType: TypeInformation[A]):
(Array[String], Array[Int]) = {
-(TableEnvironment.getFieldNames(inputType),
TableEnvironment.getFieldIndices(inputType))
+
+if (inputType.isInstanceOf[GenericTypeInfo[A]] && inputType.getTypeClass
== classOf[Row]) {
+ throw new TableException(
+"An input of GenericTypeInfo cannot be converted to Table. " +
+ "Please specify the type of the input with a RowTypeInfo.")
+} else {
+ (TableEnvironment.getFieldNames(inputType),
TableEnvironment.getFieldIndices(inputType))
+}
}
/**
@@ -584,6 +591,10 @@ abstract class TableEnvironment(val config: TableConfig) {
TableEnvironment.validateType(inputType)
val indexedNames: Array[(Int, String)] = inputType match {
+ case g: GenericTypeInfo[A] if g.getTypeClass == classOf[Row] =>
+throw new TableException(
+ "An input of GenericTypeInfo cannot be converted to Table. " +
+"Please specify the type of the input with a RowTypeInfo.")
case a: AtomicType[A] =>
if (exprs.length != 1) {
throw new TableException("Table of atomic type can only have a
single field.")
http://git-wip-us.apache.org/repos/asf/flink/blob/c8eb55f1/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
--
diff --git
a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
index e165983..cab3855 100644
---
a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
+++
b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
@@ -28,6 +28,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.calcite.tools.RuleSets;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import org.apache.flink.table.api.java.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple4;
@@ -46,6 +47,8 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import static org.junit.Assert.assertTrue;
+
@RunWith(Parameterized.class)
public class TableEnvironmentITCase extends TableProgramsCollectionTestBase {
@@ -375,6 +378,34 @@ public class TableEnvironmentITCase extends
TableProgramsCollectionTestBase {
}
@Test(expected = TableException.class)
+ public void testGenericRow() throws Exception {
+ ExecutionEnvironment env =
ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv =
TableEnvironment.getTableEnvironment(env, config());
+
+ // use null value the enforce GenericType
+ DataSet dataSet = env.fromE
[13/50] [abbrv] flink git commit: [FLINK-6341] [jm] Add test case to guard against RM registration loop
[FLINK-6341] [jm] Add test case to guard against RM registration loop
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/591841f3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/591841f3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/591841f3
Branch: refs/heads/table-retraction
Commit: 591841f30f7a7652b6f418abde17c7f23becf1c0
Parents: 2383839
Author: Till Rohrmann
Authored: Fri Apr 28 15:21:56 2017 +0200
Committer: Till Rohrmann
Committed: Fri Apr 28 17:59:28 2017 +0200
--
.../flink/configuration/JobManagerOptions.java | 10 ++
.../messages/ReconnectResourceManager.java | 18 ++--
.../flink/runtime/jobmanager/JobManager.scala | 24 +++--
.../runtime/jobmanager/JobManagerTest.java | 105 ++-
4 files changed, 138 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/591841f3/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
--
diff --git
a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index d129405..10d9e16 100644
---
a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++
b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -72,6 +72,16 @@ public class JobManagerOptions {
.defaultValue(16)
.withDeprecatedKeys("job-manager.max-attempts-history-size");
+ /**
+* This option specifies the interval in order to trigger a resource
manager reconnection if the connection
+* to the resource manager has been lost.
+*
+* This option is only intended for internal use.
+*/
+ public static final ConfigOption
RESOURCE_MANAGER_RECONNECT_INTERVAL =
+ key("jobmanager.resourcemanager.reconnect-interval")
+ .defaultValue(2000L);
+
//
// JobManager web UI
//
http://git-wip-us.apache.org/repos/asf/flink/blob/591841f3/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
index d02193e..1f852e8 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
@@ -22,7 +22,7 @@ import akka.actor.ActorRef;
import org.apache.flink.runtime.messages.RequiresLeaderSessionID;
import org.apache.flink.util.Preconditions;
-import java.util.UUID;
+import java.io.Serializable;
/**
* This message signals that the ResourceManager should reconnect to the
JobManager. It is processed
@@ -30,28 +30,30 @@ import java.util.UUID;
* the ResourceManager to go through the reconciliation phase to sync up with
the JobManager bookkeeping.
* This is done by forcing the ResourceManager to reconnect.
*/
-public class ReconnectResourceManager implements RequiresLeaderSessionID,
java.io.Serializable {
+public class ReconnectResourceManager implements RequiresLeaderSessionID,
Serializable {
private static final long serialVersionUID = 1L;
private final ActorRef resourceManager;
- private final UUID currentConnID;
+ private final long connectionId;
- public ReconnectResourceManager(ActorRef resourceManager, UUID
currentConnID) {
+ public ReconnectResourceManager(ActorRef resourceManager, long
connectionId) {
this.resourceManager =
Preconditions.checkNotNull(resourceManager);
- this.currentConnID = Preconditions.checkNotNull(currentConnID);
+ this.connectionId = Preconditions.checkNotNull(connectionId);
}
public ActorRef resourceManager() {
return resourceManager;
}
- public UUID connID() {
- return currentConnID;
+ public long getConnectionId() {
+ return connectionId;
}
@Override
public String toString() {
- return "ReconnectResourceManager " + resourceManager.path();
+ return "ReconnectResourceManager(" +
+
[27/50] [abbrv] flink git commit: [FLINK-6382] [gelly] Support additional types for generated graphs in Gelly examples
http://git-wip-us.apache.org/repos/asf/flink/blob/33695781/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/GraphMetricsITCase.java
--
diff --git
a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/GraphMetricsITCase.java
b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/GraphMetricsITCase.java
index a5ea486..8c5ed86 100644
---
a/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/GraphMetricsITCase.java
+++
b/flink-libraries/flink-gelly-examples/src/test/java/org/apache/flink/graph/drivers/GraphMetricsITCase.java
@@ -18,8 +18,8 @@
package org.apache.flink.graph.drivers;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.flink.client.program.ProgramParametrizationException;
+import org.junit.Assume;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -28,8 +28,15 @@ import org.junit.runners.Parameterized;
public class GraphMetricsITCase
extends DriverBaseITCase {
- public GraphMetricsITCase(TestExecutionMode mode) {
- super(mode);
+ public GraphMetricsITCase(String idType, TestExecutionMode mode) {
+ super(idType, mode);
+ }
+
+ private String[] parameters(int scale, String order, String output) {
+ return new String[] {
+ "--algorithm", "GraphMetrics", "--order", order,
+ "--input", "RMatGraph", "--scale",
Integer.toString(scale), "--type", idType, "--simplify", order,
+ "--output", output};
}
@Test
@@ -43,58 +50,110 @@ extends DriverBaseITCase {
}
@Test
- public void testWithDirectedRMatIntegerGraph() throws Exception {
+ public void testWithSmallDirectedRMatIntegerGraph() throws Exception {
String expected = "\n" +
"Vertex metrics:\n" +
- " vertex count: 902\n" +
- " edge count: 12,009\n" +
- " unidirectional edge count: 8,875\n" +
- " bidirectional edge count: 1,567\n" +
- " average degree: 13.314\n" +
- " density: 0.01477663\n" +
- " triplet count: 1,003,442\n" +
- " maximum degree: 463\n" +
- " maximum out degree: 334\n" +
- " maximum in degree: 342\n" +
- " maximum triplets: 106,953\n" +
+ " vertex count: 117\n" +
+ " edge count: 1,168\n" +
+ " unidirectional edge count: 686\n" +
+ " bidirectional edge count: 241\n" +
+ " average degree: 9.983\n" +
+ " density: 0.08605953\n" +
+ " triplet count: 29,286\n" +
+ " maximum degree: 91\n" +
+ " maximum out degree: 77\n" +
+ " maximum in degree: 68\n" +
+ " maximum triplets: 4,095\n" +
"\n" +
"Edge metrics:\n" +
- " triangle triplet count: 107,817\n" +
- " rectangle triplet count: 315,537\n" +
- " maximum triangle triplets: 820\n" +
- " maximum rectangle triplets: 3,822\n";
+ " triangle triplet count: 4,575\n" +
+ " rectangle triplet count: 11,756\n" +
+ " maximum triangle triplets: 153\n" +
+ " maximum rectangle triplets: 391\n";
+
+ expectedOutput(parameters(7, "directed", "hash"), expected);
+ expectedOutput(parameters(7, "directed", "print"), expected);
+ }
+
+ @Test
+ public void testWithLargeDirectedRMatIntegerGraph() throws Exception {
+ // skip 'byte' which cannot store vertex IDs for scale > 8
+ Assume.assumeFalse(idType.equals("byte") ||
idType.equals("nativeByte"));
+
+ // skip 'string' which does not compare numerically and
generates a different triangle count
+ Assume.assumeFalse(idType.equals("string") ||
idType.equals("nativeString"));
- String[] arguments = new String[]{"--algorithm",
"GraphMetrics", "--order", "directed",
- "--input", "RMatGraph", "--type", "integer",
"--simplify", "directed",
- "--output"};
+ String expected = "\n" +
+ "Vertex metrics:\n" +
+ " vertex count: 3,349\n" +
+ " edge count: 53,368\n" +
+ " unidirectional edge count: 43,602\n" +
+
[01/50] [abbrv] flink git commit: [FLINK-6208] [cep] Implement skip till next match strategy [Forced Update!]
Repository: flink
Updated Branches:
refs/heads/table-retraction b237a3ef0 -> e265620d4 (forced update)
http://git-wip-us.apache.org/repos/asf/flink/blob/7c35dc0e/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
--
diff --git
a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
index f4bf647..4a00c1e 100644
---
a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
+++
b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
@@ -26,6 +26,7 @@ import org.apache.flink.cep.Event;
import org.apache.flink.cep.SubEvent;
import org.apache.flink.cep.nfa.compiler.NFACompiler;
import org.apache.flink.cep.pattern.Pattern;
+import org.apache.flink.cep.pattern.Quantifier;
import org.apache.flink.cep.pattern.conditions.IterativeCondition;
import org.apache.flink.cep.pattern.conditions.SimpleCondition;
import org.apache.flink.streaming.api.windowing.time.Time;
@@ -46,6 +47,7 @@ import java.util.Set;
import static org.junit.Assert.assertEquals;
+@SuppressWarnings("unchecked")
public class NFAITCase extends TestLogger {
@Test
@@ -56,12 +58,12 @@ public class NFAITCase extends TestLogger {
SubEvent middleEvent = new SubEvent(42, "foo", 1.0, 10.0);
Event endEvent = new Event(43, "end", 1.0);
- inputEvents.add(new StreamRecord(startEvent, 1));
- inputEvents.add(new StreamRecord(new Event(43, "foobar",
1.0), 2));
+ inputEvents.add(new StreamRecord<>(startEvent, 1));
+ inputEvents.add(new StreamRecord<>(new Event(43, "foobar",
1.0), 2));
inputEvents.add(new StreamRecord(new SubEvent(41,
"barfoo", 1.0, 5.0), 3));
inputEvents.add(new StreamRecord(middleEvent, 3));
- inputEvents.add(new StreamRecord(new Event(43, "start",
1.0), 4));
- inputEvents.add(new StreamRecord(endEvent, 5));
+ inputEvents.add(new StreamRecord<>(new Event(43, "start", 1.0),
4));
+ inputEvents.add(new StreamRecord<>(endEvent, 5));
Pattern pattern =
Pattern.begin("start").where(new SimpleCondition() {
private static final long serialVersionUID =
5726188262756267490L;
@@ -212,12 +214,12 @@ public class NFAITCase extends TestLogger {
final Event middleEvent;
final Event endEvent;
- events.add(new StreamRecord(new Event(1, "start", 1.0),
1));
- events.add(new StreamRecord(startEvent = new Event(2,
"start", 1.0), 2));
- events.add(new StreamRecord(middleEvent = new Event(3,
"middle", 1.0), 3));
- events.add(new StreamRecord(new Event(4, "foobar", 1.0),
4));
- events.add(new StreamRecord(endEvent = new Event(5,
"end", 1.0), 11));
- events.add(new StreamRecord(new Event(6, "end", 1.0),
13));
+ events.add(new StreamRecord<>(new Event(1, "start", 1.0), 1));
+ events.add(new StreamRecord<>(startEvent = new Event(2,
"start", 1.0), 2));
+ events.add(new StreamRecord<>(middleEvent = new Event(3,
"middle", 1.0), 3));
+ events.add(new StreamRecord<>(new Event(4, "foobar", 1.0), 4));
+ events.add(new StreamRecord<>(endEvent = new Event(5, "end",
1.0), 11));
+ events.add(new StreamRecord<>(new Event(6, "end", 1.0), 13));
Pattern pattern =
Pattern.begin("start").where(new SimpleCondition() {
@@ -273,12 +275,12 @@ public class NFAITCase extends TestLogger {
Set, Long>> resultingTimeoutPatterns
= new HashSet<>();
Set, Long>> expectedTimeoutPatterns =
new HashSet<>();
- events.add(new StreamRecord(new Event(1, "start", 1.0),
1));
- events.add(new StreamRecord(new Event(2, "start", 1.0),
2));
- events.add(new StreamRecord(new Event(3, "middle", 1.0),
3));
- events.add(new StreamRecord(new Event(4, "foobar", 1.0),
4));
- events.add(new StreamRecord(new Event(5, "end", 1.0),
11));
- events.add(new StreamRecord(new Event(6, "end", 1.0),
13));
+ events.add(new StreamRecord<>(new Event(1, "start", 1.0), 1));
+ events.add(new StreamRecord<>(new Event(2, "start", 1.0), 2));
+ events.add(new StreamRecord<>(new Event(3, "middle", 1.0), 3));
+ events.add(new StreamRecord<>(new Event(4, "foobar", 1.0), 4));
+ events.add(new StreamRecord<>(new Event(5, "end", 1.0), 11));
+ events.add(new StreamRecord<>(new Event(6, "end", 1.0), 13));
Map timeoutPattern1 = new HashMap<>();
timeoutPattern1.put("start", new Event(1, "st
[47/50] [abbrv] flink git commit: [FLINK-6091] [table] Implement and turn on retraction for non-windowed aggregates.
[FLINK-6091] [table] Implement and turn on retraction for non-windowed aggregates. Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/856485be Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/856485be Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/856485be Branch: refs/heads/table-retraction Commit: 856485be3be3bb240a068019fe21da3556f8b26f Parents: 455a3c5 Author: Hequn Cheng Authored: Tue Apr 18 16:54:09 2017 +0800 Committer: Fabian Hueske Committed: Wed May 3 11:33:07 2017 +0200 -- .../flink/table/api/BatchTableEnvironment.scala | 63 +++- .../table/api/StreamTableEnvironment.scala | 62 +++- .../flink/table/api/TableEnvironment.scala | 65 ++-- .../flink/table/plan/nodes/CommonCalc.scala | 26 +- .../table/plan/nodes/CommonCorrelate.scala | 58 +--- .../flink/table/plan/nodes/CommonScan.scala | 24 +- .../table/plan/nodes/dataset/BatchScan.scala| 10 +- .../table/plan/nodes/dataset/DataSetCalc.scala | 23 +- .../plan/nodes/dataset/DataSetCorrelate.scala | 32 +- .../plan/nodes/dataset/DataSetValues.scala | 2 +- .../plan/nodes/datastream/DataStreamCalc.scala | 30 +- .../nodes/datastream/DataStreamCorrelate.scala | 38 ++- .../datastream/DataStreamGroupAggregate.scala | 24 +- .../DataStreamGroupWindowAggregate.scala| 47 ++- .../datastream/DataStreamOverAggregate.scala| 62 ++-- .../plan/nodes/datastream/DataStreamRel.scala | 6 +- .../plan/nodes/datastream/DataStreamScan.scala | 3 +- .../plan/nodes/datastream/DataStreamUnion.scala | 3 +- .../nodes/datastream/DataStreamValues.scala | 14 +- .../plan/nodes/datastream/StreamScan.scala | 23 +- .../datastream/StreamTableSourceScan.scala | 4 +- .../plan/nodes/logical/FlinkLogicalCalc.scala | 2 +- .../datastream/DataStreamRetractionRules.scala | 16 +- .../runtime/CRowCorrelateFlatMapRunner.scala| 83 + .../flink/table/runtime/CRowFlatMapRunner.scala | 72 .../table/runtime/CRowInputMapRunner.scala | 57 .../table/runtime/CRowOutputMapRunner.scala | 60 .../table/runtime/CRowWrappingCollector.scala | 41 +++ .../flink/table/runtime/FlatMapRunner.scala | 17 +- .../aggregate/AggregateAggFunction.scala| 15 +- .../table/runtime/aggregate/AggregateUtil.scala | 47 +-- ...SetSessionWindowAggReduceGroupFunction.scala | 4 +- ...taSetSlideWindowAggReduceGroupFunction.scala | 4 +- ...TumbleTimeWindowAggReduceGroupFunction.scala | 4 +- .../aggregate/GroupAggProcessFunction.scala | 58 +++- ...rementalAggregateAllTimeWindowFunction.scala | 7 +- .../IncrementalAggregateAllWindowFunction.scala | 11 +- ...IncrementalAggregateTimeWindowFunction.scala | 7 +- .../IncrementalAggregateWindowFunction.scala| 13 +- .../aggregate/ProcTimeBoundedRangeOver.scala| 30 +- .../aggregate/ProcTimeBoundedRowsOver.scala | 26 +- .../ProcTimeUnboundedNonPartitionedOver.scala | 23 +- .../ProcTimeUnboundedPartitionedOver.scala | 22 +- .../aggregate/RowTimeBoundedRangeOver.scala | 32 +- .../aggregate/RowTimeBoundedRowsOver.scala | 30 +- .../aggregate/RowTimeUnboundedOver.scala| 50 +-- .../aggregate/TimeWindowPropertyCollector.scala | 34 +- .../runtime/io/CRowValuesInputFormat.scala | 59 .../table/runtime/io/ValuesInputFormat.scala| 17 +- .../apache/flink/table/runtime/types/CRow.scala | 55 +++ .../table/runtime/types/CRowComparator.scala| 83 + .../table/runtime/types/CRowSerializer.scala| 78 + .../table/runtime/types/CRowTypeInfo.scala | 98 ++ .../apache/flink/table/sinks/CsvTableSink.scala | 2 + .../flink/table/TableEnvironmentTest.scala | 72 +++- .../scala/batch/TableEnvironmentITCase.scala| 20 ++ .../api/scala/stream/RetractionITCase.scala | 331 +++ .../api/scala/stream/TableSinkITCase.scala | 2 +- .../api/scala/stream/utils/StreamITCase.scala | 11 +- ...ProcessingOverRangeProcessFunctionTest.scala | 105 +++--- .../runtime/types/CRowComparatorTest.scala | 61 .../table/utils/MockTableEnvironment.scala | 8 + 62 files changed, 1896 insertions(+), 490 deletions(-) -- http://git-wip-us.apache.org/repos/asf/flink/blob/856485be/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala -- diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala index 00cf11c..34f5018 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvir
[44/50] [abbrv] flink git commit: [FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
[FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
This closes #3646.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/24fa1a1c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/24fa1a1c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/24fa1a1c
Branch: refs/heads/table-retraction
Commit: 24fa1a1c2516726a5ecd446bc79b9fb6664bec38
Parents: 6181302
Author: shaoxuan-wang
Authored: Thu Mar 30 03:57:58 2017 +0800
Committer: Fabian Hueske
Committed: Wed May 3 11:27:12 2017 +0200
--
.../flink/table/plan/logical/operators.scala| 3 -
.../nodes/datastream/DataStreamAggregate.scala | 272 ---
.../datastream/DataStreamGroupAggregate.scala | 140 ++
.../DataStreamGroupWindowAggregate.scala| 272 +++
.../flink/table/plan/rules/FlinkRuleSets.scala | 3 +-
.../datastream/DataStreamAggregateRule.scala| 76 --
.../DataStreamGroupAggregateRule.scala | 79 ++
.../DataStreamGroupWindowAggregateRule.scala| 76 ++
.../table/runtime/aggregate/AggregateUtil.scala | 63 -
.../aggregate/GroupAggProcessFunction.scala | 90 ++
.../aggregate/ProcTimeBoundedRangeOver.scala| 2 +-
.../scala/batch/table/FieldProjectionTest.scala | 4 +-
.../table/api/scala/stream/sql/SqlITCase.scala | 21 ++
.../scala/stream/sql/WindowAggregateTest.scala | 46 ++--
.../scala/stream/table/AggregationsITCase.scala | 180
.../stream/table/GroupAggregationsITCase.scala | 132 +
.../stream/table/GroupAggregationsTest.scala| 214 +++
.../table/GroupWindowAggregationsITCase.scala | 180
.../scala/stream/table/GroupWindowTest.scala| 56 ++--
.../scala/stream/table/UnsupportedOpsTest.scala | 7 -
20 files changed, 1318 insertions(+), 598 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/24fa1a1c/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index f1bb644..66b26ed 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -221,9 +221,6 @@ case class Aggregate(
}
override def validate(tableEnv: TableEnvironment): LogicalNode = {
-if (tableEnv.isInstanceOf[StreamTableEnvironment]) {
- failValidation(s"Aggregate on stream tables is currently not supported.")
-}
val resolvedAggregate = super.validate(tableEnv).asInstanceOf[Aggregate]
val groupingExprs = resolvedAggregate.groupingExpressions
http://git-wip-us.apache.org/repos/asf/flink/blob/24fa1a1c/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
deleted file mode 100644
index 187773d..000
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.plan.nodes.datastream
-
-import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
-import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.core.AggregateCall
-import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
-import org.apache.flink.api.java.tuple.Tuple
-import org.apache.flink.streami
[33/50] [abbrv] flink git commit: [FLINK-6337] [network] Fix instability in SuccessAfterNetworkBuffersFailureITCase
[FLINK-6337] [network] Fix instability in
SuccessAfterNetworkBuffersFailureITCase
The reduced number of network buffers as part of this issue was too low and
lead to instable tests.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/aa8f81cc
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/aa8f81cc
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/aa8f81cc
Branch: refs/heads/table-retraction
Commit: aa8f81ccdf6a91cd95e29ce501fd7b454d516264
Parents: 2d33c0b
Author: Ufuk Celebi
Authored: Tue May 2 15:43:32 2017 +0200
Committer: Ufuk Celebi
Committed: Tue May 2 15:44:04 2017 +0200
--
.../flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/aa8f81cc/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
--
diff --git
a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
index 710e7df..4f24452 100644
---
a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
+++
b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java
@@ -42,7 +42,6 @@ import static org.junit.Assert.*;
public class SuccessAfterNetworkBuffersFailureITCase extends TestLogger {
-
@Test
public void testSuccessfulProgramAfterFailure() {
LocalFlinkMiniCluster cluster = null;
@@ -52,7 +51,7 @@ public class SuccessAfterNetworkBuffersFailureITCase extends
TestLogger {
config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE,
80L);
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
-
config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 640);
+
config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 800);
cluster = new LocalFlinkMiniCluster(config, false);
[32/50] [abbrv] flink git commit: [FLINK-6409] [table] TUMBLE/HOP/SESSION_START/END do not resolve time field correctly
[FLINK-6409] [table] TUMBLE/HOP/SESSION_START/END do not resolve time field
correctly
This closes #3799.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2d33c0be
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2d33c0be
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2d33c0be
Branch: refs/heads/table-retraction
Commit: 2d33c0bead3f13417acff0f18f78a5c2c5bef22e
Parents: 464d6f5
Author: twalthr
Authored: Fri Apr 28 17:17:30 2017 +0200
Committer: twalthr
Committed: Tue May 2 15:31:35 2017 +0200
--
.../calcite/sql/fun/SqlGroupFunction.java | 103 +
.../calcite/sql/fun/SqlStdOperatorTable.java| 2133 +++
.../apache/calcite/sql/validate/AggChecker.java | 225 +
.../calcite/sql2rel/AuxiliaryConverter.java | 79 +
.../calcite/sql2rel/SqlToRelConverter.java | 5356 ++
.../scala/batch/sql/WindowAggregateTest.scala | 31 +
tools/maven/suppressions.xml|2 +
7 files changed, 7929 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/2d33c0be/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlGroupFunction.java
--
diff --git
a/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlGroupFunction.java
b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlGroupFunction.java
new file mode 100644
index 000..a57cf10
--- /dev/null
+++
b/flink-libraries/flink-table/src/main/java/org/apache/calcite/sql/fun/SqlGroupFunction.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql.fun;
+
+/*
+ * THIS FILE HAS BEEN COPIED FROM THE APACHE CALCITE PROJECT UNTIL
CALCITE-1761 IS FIXED.
+ */
+
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlOperandTypeChecker;
+import org.apache.calcite.sql.validate.SqlMonotonicity;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * SQL function that computes keys by which rows can be partitioned and
+ * aggregated.
+ *
+ * Grouped window functions always occur in the GROUP BY clause. They often
+ * have auxiliary functions that access information about the group. For
+ * example, {@code HOP} is a group function, and its auxiliary functions are
+ * {@code HOP_START} and {@code HOP_END}. Here they are used in a streaming
+ * query:
+ *
+ *
+ * SELECT STREAM HOP_START(rowtime, INTERVAL '1' HOUR),
+ * HOP_END(rowtime, INTERVAL '1' HOUR),
+ * MIN(unitPrice)
+ * FROM Orders
+ * GROUP BY HOP(rowtime, INTERVAL '1' HOUR), productId
+ *
+ */
+class SqlGroupFunction extends SqlFunction {
+ /** The grouped function, if this an auxiliary function; null
otherwise. */
+ final SqlGroupFunction groupFunction;
+
+ /** Creates a SqlGroupFunction.
+*
+* @param kind Kind; also determines function name
+* @param groupFunction Group function, if this is an auxiliary;
+* null, if this is a group function
+* @param operandTypeChecker Operand type checker
+*/
+ SqlGroupFunction(SqlKind kind, SqlGroupFunction groupFunction,
+ SqlOperandTypeChecker operandTypeChecker) {
+ super(kind.name(), kind, ReturnTypes.ARG0, null,
+ operandTypeChecker, SqlFunctionCategory.SYSTEM);
+ this.groupFunction = groupFunction;
+ if (groupFunction != null) {
+ assert groupFunction.groupFunction == null;
+ }
+ }
+
+ /** Creates an auxiliary function from this grouped window function. */
+ SqlGroupFunction auxiliary(SqlKind kind) {
+ return new SqlGroupFunction(kind, this,
getOperandTypeChecker());
+ }
+
+
[28/50] [abbrv] flink git commit: [FLINK-6382] [gelly] Support additional types for generated graphs in Gelly examples
[FLINK-6382] [gelly] Support additional types for generated graphs in Gelly
examples
The Gelly examples current support IntValue, LongValue, and StringValue
for RMatGraph. Allow transformations and tests for all generated graphs
for ByteValue, Byte, ShortValue, Short, CharValue, Character, Integer,
Long, and String.
This closes #3779
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/33695781
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/33695781
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/33695781
Branch: refs/heads/table-retraction
Commit: 33695781f9ce2599d18f45de6a465eaefe7d71f4
Parents: d49efbd
Author: Greg Hogan
Authored: Tue Apr 25 11:36:08 2017 -0400
Committer: Greg Hogan
Committed: Mon May 1 10:35:10 2017 -0400
--
.../flink/graph/drivers/GraphMetrics.java | 3 +-
.../flink/graph/drivers/JaccardIndex.java | 4 +
.../graph/drivers/input/CompleteGraph.java | 13 +-
.../flink/graph/drivers/input/CycleGraph.java | 13 +-
.../flink/graph/drivers/input/EmptyGraph.java | 13 +-
.../graph/drivers/input/GeneratedGraph.java | 372 +
.../drivers/input/GeneratedMultiGraph.java | 59 +++
.../flink/graph/drivers/input/GridGraph.java| 33 +-
.../graph/drivers/input/HypercubeGraph.java | 16 +-
.../flink/graph/drivers/input/PathGraph.java| 13 +-
.../flink/graph/drivers/input/RMatGraph.java| 80 +--
.../graph/drivers/input/SingletonEdgeGraph.java | 16 +-
.../flink/graph/drivers/input/StarGraph.java| 13 +-
.../drivers/parameter/ChoiceParameter.java | 23 +-
.../org/apache/flink/graph/RunnerITCase.java| 4 +-
.../flink/graph/drivers/AdamicAdarITCase.java | 28 +-
.../drivers/ClusteringCoefficientITCase.java| 179 +--
.../drivers/ConnectedComponentsITCase.java | 123 -
.../drivers/CopyableValueDriverBaseITCase.java | 52 ++
.../flink/graph/drivers/DriverBaseITCase.java | 74 ++-
.../flink/graph/drivers/EdgeListITCase.java | 522 ++-
.../flink/graph/drivers/GraphMetricsITCase.java | 141 +++--
.../apache/flink/graph/drivers/HITSITCase.java | 34 +-
.../flink/graph/drivers/JaccardIndexITCase.java | 91 +++-
.../flink/graph/drivers/PageRankITCase.java | 34 +-
.../graph/drivers/TriangleListingITCase.java| 276 --
.../graph/drivers/input/GeneratedGraphTest.java | 203
.../translators/LongValueToSignedIntValue.java | 2 +
.../LongValueToUnsignedIntValue.java| 6 +-
.../flink/graph/generator/CompleteGraph.java| 7 +-
.../apache/flink/graph/generator/GridGraph.java | 2 +-
.../apache/flink/graph/generator/RMatGraph.java | 2 +-
.../apache/flink/graph/generator/StarGraph.java | 2 +-
.../flink/graph/library/link_analysis/HITS.java | 2 -
.../graph/library/similarity/JaccardIndex.java | 50 +-
.../org/apache/flink/graph/asm/AsmTestBase.java | 2 +-
.../LongValueToSignedIntValueTest.java | 4 +-
.../LongValueToUnsignedIntValueTest.java| 6 +-
38 files changed, 2054 insertions(+), 463 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/33695781/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/GraphMetrics.java
--
diff --git
a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/GraphMetrics.java
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/GraphMetrics.java
index cc5a894..aef8f9f 100644
---
a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/GraphMetrics.java
+++
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/GraphMetrics.java
@@ -26,7 +26,6 @@ import org.apache.flink.graph.drivers.output.Hash;
import org.apache.flink.graph.drivers.output.Print;
import org.apache.flink.graph.drivers.parameter.ChoiceParameter;
import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
-import org.apache.flink.types.CopyableValue;
/**
* Driver for directed and undirected graph metrics analytics.
@@ -36,7 +35,7 @@ import org.apache.flink.types.CopyableValue;
* @see org.apache.flink.graph.library.metric.undirected.EdgeMetrics
* @see org.apache.flink.graph.library.metric.undirected.VertexMetrics
*/
-public class GraphMetrics & CopyableValue, VV, EV>
+public class GraphMetrics, VV, EV>
extends ParameterizedBase
implements Driver, Hash, Print {
http://git-wip-us.apache.org/repos/asf/flink/blob/33695781/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/JaccardIndex.java
--
diff --git
a/flink-libraries/flink-gelly-examples/src/
[36/50] [abbrv] flink git commit: [hotfix] [tests] Fix test log level in flink-runtime
[hotfix] [tests] Fix test log level in flink-runtime Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9cb76213 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9cb76213 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9cb76213 Branch: refs/heads/table-retraction Commit: 9cb7621399ab7b97663543f35eaf3f6c88fec137 Parents: aa8f81c Author: Stephan Ewen Authored: Tue May 2 22:48:47 2017 +0200 Committer: Stephan Ewen Committed: Tue May 2 22:49:45 2017 +0200 -- flink-runtime/src/test/resources/log4j-test.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/flink/blob/9cb76213/flink-runtime/src/test/resources/log4j-test.properties -- diff --git a/flink-runtime/src/test/resources/log4j-test.properties b/flink-runtime/src/test/resources/log4j-test.properties index 812a256..7ba1633 100644 --- a/flink-runtime/src/test/resources/log4j-test.properties +++ b/flink-runtime/src/test/resources/log4j-test.properties @@ -16,7 +16,7 @@ # limitations under the License. -log4j.rootLogger=DEBUG, console +log4j.rootLogger=OFF, console # - # Console (use 'console')
[24/50] [abbrv] flink git commit: [FLINK-6112] [table] Improve documentation, consistency, and fix bugs
[FLINK-6112] [table] Improve documentation, consistency, and fix bugs
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0a334312
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0a334312
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0a334312
Branch: refs/heads/table-retraction
Commit: 0a3343128be3c0f28772f803f82d14c23b084221
Parents: a06b322
Author: twalthr
Authored: Sun Apr 30 15:03:33 2017 +0200
Committer: twalthr
Committed: Sun Apr 30 15:12:46 2017 +0200
--
docs/dev/table_api.md | 297 ++-
.../flink/table/api/scala/expressionDsl.scala | 16 +-
.../table/codegen/calls/ConstantCallGen.scala | 2 +-
.../table/codegen/calls/FunctionGenerator.scala | 66 ++---
.../table/expressions/mathExpressions.scala | 22 +-
.../flink/table/validate/FunctionCatalog.scala | 4 +-
.../table/expressions/ScalarFunctionsTest.scala | 48 ++-
.../table/expressions/SqlExpressionTest.scala | 3 +-
8 files changed, 249 insertions(+), 209 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/0a334312/docs/dev/table_api.md
--
diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 6d52a78..022a73d 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1977,303 +1977,303 @@ BOOLEAN.isNotFalse
+
+
+
+
+
+
+ Arithmetic functions
+ Description
+
+
+
+
+
+
{% highlight java %}
-NUMERIC.sin()
++ numeric
{% endhighlight %}
-Calculates the sine of a given number.
+Returns numeric.
{% highlight java %}
-NUMERIC.cos()
+- numeric
{% endhighlight %}
-Calculates the cosine of a given number.
+Returns negative numeric.
-
+
{% highlight java %}
-NUMERIC.tan()
+numeric1 + numeric2
{% endhighlight %}
-Calculates the tangent of a given number.
+Returns numeric1 plus numeric2.
{% highlight java %}
-NUMERIC.cot()
+numeric1 - numeric2
{% endhighlight %}
-Calculates the tangent of a given number.
+Returns numeric1 minus numeric2.
{% highlight java %}
-NUMERIC.asin()
+numeric1 * numeric2
{% endhighlight %}
-Calculates the tangent of a given number.
+Returns numeric1 multiplied by numeric2.
{% highlight java %}
-NUMERIC.acos()
+numeric1 / numeric2
{% endhighlight %}
-Calculates the tangent of a given number.
+Returns numeric1 divided by numeric2.
{% highlight java %}
-NUMERIC.atan()
+numeric1.power(numeric2)
{% endhighlight %}
-Calculates the tangent of a given number.
+Returns numeric1 raised to the power of numeric2.
{% highlight java %}
-NUMERIC.degrees()
+NUMERIC.abs()
{% endhighlight %}
-Converts numeric from radians to degrees.
+Calculates the absolute value of given value.
{% highlight java %}
-NUMERIC.radians()
+numeric1 % numeric2
{% endhighlight %}
-Converts numeric from degrees to radians.
+Returns the remainder (modulus) of numeric1 divided by
numeric2. The result is negative only if numeric1 is negative.
{% highlight java %}
-NUMERIC.sign()
+NUMERIC.sqrt()
{% endhighlight %}
-Calculates the signum of a given number.
+Calculates the square root of a given value.
{% highlight java %}
-NUMERIC.round(numeric2)
+NUMERIC.ln()
{% endhighlight %}
-Rounds the given number to the other value places right to the
decimal point.
+Calculates the natural logarithm of given value.
-
-
-
-
-
-
- Arithmetic functions
- Description
-
-
-
-
-
-
{% highlight java %}
-+ numeric
+NUMERIC.log10()
{% endhighlight %}
-Returns numeric.
+Calculates the base 10 logarithm of given value.
{% highlight java %}
-- numeric
+NUMERIC.exp()
{% endhighlight %}
-Returns negative numeric.
+Calculates the Euler's number raised to the given power.
-
+
{% highlight java %}
-numeric1 + numeric2
+NUMERIC.ceil()
{% endhighlight %}
-Returns numeric1 plus numeric2.
+
[17/50] [abbrv] flink git commit: [Flink-5892] Restore state on operator level
http://git-wip-us.apache.org/repos/asf/flink/blob/f7980a7e/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
deleted file mode 100644
index 83c188c..000
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationV2.java
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.runtime.checkpoint;
-
-//import com.google.common.collect.Lists;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.runtime.executiongraph.Execution;
-import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.state.ChainedStateHandle;
-import org.apache.flink.runtime.state.KeyGroupRange;
-import org.apache.flink.runtime.state.KeyGroupsStateHandle;
-import org.apache.flink.runtime.state.KeyedStateHandle;
-import org.apache.flink.runtime.state.OperatorStateHandle;
-import org.apache.flink.runtime.state.StreamStateHandle;
-import org.apache.flink.runtime.state.TaskStateHandles;
-import org.apache.flink.util.Preconditions;
-import org.slf4j.Logger;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * This class encapsulates the operation of assigning restored state when
restoring from a checkpoint that works on the
- * granularity of operators. This is the case for checkpoints that were
created either with a Flink version >= 1.3 or
- * 1.2 if the savepoint only contains {@link SubtaskState}s for which the
length of contained
- * {@link ChainedStateHandle}s is equal to 1.
- */
-public class StateAssignmentOperationV2 {
-
- private final Logger logger;
- private final Map tasks;
- private final Map taskStates;
- private final boolean allowNonRestoredState;
-
- public StateAssignmentOperationV2(
- Logger logger,
- Map tasks,
- Map taskStates,
- boolean allowNonRestoredState) {
-
- this.logger = Preconditions.checkNotNull(logger);
- this.tasks = Preconditions.checkNotNull(tasks);
- this.taskStates = Preconditions.checkNotNull(taskStates);
- this.allowNonRestoredState = allowNonRestoredState;
- }
-
- public boolean assignStates() throws Exception {
- Map localStates = new
HashMap<>(taskStates);
- Map localTasks = this.tasks;
-
- Set allOperatorIDs = new HashSet<>();
- for (ExecutionJobVertex executionJobVertex : tasks.values()) {
-
//allOperatorIDs.addAll(Lists.newArrayList(executionJobVertex.getOperatorIDs()));
- }
- for (Map.Entry taskGroupStateEntry :
taskStates.entrySet()) {
- TaskState taskState = taskGroupStateEntry.getValue();
- //find operator
for state-
-
- if
(!allOperatorIDs.contains(taskGroupStateEntry.getKey())) {
- if (allowNonRestoredState) {
- logger.info("Skipped checkpoint state
for operator {}.", taskState.getJobVertexID());
- continue;
- } else {
- throw new IllegalStateException("There
is no operator for the state " + taskState.getJobVertexID());
- }
- }
- }
-
- for (Map.Entry task :
localTasks.entrySet()) {
- final ExecutionJobVertex executionJobVertex =
[37/50] [abbrv] flink git commit: [hotfix] [security] Reduce logging verbosity for SSLUtils
[hotfix] [security] Reduce logging verbosity for SSLUtils
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6f93352b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6f93352b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6f93352b
Branch: refs/heads/table-retraction
Commit: 6f93352b78ee019388b8e8a4684730d61549f786
Parents: bad7e0b
Author: Stephan Ewen
Authored: Sat Apr 29 17:40:30 2017 +0200
Committer: Stephan Ewen
Committed: Tue May 2 22:49:46 2017 +0200
--
.../org/apache/flink/runtime/net/SSLUtils.java | 26 +---
1 file changed, 17 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/6f93352b/flink-runtime/src/main/java/org/apache/flink/runtime/net/SSLUtils.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/net/SSLUtils.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/net/SSLUtils.java
index 5bafeb8..2267eac 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/net/SSLUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/net/SSLUtils.java
@@ -18,10 +18,10 @@
package org.apache.flink.runtime.net;
-
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.util.Preconditions;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,6 +35,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.net.ServerSocket;
import java.security.KeyStore;
+import java.util.Arrays;
/**
* Common utilities to manage SSL transport settings
@@ -66,14 +67,21 @@ public class SSLUtils {
*/
public static void setSSLVerAndCipherSuites(ServerSocket socket,
Configuration config) {
if (socket instanceof SSLServerSocket) {
- ((SSLServerSocket)
socket).setEnabledProtocols(config.getString(
- ConfigConstants.SECURITY_SSL_PROTOCOL,
-
ConfigConstants.DEFAULT_SECURITY_SSL_PROTOCOL).split(","));
- ((SSLServerSocket)
socket).setEnabledCipherSuites(config.getString(
- ConfigConstants.SECURITY_SSL_ALGORITHMS,
-
ConfigConstants.DEFAULT_SECURITY_SSL_ALGORITHMS).split(","));
- } else {
- LOG.warn("Not a SSL socket, will skip setting tls
version and cipher suites.");
+ final String[] protocols = config.getString(
+ ConfigConstants.SECURITY_SSL_PROTOCOL,
+
ConfigConstants.DEFAULT_SECURITY_SSL_PROTOCOL).split(",");
+
+ final String[] cipherSuites = config.getString(
+ ConfigConstants.SECURITY_SSL_ALGORITHMS,
+
ConfigConstants.DEFAULT_SECURITY_SSL_ALGORITHMS).split(",");
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Configuring TLS version and cipher
suites on SSL socket {} / {}",
+ Arrays.toString(protocols),
Arrays.toString(cipherSuites));
+ }
+
+ ((SSLServerSocket)
socket).setEnabledProtocols(protocols);
+ ((SSLServerSocket)
socket).setEnabledCipherSuites(cipherSuites);
}
}
[12/50] [abbrv] flink git commit: [FLINK-6341] [jm] Don't let JM fall into infinite loop
[FLINK-6341] [jm] Don't let JM fall into infinite loop
This closes #3745.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/23838392
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/23838392
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/23838392
Branch: refs/heads/table-retraction
Commit: 238383926b762c1d47159a2b4dabe8fd59777307
Parents: c36d6b8
Author: WangTaoTheTonic
Authored: Thu Apr 20 20:28:10 2017 +0800
Committer: Till Rohrmann
Committed: Fri Apr 28 17:59:27 2017 +0200
--
.../messages/ReconnectResourceManager.java | 11 ++-
.../org/apache/flink/runtime/jobmanager/JobManager.scala | 7 +--
2 files changed, 15 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/23838392/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
index 6f6f878..d02193e 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/messages/ReconnectResourceManager.java
@@ -22,6 +22,8 @@ import akka.actor.ActorRef;
import org.apache.flink.runtime.messages.RequiresLeaderSessionID;
import org.apache.flink.util.Preconditions;
+import java.util.UUID;
+
/**
* This message signals that the ResourceManager should reconnect to the
JobManager. It is processed
* by the JobManager if it fails to register resources with the
ResourceManager. The JobManager wants
@@ -33,14 +35,21 @@ public class ReconnectResourceManager implements
RequiresLeaderSessionID, java.i
private final ActorRef resourceManager;
- public ReconnectResourceManager(ActorRef resourceManager) {
+ private final UUID currentConnID;
+
+ public ReconnectResourceManager(ActorRef resourceManager, UUID
currentConnID) {
this.resourceManager =
Preconditions.checkNotNull(resourceManager);
+ this.currentConnID = Preconditions.checkNotNull(currentConnID);
}
public ActorRef resourceManager() {
return resourceManager;
}
+ public UUID connID() {
+ return currentConnID;
+ }
+
@Override
public String toString() {
return "ReconnectResourceManager " + resourceManager.path();
http://git-wip-us.apache.org/repos/asf/flink/blob/23838392/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
--
diff --git
a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 2fc3ef4..da9df2b 100644
---
a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++
b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -178,6 +178,8 @@ class JobManager(
/** The resource manager actor responsible for allocating and managing task
manager resources. */
var currentResourceManager: Option[ActorRef] = None
+ var currentRMConnID: UUID = null
+
val taskManagerMap = mutable.Map[ActorRef, InstanceID]()
/**
@@ -337,6 +339,7 @@ class JobManager(
// ditch current resource manager (if any)
currentResourceManager = Option(msg.resourceManager())
+ currentRMConnID = UUID.randomUUID()
val taskManagerResources =
instanceManager.getAllRegisteredInstances.asScala.map(
instance => instance.getTaskManagerID).toList.asJava
@@ -361,7 +364,7 @@ class JobManager(
}
currentResourceManager match {
-case Some(rm) if rm.equals(msg.resourceManager()) =>
+case Some(rm) if rm.equals(msg.resourceManager()) &&
currentRMConnID.equals(msg.connID()) =>
// we should ditch the current resource manager
log.debug(s"Disconnecting resource manager $rm and forcing a
reconnect.")
currentResourceManager = None
@@ -396,7 +399,7 @@ class JobManager(
case _ =>
log.warn("Failure while asking ResourceManager for
RegisterResource. Retrying", t)
}
- self ! decorateMessage(new ReconnectResourceManager(rm))
+ self ! decorateMessage(new ReconnectResourceManager(rm,
currentRMConnID))
}(context.dispatcher)
case None =>
flink git commit: [FLINK-6059] [table] Reject GenericType when converting DataSet or DataStream to Table.
Repository: flink
Updated Branches:
refs/heads/release-1.2 c0fb70f1c -> fdb3f65f2
[FLINK-6059] [table] Reject GenericType when converting DataSet or
DataStream to Table.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fdb3f65f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fdb3f65f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fdb3f65f
Branch: refs/heads/release-1.2
Commit: fdb3f65f2d6595b88edae849ae6c848e5bbfaa2d
Parents: c0fb70f
Author: Fabian Hueske
Authored: Wed Mar 15 13:24:42 2017 +0100
Committer: Fabian Hueske
Committed: Sat Apr 29 01:40:07 2017 +0200
--
.../flink/table/api/TableEnvironment.scala | 16 +++--
.../api/java/batch/TableEnvironmentITCase.java | 38
.../flink/table/TableEnvironmentTest.scala | 23 +---
.../scala/batch/TableEnvironmentITCase.scala| 34 ++
4 files changed, 104 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/fdb3f65f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 5dc04d1..5cafe4f 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -32,7 +32,7 @@ import org.apache.calcite.sql.util.ChainedSqlOperatorTable
import org.apache.calcite.tools.{FrameworkConfig, Frameworks, RuleSet,
RuleSets}
import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
import org.apache.flink.api.common.typeutils.CompositeType
-import org.apache.flink.api.java.typeutils.{PojoTypeInfo, TupleTypeInfo}
+import org.apache.flink.api.java.typeutils.{GenericTypeInfo, PojoTypeInfo,
TupleTypeInfo}
import org.apache.flink.api.java.{ExecutionEnvironment => JavaBatchExecEnv}
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
import org.apache.flink.api.scala.{ExecutionEnvironment => ScalaBatchExecEnv}
@@ -50,6 +50,7 @@ import org.apache.flink.table.plan.schema.RelTable
import org.apache.flink.table.sinks.TableSink
import org.apache.flink.table.sources.{DefinedFieldNames, TableSource}
import org.apache.flink.table.validate.FunctionCatalog
+import org.apache.flink.types.Row
import _root_.scala.collection.JavaConverters._
@@ -328,7 +329,14 @@ abstract class TableEnvironment(val config: TableConfig) {
*/
protected[flink] def getFieldInfo[A](inputType: TypeInformation[A]):
(Array[String], Array[Int]) = {
-(TableEnvironment.getFieldNames(inputType),
TableEnvironment.getFieldIndices(inputType))
+
+if (inputType.isInstanceOf[GenericTypeInfo[A]] && inputType.getTypeClass
== classOf[Row]) {
+ throw new TableException(
+"An input of GenericTypeInfo cannot be converted to Table. " +
+ "Please specify the type of the input with a RowTypeInfo.")
+} else {
+ (TableEnvironment.getFieldNames(inputType),
TableEnvironment.getFieldIndices(inputType))
+}
}
/**
@@ -347,6 +355,10 @@ abstract class TableEnvironment(val config: TableConfig) {
TableEnvironment.validateType(inputType)
val indexedNames: Array[(Int, String)] = inputType match {
+ case g: GenericTypeInfo[A] if g.getTypeClass == classOf[Row] =>
+throw new TableException(
+ "An input of GenericTypeInfo cannot be converted to Table. " +
+"Please specify the type of the input with a RowTypeInfo.")
case a: AtomicType[A] =>
if (exprs.length != 1) {
throw new TableException("Table of atomic type can only have a
single field.")
http://git-wip-us.apache.org/repos/asf/flink/blob/fdb3f65f/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
--
diff --git
a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
index 67eb2d1..1cc3071 100644
---
a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
+++
b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
@@ -28,6 +28,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.calcite.tools.RuleSets;
import org.apache.flink.api.java.
flink git commit: [FLINK-6059] [table] Reject GenericType when converting DataSet or DataStream to Table.
Repository: flink
Updated Branches:
refs/heads/master 2c68085f6 -> c8eb55f17
[FLINK-6059] [table] Reject GenericType when converting DataSet or
DataStream to Table.
This closes #3546.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c8eb55f1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c8eb55f1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c8eb55f1
Branch: refs/heads/master
Commit: c8eb55f17d64722bb600c1083a478ab99e53f4ec
Parents: 2c68085
Author: Fabian Hueske
Authored: Wed Mar 15 13:24:42 2017 +0100
Committer: Fabian Hueske
Committed: Sat Apr 29 00:44:55 2017 +0200
--
.../flink/table/api/TableEnvironment.scala | 13 +++-
.../api/java/batch/TableEnvironmentITCase.java | 31
.../flink/table/TableEnvironmentTest.scala | 18 ++--
.../scala/batch/TableEnvironmentITCase.scala| 30 +++
4 files changed, 88 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/c8eb55f1/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 16c40fe..bd974b0 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -565,7 +565,14 @@ abstract class TableEnvironment(val config: TableConfig) {
*/
protected[flink] def getFieldInfo[A](inputType: TypeInformation[A]):
(Array[String], Array[Int]) = {
-(TableEnvironment.getFieldNames(inputType),
TableEnvironment.getFieldIndices(inputType))
+
+if (inputType.isInstanceOf[GenericTypeInfo[A]] && inputType.getTypeClass
== classOf[Row]) {
+ throw new TableException(
+"An input of GenericTypeInfo cannot be converted to Table. " +
+ "Please specify the type of the input with a RowTypeInfo.")
+} else {
+ (TableEnvironment.getFieldNames(inputType),
TableEnvironment.getFieldIndices(inputType))
+}
}
/**
@@ -584,6 +591,10 @@ abstract class TableEnvironment(val config: TableConfig) {
TableEnvironment.validateType(inputType)
val indexedNames: Array[(Int, String)] = inputType match {
+ case g: GenericTypeInfo[A] if g.getTypeClass == classOf[Row] =>
+throw new TableException(
+ "An input of GenericTypeInfo cannot be converted to Table. " +
+"Please specify the type of the input with a RowTypeInfo.")
case a: AtomicType[A] =>
if (exprs.length != 1) {
throw new TableException("Table of atomic type can only have a
single field.")
http://git-wip-us.apache.org/repos/asf/flink/blob/c8eb55f1/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
--
diff --git
a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
index e165983..cab3855 100644
---
a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
+++
b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/batch/TableEnvironmentITCase.java
@@ -28,6 +28,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.calcite.tools.RuleSets;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import org.apache.flink.table.api.java.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple4;
@@ -46,6 +47,8 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import static org.junit.Assert.assertTrue;
+
@RunWith(Parameterized.class)
public class TableEnvironmentITCase extends TableProgramsCollectionTestBase {
@@ -375,6 +378,34 @@ public class TableEnvironmentITCase extends
TableProgramsCollectionTestBase {
}
@Test(expected = TableException.class)
+ public void testGenericRow() throws Exception {
+ ExecutionEnvironment env =
ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv =
TableEnvironment.getTableEnvironment(env, config());
+
+ // use null va
[1/2] flink git commit: [FLINK-6247] [table] Put flink-table.jar with all dependencies into ./opt folder.
Repository: flink
Updated Branches:
refs/heads/master fff04bfbe -> bb972b85a
[FLINK-6247] [table] Put flink-table.jar with all dependencies into ./opt
folder.
This closes #3666.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/69905e19
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/69905e19
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/69905e19
Branch: refs/heads/master
Commit: 69905e19c636a05992443a647ba4142af5a40a01
Parents: fff04bf
Author: sunjincheng121
Authored: Fri Mar 31 16:31:18 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 27 17:03:59 2017 +0200
--
.../flink-connector-kafka-0.10/pom.xml | 9
flink-connectors/flink-hbase/pom.xml| 22
flink-dist/pom.xml | 7 +++
flink-dist/src/main/assemblies/opt.xml | 8 +++
flink-examples/flink-examples-table/pom.xml | 9
flink-libraries/flink-table/pom.xml | 10 +++--
6 files changed, 63 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/69905e19/flink-connectors/flink-connector-kafka-0.10/pom.xml
--
diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml
b/flink-connectors/flink-connector-kafka-0.10/pom.xml
index ab60946..9fb0e61 100644
--- a/flink-connectors/flink-connector-kafka-0.10/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml
@@ -56,6 +56,15 @@ under the License.
+
+
+
+ org.apache.flink
+ flink-streaming-java_2.10
+ ${project.version}
+ provided
+
+
http://git-wip-us.apache.org/repos/asf/flink/blob/69905e19/flink-connectors/flink-hbase/pom.xml
--
diff --git a/flink-connectors/flink-hbase/pom.xml
b/flink-connectors/flink-hbase/pom.xml
index 00c08ef..4fc0556 100644
--- a/flink-connectors/flink-hbase/pom.xml
+++ b/flink-connectors/flink-hbase/pom.xml
@@ -76,6 +76,8 @@ under the License.
provided
+
+
org.apache.flink
flink-java
@@ -83,6 +85,25 @@ under the License.
provided
+
+
+
+ org.apache.flink
+ flink-scala_2.10
+ ${project.version}
+ provided
+
+
+
+
+
+ org.apache.flink
+ flink-streaming-scala_2.10
+ ${project.version}
+ provided
+
+
+
org.apache.flink
@@ -93,6 +114,7 @@ under the License.
won't depend on flink-table. -->
true
+
org.apache.flink
flink-shaded-hadoop2
http://git-wip-us.apache.org/repos/asf/flink/blob/69905e19/flink-dist/pom.xml
--
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 76df09e..a700870 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -239,6 +239,13 @@ under the License.
${project.version}
provided
+
+
+ org.apache.flink
+ flink-table_2.10
+ ${project.version}
+ provided
+
http://git-wip-us.apache.org/repos/asf/flink/blob/69905e19/flink-dist/src/main/assemblies/opt.xml
--
diff --git a/flink-dist/src/main/assemblies/opt.xml
b/flink-dist/src/main/assemblies/opt.xml
index 3622ece..0282315 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -59,6 +59,14 @@
0644
+
+
+
../flink-libraries/flink-table/target/flink-table_2.10-${project.version}.jar
+ opt/
+
flink-table_2.10-${project.version}.jar
+ 0644
+
+
../flink-libraries/flink-ml/target/flink-ml_2.10-${project.version}-jar-with-dependencies.jar
http://git
[2/2] flink git commit: [FLINK-6392] [table] Improve group window API to make alias mandatory.
[FLINK-6392] [table] Improve group window API to make alias mandatory.
This closes #3786.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bb972b85
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bb972b85
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bb972b85
Branch: refs/heads/master
Commit: bb972b85a76d9b98ca5e90033e448f68f9109485
Parents: 69905e1
Author: sunjincheng121
Authored: Thu Apr 27 09:58:18 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 27 17:04:00 2017 +0200
--
.../apache/flink/table/api/java/windows.scala | 18 +-
.../apache/flink/table/api/scala/windows.scala | 18 +-
.../org/apache/flink/table/api/table.scala | 5 +-
.../org/apache/flink/table/api/windows.scala| 297 ---
.../table/plan/logical/LogicalWindow.scala | 7 +-
.../flink/table/plan/logical/groupWindows.scala | 28 +-
.../flink/table/plan/logical/operators.scala| 2 +-
.../common/WindowStartEndPropertiesRule.scala | 4 +-
.../scala/batch/sql/WindowAggregateTest.scala | 12 +-
.../scala/batch/table/FieldProjectionTest.scala | 4 +-
.../api/scala/batch/table/GroupWindowTest.scala | 18 +-
.../scala/stream/sql/WindowAggregateTest.scala | 6 +-
.../scala/stream/table/GroupWindowTest.scala| 48 +--
13 files changed, 274 insertions(+), 193 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/bb972b85/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
index 1e9d65a..15208ce 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/windows.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.api.java
-import org.apache.flink.table.api.{OverWindowPredefined, SessionWindow,
SlideWithSize, TumblingWindow}
+import org.apache.flink.table.api.{TumbleWithSize, OverWindowWithOrderBy,
SlideWithSize, SessionWithGap}
import org.apache.flink.table.expressions.{Expression, ExpressionParser}
/**
@@ -34,9 +34,9 @@ object Tumble {
* elements in 5 minutes intervals.
*
* @param size the size of the window as time or row-count interval.
-* @return a tumbling window
+* @return a partially defined tumbling window
*/
- def over(size: String): TumblingWindow = new TumblingWindow(size)
+ def over(size: String): TumbleWithSize = new TumbleWithSize(size)
}
/**
@@ -79,9 +79,9 @@ object Session {
*
* @param gap specifies how long (as interval of milliseconds) to wait for
new data before
*closing the session window.
-* @return a session window
+* @return a partially defined session window
*/
- def withGap(gap: String): SessionWindow = new SessionWindow(gap)
+ def withGap(gap: String): SessionWithGap = new SessionWithGap(gap)
}
/**
@@ -96,9 +96,9 @@ object Over {
*
* For batch tables, refer to a timestamp or long attribute.
*/
- def orderBy(orderBy: String): OverWindowPredefined = {
+ def orderBy(orderBy: String): OverWindowWithOrderBy = {
val orderByExpr = ExpressionParser.parseExpression(orderBy)
-new OverWindowPredefined(Seq[Expression](), orderByExpr)
+new OverWindowWithOrderBy(Seq[Expression](), orderByExpr)
}
/**
@@ -122,8 +122,8 @@ class PartitionedOver(private val partitionByExpr:
Array[Expression]) {
*
* For batch tables, refer to a timestamp or long attribute.
*/
- def orderBy(orderBy: String): OverWindowPredefined = {
+ def orderBy(orderBy: String): OverWindowWithOrderBy = {
val orderByExpr = ExpressionParser.parseExpression(orderBy)
-new OverWindowPredefined(partitionByExpr, orderByExpr)
+new OverWindowWithOrderBy(partitionByExpr, orderByExpr)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/bb972b85/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
index 1230027..5e70440 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/windows.scala
@@ -19,7 +19,7 @@
package org.apache.flink.table.api.scala
import org.apache.
[1/3] flink git commit: [FLINK-6091] [table] Implement and turn on retraction for non-windowed aggregates.
Repository: flink
Updated Branches:
refs/heads/table-retraction 5ba0f02a6 -> b237a3ef0
http://git-wip-us.apache.org/repos/asf/flink/blob/b237a3ef/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
index 6cbe834..4f14ce3 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/TableEnvironmentITCase.scala
@@ -27,6 +27,7 @@ import
org.apache.flink.table.api.scala.batch.utils.{TableProgramsCollectionTest
import
org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.types.Row
import org.apache.flink.table.api.{TableEnvironment, TableException}
+import org.apache.flink.table.runtime.types.CRow
import org.apache.flink.test.util.TestBaseUtils
import org.junit._
import org.junit.runner.RunWith
@@ -162,6 +163,25 @@ class TableEnvironmentITCase(
}
@Test
+ def testToDataSetWithTypeOfCRow(): Unit = {
+val env = ExecutionEnvironment.getExecutionEnvironment
+val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+val t = CollectionDataSets.get3TupleDataSet(env)
+ .toTable(tEnv, 'a, 'b, 'c)
+ .select('a, 'b, 'c)
+
+val expected = "+1,1,Hi\n" + "+2,2,Hello\n" + "+3,2,Hello world\n" +
+ "+4,3,Hello world, how are you?\n" + "+5,3,I am fine.\n" + "+6,3,Luke
Skywalker\n" +
+ "+7,4,Comment#1\n" + "+8,4,Comment#2\n" + "+9,4,Comment#3\n" +
"+10,4,Comment#4\n" +
+ "+11,5,Comment#5\n" + "+12,5,Comment#6\n" + "+13,5,Comment#7\n" +
"+14,5,Comment#8\n" +
+ "+15,5,Comment#9\n" + "+16,6,Comment#10\n" + "+17,6,Comment#11\n" +
"+18,6,Comment#12\n" +
+ "+19,6,Comment#13\n" + "+20,6,Comment#14\n" + "+21,6,Comment#15\n"
+val results = t.toDataSet[CRow].collect()
+TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+
+ @Test
def testToTableFromCaseClass(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val tEnv = TableEnvironment.getTableEnvironment(env, config)
http://git-wip-us.apache.org/repos/asf/flink/blob/b237a3ef/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
new file mode 100644
index 000..62fcfcd
--- /dev/null
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/RetractionITCase.scala
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream
+
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.table.api.{TableEnvironment, TableException}
+import org.apache.flink.table.api.scala.stream.utils.{StreamITCase,
StreamingWithStateTestBase}
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+import org.apache.flink.table.api.scala._
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.utils.TableFunc0
+
+import scala.collection.mutable
+
+/**
+ * tests for retraction
+ */
+class RetractionITCase extends StreamingWithStateTestBase {
+ // input data
+ val data = List(
+("Hello", 1),
+("word", 1),
+("Hello", 1),
+("bark", 1),
+("bark", 1),
+("bark", 1),
+("bark", 1),
+("bark", 1),
+("bark", 1),
+("flink", 1)
+ )
+
+ // keyed groupby + keyed groupby
+ @Test
+ def testWordCount(): Uni
[3/3] flink git commit: [FLINK-6091] [table] Implement and turn on retraction for non-windowed aggregates.
[FLINK-6091] [table] Implement and turn on retraction for non-windowed aggregates. Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b237a3ef Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b237a3ef Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b237a3ef Branch: refs/heads/table-retraction Commit: b237a3ef0a6632a5a4fa474e42b2f69d24b74dd5 Parents: 5ba0f02 Author: Hequn Cheng Authored: Tue Apr 18 16:54:09 2017 +0800 Committer: Fabian Hueske Committed: Wed Apr 26 12:20:22 2017 +0200 -- .../flink/table/api/BatchTableEnvironment.scala | 63 +++- .../table/api/StreamTableEnvironment.scala | 62 +++- .../flink/table/api/TableEnvironment.scala | 65 ++-- .../flink/table/plan/nodes/CommonCalc.scala | 26 +- .../table/plan/nodes/CommonCorrelate.scala | 58 +--- .../flink/table/plan/nodes/CommonScan.scala | 25 +- .../table/plan/nodes/dataset/BatchScan.scala| 10 +- .../table/plan/nodes/dataset/DataSetCalc.scala | 23 +- .../plan/nodes/dataset/DataSetCorrelate.scala | 34 +- .../plan/nodes/dataset/DataSetValues.scala | 2 +- .../plan/nodes/datastream/DataStreamCalc.scala | 30 +- .../nodes/datastream/DataStreamCorrelate.scala | 40 ++- .../datastream/DataStreamGroupAggregate.scala | 23 +- .../DataStreamGroupWindowAggregate.scala| 38 ++- .../datastream/DataStreamOverAggregate.scala| 61 ++-- .../plan/nodes/datastream/DataStreamRel.scala | 6 +- .../datastream/DataStreamRetractionRules.scala | 16 +- .../plan/nodes/datastream/DataStreamScan.scala | 3 +- .../plan/nodes/datastream/DataStreamUnion.scala | 3 +- .../nodes/datastream/DataStreamValues.scala | 14 +- .../plan/nodes/datastream/StreamScan.scala | 23 +- .../datastream/StreamTableSourceScan.scala | 4 +- .../runtime/CRowCorrelateFlatMapRunner.scala| 83 + .../flink/table/runtime/CRowFlatMapRunner.scala | 72 .../table/runtime/CRowInputMapRunner.scala | 57 .../table/runtime/CRowOutputMapRunner.scala | 60 .../table/runtime/CRowWrappingCollector.scala | 41 +++ .../flink/table/runtime/FlatMapRunner.scala | 17 +- .../aggregate/AggregateAggFunction.scala| 11 +- .../table/runtime/aggregate/AggregateUtil.scala | 36 +- ...SetSessionWindowAggReduceGroupFunction.scala | 4 +- ...taSetSlideWindowAggReduceGroupFunction.scala | 4 +- ...TumbleTimeWindowAggReduceGroupFunction.scala | 4 +- .../aggregate/GroupAggProcessFunction.scala | 74 +++-- ...rementalAggregateAllTimeWindowFunction.scala | 7 +- .../IncrementalAggregateAllWindowFunction.scala | 11 +- ...IncrementalAggregateTimeWindowFunction.scala | 7 +- .../IncrementalAggregateWindowFunction.scala| 13 +- .../aggregate/ProcTimeBoundedRangeOver.scala| 30 +- .../aggregate/ProcTimeBoundedRowsOver.scala | 26 +- .../ProcTimeUnboundedNonPartitionedOver.scala | 23 +- .../ProcTimeUnboundedPartitionedOver.scala | 22 +- .../aggregate/RowTimeBoundedRangeOver.scala | 32 +- .../aggregate/RowTimeBoundedRowsOver.scala | 30 +- .../aggregate/RowTimeUnboundedOver.scala| 50 +-- .../aggregate/TimeWindowPropertyCollector.scala | 34 +- .../runtime/io/CRowValuesInputFormat.scala | 59 .../table/runtime/io/ValuesInputFormat.scala| 17 +- .../apache/flink/table/runtime/types/CRow.scala | 55 +++ .../table/runtime/types/CRowComparator.scala| 83 + .../table/runtime/types/CRowSerializer.scala| 78 + .../table/runtime/types/CRowTypeInfo.scala | 98 ++ .../apache/flink/table/sinks/CsvTableSink.scala | 2 + .../flink/table/TableEnvironmentTest.scala | 68 +++- .../scala/batch/TableEnvironmentITCase.scala| 20 ++ .../api/scala/stream/RetractionITCase.scala | 331 +++ .../api/scala/stream/TableSinkITCase.scala | 2 +- .../api/scala/stream/utils/StreamITCase.scala | 11 +- ...ProcessingOverRangeProcessFunctionTest.scala | 105 +++--- .../runtime/types/CRowComparatorTest.scala | 61 .../table/utils/MockTableEnvironment.scala | 8 + 61 files changed, 1889 insertions(+), 486 deletions(-) -- http://git-wip-us.apache.org/repos/asf/flink/blob/b237a3ef/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala -- diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala index b48e9f9..5111926 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala +++ b/flink-libraries/flink-table/src/main/
[2/3] flink git commit: [FLINK-6091] [table] Implement and turn on retraction for non-windowed aggregates.
http://git-wip-us.apache.org/repos/asf/flink/blob/b237a3ef/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowWrappingCollector.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowWrappingCollector.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowWrappingCollector.scala
new file mode 100644
index 000..b2b062e
--- /dev/null
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowWrappingCollector.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime
+
+import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.types.Row
+import org.apache.flink.util.Collector
+
+/**
+ * The collector is used to wrap a [[Row]] to a [[CRow]]
+ */
+class CRowWrappingCollector() extends Collector[Row] {
+
+ var out: Collector[CRow] = _
+ val outCRow: CRow = new CRow()
+
+ def setChange(change: Boolean): Unit = this.outCRow.change = change
+
+ override def collect(record: Row): Unit = {
+outCRow.row = record
+out.collect(outCRow)
+ }
+
+ override def close(): Unit = out.close()
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b237a3ef/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
index b446306..2e37baf 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
@@ -24,20 +24,21 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ResultTypeQueryable
import org.apache.flink.table.codegen.Compiler
import org.apache.flink.configuration.Configuration
+import org.apache.flink.types.Row
import org.apache.flink.util.Collector
import org.slf4j.LoggerFactory
-class FlatMapRunner[IN, OUT](
+class FlatMapRunner(
name: String,
code: String,
-@transient returnType: TypeInformation[OUT])
- extends RichFlatMapFunction[IN, OUT]
- with ResultTypeQueryable[OUT]
- with Compiler[FlatMapFunction[IN, OUT]] {
+@transient returnType: TypeInformation[Row])
+ extends RichFlatMapFunction[Row, Row]
+ with ResultTypeQueryable[Row]
+ with Compiler[FlatMapFunction[Row, Row]] {
val LOG = LoggerFactory.getLogger(this.getClass)
- private var function: FlatMapFunction[IN, OUT] = _
+ private var function: FlatMapFunction[Row, Row] = _
override def open(parameters: Configuration): Unit = {
LOG.debug(s"Compiling FlatMapFunction: $name \n\n Code:\n$code")
@@ -48,10 +49,10 @@ class FlatMapRunner[IN, OUT](
FunctionUtils.openFunction(function, parameters)
}
- override def flatMap(in: IN, out: Collector[OUT]): Unit =
+ override def flatMap(in: Row, out: Collector[Row]): Unit =
function.flatMap(in, out)
- override def getProducedType: TypeInformation[OUT] = returnType
+ override def getProducedType: TypeInformation[Row] = returnType
override def close(): Unit = {
FunctionUtils.closeFunction(function)
http://git-wip-us.apache.org/repos/asf/flink/blob/b237a3ef/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
index c608b97..88cf2f1 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/Agg
[2/2] flink git commit: [FLINK-6361] [table] Refactor the AggregateFunction interface and built-in aggregates.
[FLINK-6361] [table] Refactor the AggregateFunction interface and built-in
aggregates.
This closes #3762.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bc6409d6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bc6409d6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bc6409d6
Branch: refs/heads/master
Commit: bc6409d624df54c2309c8bdb767f95de74ea1475
Parents: fe01892
Author: shaoxuan-wang
Authored: Tue Apr 25 00:28:37 2017 +0800
Committer: Fabian Hueske
Committed: Tue Apr 25 14:21:05 2017 +0200
--
.../org/apache/flink/table/api/Types.scala | 3 +-
.../flink/table/codegen/CodeGenerator.scala | 60 +++---
.../table/functions/AggregateFunction.scala | 157 +++---
.../functions/aggfunctions/AvgAggFunction.scala | 206 ---
.../aggfunctions/CountAggFunction.scala | 39 ++--
.../functions/aggfunctions/MaxAggFunction.scala | 48 ++---
.../MaxAggFunctionWithRetract.scala | 86
.../functions/aggfunctions/MinAggFunction.scala | 48 ++---
.../MinAggFunctionWithRetract.scala | 86
.../functions/aggfunctions/SumAggFunction.scala | 89
.../SumWithRetractAggFunction.scala | 107 +-
.../table/runtime/aggregate/AggregateUtil.scala | 96 ++---
.../aggregate/GeneratedAggregations.scala | 32 +++
.../aggfunctions/AggFunctionTestBase.scala | 62 +++---
.../aggfunctions/AvgFunctionTest.scala | 23 ++-
.../aggfunctions/CountAggFunctionTest.scala | 6 +-
.../aggfunctions/MaxAggFunctionTest.scala | 44 ++--
.../MaxWithRetractAggFunctionTest.scala | 47 +++--
.../aggfunctions/MinAggFunctionTest.scala | 45 ++--
.../MinWithRetractAggFunctionTest.scala | 47 +++--
.../aggfunctions/SumAggFunctionTest.scala | 31 +--
.../SumWithRetractAggFunctionTest.scala | 31 ++-
...ProcessingOverRangeProcessFunctionTest.scala | 28 ++-
23 files changed, 772 insertions(+), 649 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/bc6409d6/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
index 262a452..d82b990 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/Types.scala
@@ -17,9 +17,8 @@
*/
package org.apache.flink.table.api
-import org.apache.flink.api.common.typeinfo.{Types, TypeInformation}
+import org.apache.flink.api.common.typeinfo.{Types => JTypes, TypeInformation}
import org.apache.flink.table.typeutils.TimeIntervalTypeInfo
-import org.apache.flink.api.common.typeinfo.{Types => JTypes}
/**
* This class enumerates all supported types of the Table API.
http://git-wip-us.apache.org/repos/asf/flink/blob/bc6409d6/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 510a870..298fb70 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -265,14 +265,16 @@ class CodeGenerator(
name: String,
generator: CodeGenerator,
inputType: RelDataType,
- aggregates: Array[AggregateFunction[_ <: Any]],
+ aggregates: Array[AggregateFunction[_ <: Any, _ <: Any]],
aggFields: Array[Array[Int]],
aggMapping: Array[Int],
partialResults: Boolean,
fwdMapping: Array[Int],
mergeMapping: Option[Array[Int]],
constantFlags: Option[Array[(Int, Boolean)]],
- outputArity: Int)
+ outputArity: Int,
+ needRetract: Boolean,
+ needMerge: Boolean)
: GeneratedAggregationsFunction = {
// get unique function name
@@ -364,9 +366,16 @@ class CodeGenerator(
| ${parameters(i)});""".stripMargin
}.mkString("\n")
- j"""$sig {
- |$retract
- | }""".stripMargin
+ if (needRetract) {
+j"""
+ |$sig {
+ |$retract
+ | }""".stripMargin
+ } else {
+j"""
+ |$sig {
+ | }""".stripMargin
+ }
}
def genCreateAccumulators: String = {
@@ -4
[1/2] flink git commit: [FLINK-6361] [table] Refactor the AggregateFunction interface and built-in aggregates.
Repository: flink
Updated Branches:
refs/heads/master fe018921e -> bc6409d62
http://git-wip-us.apache.org/repos/asf/flink/blob/bc6409d6/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AvgFunctionTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AvgFunctionTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AvgFunctionTest.scala
index a388acf..fd510ff 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AvgFunctionTest.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/AvgFunctionTest.scala
@@ -25,7 +25,7 @@ import org.apache.flink.table.functions.AggregateFunction
*
* @tparam T the type for the aggregation result
*/
-abstract class AvgAggFunctionTestBase[T: Numeric] extends
AggFunctionTestBase[T] {
+abstract class AvgAggFunctionTestBase[T: Numeric, ACC] extends
AggFunctionTestBase[T, ACC] {
private val numeric: Numeric[T] = implicitly[Numeric[T]]
@@ -91,9 +91,11 @@ abstract class AvgAggFunctionTestBase[T: Numeric] extends
AggFunctionTestBase[T]
numeric.fromInt(3),
null.asInstanceOf[T]
)
+
+ override def retractFunc = aggregator.getClass.getMethod("retract", accType,
classOf[Any])
}
-class ByteAvgAggFunctionTest extends AvgAggFunctionTestBase[Byte] {
+class ByteAvgAggFunctionTest extends AvgAggFunctionTestBase[Byte,
IntegralAvgAccumulator] {
override def minVal = (Byte.MinValue + 1).toByte
@@ -102,7 +104,7 @@ class ByteAvgAggFunctionTest extends
AvgAggFunctionTestBase[Byte] {
override def aggregator = new ByteAvgAggFunction()
}
-class ShortAvgAggFunctionTest extends AvgAggFunctionTestBase[Short] {
+class ShortAvgAggFunctionTest extends AvgAggFunctionTestBase[Short,
IntegralAvgAccumulator] {
override def minVal = (Short.MinValue + 1).toShort
@@ -111,7 +113,7 @@ class ShortAvgAggFunctionTest extends
AvgAggFunctionTestBase[Short] {
override def aggregator = new ShortAvgAggFunction()
}
-class IntAvgAggFunctionTest extends AvgAggFunctionTestBase[Int] {
+class IntAvgAggFunctionTest extends AvgAggFunctionTestBase[Int,
IntegralAvgAccumulator] {
override def minVal = Int.MinValue + 1
@@ -120,7 +122,7 @@ class IntAvgAggFunctionTest extends
AvgAggFunctionTestBase[Int] {
override def aggregator = new IntAvgAggFunction()
}
-class LongAvgAggFunctionTest extends AvgAggFunctionTestBase[Long] {
+class LongAvgAggFunctionTest extends AvgAggFunctionTestBase[Long,
BigIntegralAvgAccumulator] {
override def minVal = Long.MinValue + 1
@@ -129,7 +131,7 @@ class LongAvgAggFunctionTest extends
AvgAggFunctionTestBase[Long] {
override def aggregator = new LongAvgAggFunction()
}
-class FloatAvgAggFunctionTest extends AvgAggFunctionTestBase[Float] {
+class FloatAvgAggFunctionTest extends AvgAggFunctionTestBase[Float,
FloatingAvgAccumulator] {
override def minVal = Float.MinValue
@@ -138,7 +140,7 @@ class FloatAvgAggFunctionTest extends
AvgAggFunctionTestBase[Float] {
override def aggregator = new FloatAvgAggFunction()
}
-class DoubleAvgAggFunctionTest extends AvgAggFunctionTestBase[Double] {
+class DoubleAvgAggFunctionTest extends AvgAggFunctionTestBase[Double,
FloatingAvgAccumulator] {
override def minVal = Float.MinValue
@@ -147,7 +149,7 @@ class DoubleAvgAggFunctionTest extends
AvgAggFunctionTestBase[Double] {
override def aggregator = new DoubleAvgAggFunction()
}
-class DecimalAvgAggFunctionTest extends AggFunctionTestBase[BigDecimal] {
+class DecimalAvgAggFunctionTest extends AggFunctionTestBase[BigDecimal,
DecimalAvgAccumulator] {
override def inputValueSets: Seq[Seq[_]] = Seq(
Seq(
@@ -182,5 +184,8 @@ class DecimalAvgAggFunctionTest extends
AggFunctionTestBase[BigDecimal] {
null
)
- override def aggregator: AggregateFunction[BigDecimal] = new
DecimalAvgAggFunction()
+ override def aggregator: AggregateFunction[BigDecimal,
DecimalAvgAccumulator] =
+new DecimalAvgAggFunction()
+
+ override def retractFunc = aggregator.getClass.getMethod("retract", accType,
classOf[Any])
}
http://git-wip-us.apache.org/repos/asf/flink/blob/bc6409d6/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunctionTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunctionTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunctionTest.scala
index d5f09b2..6830b8f 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/functions/aggfunctions/CountAggFunctionTest.scala
+++
b/flink-libraries/flink-table/src/t
[1/2] flink git commit: [FLINK-6228] [table] Add support for OVER windows to streaming Table API.
Repository: flink
Updated Branches:
refs/heads/master 166444767 -> fe018921e
http://git-wip-us.apache.org/repos/asf/flink/blob/fe018921/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
new file mode 100644
index 000..0a5e001
--- /dev/null
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream.table.stringexpr
+
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.java.{Over => JOver}
+import org.apache.flink.table.api.scala.{Over => SOver, _}
+import org.apache.flink.table.utils.TableTestBase
+import org.junit.Test
+
+class OverWindowStringExpressionTest extends TableTestBase {
+
+ @Test
+ def testPartitionedUnboundedOverRow(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+ .window(SOver partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_ROW as
'w)
+ .select('a, 'b.sum over 'w)
+val resJava = t
+
.window(JOver.partitionBy("a").orderBy("rowtime").preceding("unbounded_row").as("w"))
+ .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+ }
+
+ @Test
+ def testUnboundedOverRow(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+ .window(SOver orderBy 'rowtime preceding UNBOUNDED_ROW following
CURRENT_ROW as 'w)
+ .select('a, 'b.sum over 'w)
+val resJava = t
+
.window(JOver.orderBy("rowtime").preceding("unbounded_row").following("current_row").as("w"))
+ .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+ }
+
+ @Test
+ def testPartitionedBoundedOverRow(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+ .window(SOver partitionBy('a, 'd) orderBy 'rowtime preceding 10.rows as
'w)
+ .select('a, 'b.sum over 'w)
+val resJava = t
+ .window(JOver.partitionBy("a,
d").orderBy("rowtime").preceding("10.rows").as("w"))
+ .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+ }
+
+ @Test
+ def testBoundedOverRow(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+ .window(SOver orderBy 'rowtime preceding 10.rows following CURRENT_ROW
as 'w)
+ .select('a, 'b.sum over 'w)
+val resJava = t
+
.window(JOver.orderBy("rowtime").preceding("10.rows").following("current_row").as("w"))
+ .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+ }
+
+ @Test
+ def testPartitionedUnboundedOverRange(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+ .window(SOver partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_RANGE
as 'w)
+ .select('a, 'b.sum over 'w)
+val resJava = t
+
.window(JOver.partitionBy("a").orderBy("rowtime").preceding("unbounded_range").as("w"))
+ .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+ }
+
+ @Test
+ def testUnboundedOverRange(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+ .window(SOver orderBy 'rowtime preceding UNBOUNDED_RANGE following
CURRENT_RANGE as 'w)
+ .select('a, 'b.sum over 'w)
+val resJava = t
+ .window(
+
JOver.ord
[2/2] flink git commit: [FLINK-6228] [table] Add support for OVER windows to streaming Table API.
[FLINK-6228] [table] Add support for OVER windows to streaming Table API.
This closes #3743.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fe018921
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fe018921
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fe018921
Branch: refs/heads/master
Commit: fe018921ed0b24f94ab2139f04293d6074ce4fba
Parents: 1664447
Author: sunjincheng121
Authored: Thu Apr 13 17:36:18 2017 +0800
Committer: Fabian Hueske
Committed: Mon Apr 24 18:39:30 2017 +0200
--
.../flink/table/api/java/groupWindows.scala | 84 ---
.../apache/flink/table/api/java/windows.scala | 129
.../flink/table/api/scala/expressionDsl.scala | 27 +-
.../flink/table/api/scala/groupWindows.scala| 85 ---
.../apache/flink/table/api/scala/windows.scala | 126
.../org/apache/flink/table/api/table.scala | 62 ++
.../org/apache/flink/table/api/windows.scala| 118 +++-
.../table/expressions/ExpressionParser.scala| 64 +-
.../flink/table/expressions/aggregations.scala | 36 +-
.../apache/flink/table/expressions/call.scala | 211 ++-
.../flink/table/plan/ProjectionTranslator.scala | 51 +-
.../scala/stream/table/OverWindowITCase.scala | 331 ++
.../api/scala/stream/table/OverWindowTest.scala | 596 +++
.../OverWindowStringExpressionTest.scala| 151 +
14 files changed, 1880 insertions(+), 191 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/fe018921/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
deleted file mode 100644
index 9c82e9b..000
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.api.java
-
-import org.apache.flink.table.api.{SessionWindow, SlideWithSize,
TumblingWindow}
-
-/**
- * Helper class for creating a tumbling window. Tumbling windows are
consecutive, non-overlapping
- * windows of a specified fixed length. For example, a tumbling window of 5
minutes size groups
- * elements in 5 minutes intervals.
- */
-object Tumble {
-
- /**
-* Creates a tumbling window. Tumbling windows are consecutive,
non-overlapping
-* windows of a specified fixed length. For example, a tumbling window of 5
minutes size groups
-* elements in 5 minutes intervals.
-*
-* @param size the size of the window as time or row-count interval.
-* @return a tumbling window
-*/
- def over(size: String): TumblingWindow = new TumblingWindow(size)
-}
-
-/**
- * Helper class for creating a sliding window. Sliding windows have a fixed
size and slide by
- * a specified slide interval. If the slide interval is smaller than the
window size, sliding
- * windows are overlapping. Thus, an element can be assigned to multiple
windows.
- *
- * For example, a sliding window of size 15 minutes with 5 minutes sliding
interval groups elements
- * of 15 minutes and evaluates every five minutes. Each element is contained
in three consecutive
- * window evaluations.
- */
-object Slide {
-
- /**
-* Creates a sliding window. Sliding windows have a fixed size and slide by
-* a specified slide interval. If the slide interval is smaller than the
window size, sliding
-* windows are overlapping. Thus, an element can be assigned to multiple
windows.
-*
-* For example, a sliding window of size 15 minutes with 5 minutes sliding
interval groups
-* elements of 15 minutes and evaluates every five minutes. Each element is
contained in three
-* consecutive window evaluations.
-*
-* @param size the size of the window
[2/5] flink git commit: [FLINK-6326] [table] Add ProjectMergeRule to logical optimization rule set.
[FLINK-6326] [table] Add ProjectMergeRule to logical optimization rule set.
This closes #3739.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4024afff
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4024afff
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4024afff
Branch: refs/heads/master
Commit: 4024afffcd5a3f011e8ba2a3a1053e8bae6cec4f
Parents: 46a950d
Author: godfreyhe
Authored: Wed Apr 19 19:46:18 2017 +0800
Committer: Fabian Hueske
Committed: Fri Apr 21 18:22:10 2017 +0200
--
.../flink/table/plan/rules/FlinkRuleSets.scala | 2 +
.../table/api/scala/batch/ExplainTest.scala | 1 +
.../batch/sql/QueryDecorrelationTest.scala | 72
.../api/scala/batch/sql/SetOperatorsTest.scala | 20 +++---
.../scala/batch/sql/WindowAggregateTest.scala | 50 ++
5 files changed, 77 insertions(+), 68 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/4024afff/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
index 94a1abb..0bee4e5 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala
@@ -49,6 +49,8 @@ object FlinkRuleSets {
FilterProjectTransposeRule.INSTANCE,
// push a projection to the children of a join
ProjectJoinTransposeRule.INSTANCE,
+// merge projections
+ProjectMergeRule.INSTANCE,
// remove identity project
ProjectRemoveRule.INSTANCE,
// reorder sort and projection
http://git-wip-us.apache.org/repos/asf/flink/blob/4024afff/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/ExplainTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/ExplainTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/ExplainTest.scala
index a323ec9..1a6b314 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/ExplainTest.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/ExplainTest.scala
@@ -119,4 +119,5 @@ class ExplainTest
"../../src/test/scala/resources/testUnion1.out").mkString.replaceAll("\\r\\n",
"\n")
assertEquals(result, source)
}
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4024afff/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
index 7496ff8..34ae346 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
@@ -44,38 +44,41 @@ class QueryDecorrelationTest extends TableTestBase {
"DataSetCalc",
binaryNode(
"DataSetJoin",
-binaryNode(
- "DataSetJoin",
- unaryNode(
-"DataSetCalc",
-batchTableNode(0),
-term("select", "empno", "ename", "job", "salary", "deptno"),
-term("where", "<(deptno, 10)")
- ),
- unaryNode(
-"DataSetCalc",
-batchTableNode(1),
-term("select", "deptno", "name"),
-term("where", "<(deptno, 15)")
+unaryNode(
+ "DataSetCalc",
+ binaryNode(
+"DataSetJoin",
+unaryNode(
+ "DataSetCalc",
+ batchTableNode(0),
+ term("select", "empno", "salary", "deptno"),
+ term("where", "<(deptno, 10)")
+),
+unaryNode(
+ "DataSetCalc",
+ batchTableNode(1),
+ term("select", "deptno"),
+ term("where", "<(deptno, 15)")
+),
+term("where", "=(deptno, deptno0)"),
+term("join", "empno", "salary", "deptno", "deptno0"),
+term("joinType", "InnerJoin")
),
- term(
[3/5] flink git commit: [FLINK-6242] [table] Add code generation for DataSet Aggregates
http://git-wip-us.apache.org/repos/asf/flink/blob/3b4542b8/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceCombineFunction.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceCombineFunction.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceCombineFunction.scala
index 8eac79d..381d443 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceCombineFunction.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceCombineFunction.scala
@@ -20,7 +20,8 @@ package org.apache.flink.table.runtime.aggregate
import java.lang.Iterable
import org.apache.flink.api.common.functions.CombineFunction
-import org.apache.flink.table.functions.{Accumulator, AggregateFunction}
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.GeneratedAggregationsFunction
import org.apache.flink.types.Row
/**
@@ -30,87 +31,62 @@ import org.apache.flink.types.Row
*
* It is used for sliding on batch for both time and count-windows.
*
- * @param aggregates aggregate functions.
- * @param groupKeysMapping index mapping of group keys between intermediate
aggregate Row
- * and output Row.
- * @param aggregateMapping index mapping between aggregate function list and
aggregated value
- * index in output Row.
- * @param finalRowArity output row field count
+ * @param genPreAggregations Code-generated [[GeneratedAggregations]] for
partial aggregation.
+ * @param genFinalAggregations Code-generated [[GeneratedAggregations]] for
final aggregation.
+ * @param keysAndAggregatesArity The total arity of keys and aggregates
* @param finalRowWindowStartPos relative window-start position to last field
of output row
* @param finalRowWindowEndPos relative window-end position to last field of
output row
* @param windowSize size of the window, used to determine window-end for
output row
*/
class DataSetSlideWindowAggReduceCombineFunction(
-aggregates: Array[AggregateFunction[_ <: Any]],
-groupKeysMapping: Array[(Int, Int)],
-aggregateMapping: Array[(Int, Int)],
-finalRowArity: Int,
+genPreAggregations: GeneratedAggregationsFunction,
+genFinalAggregations: GeneratedAggregationsFunction,
+keysAndAggregatesArity: Int,
finalRowWindowStartPos: Option[Int],
finalRowWindowEndPos: Option[Int],
windowSize: Long)
extends DataSetSlideWindowAggReduceGroupFunction(
-aggregates,
-groupKeysMapping,
-aggregateMapping,
-finalRowArity,
+genFinalAggregations,
+keysAndAggregatesArity,
finalRowWindowStartPos,
finalRowWindowEndPos,
windowSize)
with CombineFunction[Row, Row] {
- private val intermediateRowArity: Int = groupKeysMapping.length +
aggregateMapping.length + 1
- private val intermediateRow: Row = new Row(intermediateRowArity)
+ private val intermediateRow: Row = new Row(keysAndAggregatesArity + 1)
- override def combine(records: Iterable[Row]): Row = {
-
-// reset first accumulator
-var i = 0
-while (i < aggregates.length) {
- aggregates(i).resetAccumulator(accumulatorList(i).get(0))
- i += 1
-}
-
-val iterator = records.iterator()
-while (iterator.hasNext) {
- val record = iterator.next()
+ protected var preAggfunction: GeneratedAggregations = _
- // accumulate
- i = 0
- while (i < aggregates.length) {
-// insert received accumulator into acc list
-val newAcc = record.getField(groupKeysMapping.length +
i).asInstanceOf[Accumulator]
-accumulatorList(i).set(1, newAcc)
-// merge acc list
-val retAcc = aggregates(i).merge(accumulatorList(i))
-// insert result into acc list
-accumulatorList(i).set(0, retAcc)
-i += 1
- }
+ override def open(config: Configuration): Unit = {
+super.open(config)
- // check if this record is the last record
- if (!iterator.hasNext) {
-// set group keys
-i = 0
-while (i < groupKeysMapping.length) {
- intermediateRow.setField(i, record.getField(i))
- i += 1
-}
+LOG.debug(s"Compiling AggregateHelper: $genPreAggregations.name \n\n " +
+ s"Code:\n$genPreAggregations.code")
+val clazz = compile(
+ getClass.getClassLoader,
+ genPreAggregations.name,
+ genPreAggregations.code)
+LOG.debug("Instantiating AggregateHelper.")
+preAggfunction = clazz.newInstance()
+ }
-// set the partial accumulated result
-i = 0
-while (i < aggregates.length) {
- intermediateRow.set
[1/5] flink git commit: [FLINK-4562] [table] Move table examples into a dedicated module in flink-examples.
Repository: flink
Updated Branches:
refs/heads/master 83061ad0f -> db31ca3f8
[FLINK-4562] [table] Move table examples into a dedicated module in
flink-examples.
This closes #2460.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/46a950df
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/46a950df
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/46a950df
Branch: refs/heads/master
Commit: 46a950df8b311e0dc64c709a02c56b0497e47c21
Parents: 83061ad
Author: shijinkui
Authored: Fri Feb 10 16:42:29 2017 +0800
Committer: Fabian Hueske
Committed: Fri Apr 21 18:21:42 2017 +0200
--
flink-dist/pom.xml | 1 +
flink-examples/flink-examples-table/pom.xml | 108 +++
.../flink/table/examples/java/WordCountSQL.java | 87 +
.../table/examples/java/WordCountTable.java | 85 +
.../table/examples/scala/StreamSQLExample.scala | 76
.../examples/scala/StreamTableExample.scala | 71
.../table/examples/scala/TPCHQuery3Table.scala | 180 +++
.../table/examples/scala/WordCountSQL.scala | 62 +++
.../table/examples/scala/WordCountTable.scala | 62 +++
flink-examples/pom.xml | 1 +
.../flink/table/examples/java/WordCountSQL.java | 87 -
.../table/examples/java/WordCountTable.java | 85 -
.../table/examples/scala/StreamSQLExample.scala | 76
.../examples/scala/StreamTableExample.scala | 72
.../table/examples/scala/TPCHQuery3Table.scala | 180 ---
.../table/examples/scala/WordCountSQL.scala | 62 ---
.../table/examples/scala/WordCountTable.scala | 62 ---
.../scala/batch/table/AggregationsITCase.scala | 15 +-
18 files changed, 741 insertions(+), 631 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/46a950df/flink-dist/pom.xml
--
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index e42aea6..76df09e 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -338,6 +338,7 @@ under the License.
org.apache.flink:flink-examples-batch
org.apache.flink:flink-examples-streaming
+
org.apache.flink:flink-examples-table
org.apache.flink:flink-python
org.slf4j:slf4j-log4j12
log4j:log4j
http://git-wip-us.apache.org/repos/asf/flink/blob/46a950df/flink-examples/flink-examples-table/pom.xml
--
diff --git a/flink-examples/flink-examples-table/pom.xml
b/flink-examples/flink-examples-table/pom.xml
new file mode 100644
index 000..de050d7
--- /dev/null
+++ b/flink-examples/flink-examples-table/pom.xml
@@ -0,0 +1,108 @@
+
+
+http://maven.apache.org/POM/4.0.0";
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ 4.0.0
+
+
+ org.apache.flink
+ flink-examples_2.10
+ 1.3-SNAPSHOT
+ ..
+
+
+ flink-examples-table
+ flink-examples-table_2.10
+ jar
+
+
+
+ org.apache.flink
+ flink-table_2.10
+ ${project.version}
+
+
+
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ scala-compile-first
+ process-resources
+
+ add-source
+ compile
+
+
+
+ scala-test-compile
+
process-test-resources
+
+ testC
[4/5] flink git commit: [FLINK-6242] [table] Add code generation for DataSet Aggregates
[FLINK-6242] [table] Add code generation for DataSet Aggregates
This closes #3735.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3b4542b8
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3b4542b8
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3b4542b8
Branch: refs/heads/master
Commit: 3b4542b8f0981f01e42c861bccbc67c8b3a20fdd
Parents: 4024aff
Author: shaoxuan-wang
Authored: Tue Apr 18 21:45:49 2017 +0800
Committer: Fabian Hueske
Committed: Fri Apr 21 21:28:54 2017 +0200
--
.../flink/table/codegen/CodeGenerator.scala | 224 +++-
.../plan/nodes/dataset/DataSetAggregate.scala | 11 +-
.../nodes/dataset/DataSetWindowAggregate.scala | 25 +-
.../table/runtime/aggregate/AggregateUtil.scala | 359 +++
.../runtime/aggregate/DataSetAggFunction.scala | 109 ++
.../aggregate/DataSetFinalAggFunction.scala | 121 ++-
.../aggregate/DataSetPreAggFunction.scala | 93 ++---
...SetSessionWindowAggReduceGroupFunction.scala | 116 ++
...aSetSessionWindowAggregatePreProcessor.scala | 145 +++-
...tSlideTimeWindowAggReduceGroupFunction.scala | 177 +++--
...SetSlideWindowAggReduceCombineFunction.scala | 98 ++---
...taSetSlideWindowAggReduceGroupFunction.scala | 123 +++
...umbleCountWindowAggReduceGroupFunction.scala | 89 ++---
...mbleTimeWindowAggReduceCombineFunction.scala | 88 ++---
...TumbleTimeWindowAggReduceGroupFunction.scala | 99 ++---
.../aggregate/DataSetWindowAggMapFunction.scala | 64 ++--
.../aggregate/GeneratedAggregations.scala | 32 +-
.../scala/batch/table/AggregationsITCase.scala | 3 +-
...ProcessingOverRangeProcessFunctionTest.scala | 8 +-
19 files changed, 878 insertions(+), 1106 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/3b4542b8/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index c6e3c9a..510a870 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -250,57 +250,88 @@ class CodeGenerator(
* @param aggregates All aggregate functions
* @param aggFields Indexes of the input fields for all aggregate
functions
* @param aggMapping The mapping of aggregates to output fields
+* @param partialResults A flag defining whether final or partial results
(accumulators) are set
+* to the output row.
* @param fwdMapping The mapping of input fields to output fields
+* @param mergeMapping An optional mapping to specify the accumulators to
merge. If not set, we
+* assume that both rows have the accumulators at the
same position.
+* @param constantFlags An optional parameter to define where to set
constant boolean flags in
+* the output row.
* @param outputArity The number of fields in the output row.
*
* @return A GeneratedAggregationsFunction
*/
def generateAggregations(
- name: String,
- generator: CodeGenerator,
- inputType: RelDataType,
- aggregates: Array[AggregateFunction[_ <: Any]],
- aggFields: Array[Array[Int]],
- aggMapping: Array[Int],
- fwdMapping: Array[(Int, Int)],
- outputArity: Int)
+ name: String,
+ generator: CodeGenerator,
+ inputType: RelDataType,
+ aggregates: Array[AggregateFunction[_ <: Any]],
+ aggFields: Array[Array[Int]],
+ aggMapping: Array[Int],
+ partialResults: Boolean,
+ fwdMapping: Array[Int],
+ mergeMapping: Option[Array[Int]],
+ constantFlags: Option[Array[(Int, Boolean)]],
+ outputArity: Int)
: GeneratedAggregationsFunction = {
-def genSetAggregationResults(
- accTypes: Array[String],
- aggs: Array[String],
- aggMapping: Array[Int]): String = {
+// get unique function name
+val funcName = newName(name)
+// register UDAGGs
+val aggs = aggregates.map(a => generator.addReusableFunction(a))
+// get java types of accumulators
+val accTypes = aggregates.map { a =>
+ a.getClass.getMethod("createAccumulator").getReturnType.getCanonicalName
+}
+
+// get java types of input fields
+val javaTypes = inputType.getFieldList
+ .map(f => FlinkTypeFactory.toTypeInfo(f.getType))
+ .map(t => t.getTypeClass.getCanonicalName)
+// get parameter lists for aggregation functions
+val parameters =
[5/5] flink git commit: [hotfix] [table] Add missing semicolons to Table API Java examples.
[hotfix] [table] Add missing semicolons to Table API Java examples.
This closes #3754.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/db31ca3f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/db31ca3f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/db31ca3f
Branch: refs/heads/master
Commit: db31ca3f820fee43e3b3cdc3da3ffa49be1f7bf8
Parents: 3b4542b
Author: Ken Geis
Authored: Fri Apr 21 11:09:25 2017 -0700
Committer: Fabian Hueske
Committed: Fri Apr 21 21:32:38 2017 +0200
--
docs/dev/table_api.md | 38 +++---
1 file changed, 19 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/db31ca3f/docs/dev/table_api.md
--
diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 2a838c7..3d77b85 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -68,7 +68,7 @@ ExecutionEnvironment env =
ExecutionEnvironment.getExecutionEnvironment();
BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
// register the DataSet cust as table "Customers" with fields derived from the
dataset
-tableEnv.registerDataSet("Customers", cust)
+tableEnv.registerDataSet("Customers", cust);
// register the DataSet ord as table "Orders" with fields user, product, and
amount
tableEnv.registerDataSet("Orders", ord, "user, product, amount");
@@ -102,7 +102,7 @@ StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironm
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
// register the DataStream cust as table "Customers" with fields derived from
the datastream
-tableEnv.registerDataStream("Customers", cust)
+tableEnv.registerDataStream("Customers", cust);
// register the DataStream ord as table "Orders" with fields user, product,
and amount
tableEnv.registerDataStream("Orders", ord, "user, product, amount");
@@ -140,10 +140,10 @@ BatchTableEnvironment tableEnv =
TableEnvironment.getTableEnvironment(env);
Table custT = tableEnv
.toTable(custDs, "name, zipcode")
.where("zipcode = '12345'")
- .select("name")
+ .select("name");
// register the Table custT as table "custNames"
-tableEnv.registerTable("custNames", custT)
+tableEnv.registerTable("custNames", custT);
{% endhighlight %}
@@ -178,10 +178,10 @@ An external table is registered in a `TableEnvironment`
using a `TableSource` as
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
-TableSource custTS = new CsvTableSource("/path/to/file", ...)
+TableSource custTS = new CsvTableSource("/path/to/file", ...);
// register a `TableSource` as external table "Customers"
-tableEnv.registerTableSource("Customers", custTS)
+tableEnv.registerTableSource("Customers", custTS);
{% endhighlight %}
@@ -1047,7 +1047,7 @@ The following example shows how to define a window
aggregation on a table.
Table table = input
.window([Window w].as("w")) // define window with alias w
.groupBy("w") // group the table by window w
- .select("b.sum") // aggregate
+ .select("b.sum"); // aggregate
{% endhighlight %}
@@ -1070,7 +1070,7 @@ The following example shows how to define a window
aggregation with additional g
Table table = input
.window([Window w].as("w")) // define window with alias w
.groupBy("w, a") // group the table by attribute a and window w
- .select("a, b.sum") // aggregate
+ .select("a, b.sum"); // aggregate
{% endhighlight %}
@@ -1092,7 +1092,7 @@ The `Window` parameter defines how rows are mapped to
windows. `Window` is not a
Table table = input
.window([Window w].as("w")) // define window with alias w
.groupBy("w, a") // group the table by attribute a and window w
- .select("a, w.start, w.end, b.count") // aggregate and add window start and
end timestamps
+ .select("a, w.start, w.end, b.count"); // aggregate and add window start and
end timestamps
{% endhighlight %}
@@ -1144,13 +1144,13 @@ Tumbling windows are defined by using the `Tumble`
class as follows:
{% highlight java %}
// Tumbling Event-time Window
-.window(Tumble.over("10.minutes").on("rowtime").as("w"))
+.window(Tumble.over("10.minutes").on("rowtime").as("w"));
// Tumbling Processing-time Window
-.window(Tumble.over("10.minutes").as("w"))
+.window(Tumble.over("10.minutes").as("w"));
// Tumbling Row-count Window
-.window(Tumble.over("10.rows").as("w"))
+.window(Tumble.over("10.rows").as("w"));
{% endhighlight %}
@@ -1211,13 +1211,13 @@ Sliding windows are defined by using the `Slide` class
as follows:
{% highlight java %}
// Sliding Event-time Window
-.window(Slide.over("10.minutes"
flink git commit: [FLINK-6240] [table] Add code generation for DataStream group window aggregations.
Repository: flink
Updated Branches:
refs/heads/master 283f5efd5 -> 9be5cc42c
[FLINK-6240] [table] Add code generation for DataStream group window
aggregations.
This closes #3694.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9be5cc42
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9be5cc42
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9be5cc42
Branch: refs/heads/master
Commit: 9be5cc42c02c258d3843c373b7350240c9570523
Parents: 283f5ef
Author: shaoxuan-wang
Authored: Fri Apr 7 16:18:14 2017 +0800
Committer: Fabian Hueske
Committed: Mon Apr 17 22:45:55 2017 +0200
--
.../flink/table/codegen/CodeGenerator.scala | 82 +---
.../nodes/datastream/DataStreamAggregate.scala | 14 +++-
.../aggregate/AggregateAggFunction.scala| 74 --
.../table/runtime/aggregate/AggregateUtil.scala | 19 -
.../aggregate/GeneratedAggregations.scala | 9 +++
.../scala/stream/table/AggregationsITCase.scala | 31 +---
...ProcessingOverRangeProcessFunctionTest.scala | 23 --
7 files changed, 175 insertions(+), 77 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/9be5cc42/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index ee6fced..c6e3c9a 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -266,7 +266,7 @@ class CodeGenerator(
outputArity: Int)
: GeneratedAggregationsFunction = {
-def generateSetAggregationResults(
+def genSetAggregationResults(
accTypes: Array[String],
aggs: Array[String],
aggMapping: Array[Int]): String = {
@@ -293,7 +293,7 @@ class CodeGenerator(
| }""".stripMargin
}
-def generateAccumulate(
+def genAccumulate(
accTypes: Array[String],
aggs: Array[String],
parameters: Array[String]): String = {
@@ -317,7 +317,7 @@ class CodeGenerator(
| }""".stripMargin
}
-def generateRetract(
+def genRetract(
accTypes: Array[String],
aggs: Array[String],
parameters: Array[String]): String = {
@@ -341,7 +341,7 @@ class CodeGenerator(
| }""".stripMargin
}
-def generateCreateAccumulators(
+def genCreateAccumulators(
aggs: Array[String]): String = {
val sig: String =
@@ -373,7 +373,7 @@ class CodeGenerator(
| }""".stripMargin
}
-def generateSetForwardedFields(
+def genSetForwardedFields(
forwardMapping: Array[(Int, Int)]): String = {
val sig: String =
@@ -396,13 +396,68 @@ class CodeGenerator(
| }""".stripMargin
}
-def generateCreateOutputRow(outputArity: Int): String = {
+def genCreateOutputRow(outputArity: Int): String = {
j"""
| public org.apache.flink.types.Row createOutputRow() {
|return new org.apache.flink.types.Row($outputArity);
| }""".stripMargin
}
+def genMergeAccumulatorsPair(
+accTypes: Array[String],
+aggs: Array[String]): String = {
+
+ val sig: String =
+j"""
+ | public org.apache.flink.types.Row mergeAccumulatorsPair(
+ |org.apache.flink.types.Row a,
+ |org.apache.flink.types.Row b)
+ """.stripMargin
+ val merge: String = {
+for (i <- aggs.indices) yield
+ j"""
+ |${accTypes(i)} aAcc$i = (${accTypes(i)}) a.getField($i);
+ |${accTypes(i)} bAcc$i = (${accTypes(i)}) b.getField($i);
+ |accList$i.set(0, aAcc$i);
+ |accList$i.set(1, bAcc$i);
+ |a.setField(
+ | $i,
+ | ${aggs(i)}.merge(accList$i));
+ """.stripMargin
+ }.mkString("\n")
+ val ret: String =
+j"""
+ | return a;
+ """.stripMargin
+
+ j"""$sig {
+ |$merge
+ |$ret
+ | }""".stripMargin
+}
+
+def genMergeList(accTypes: Array[String]): String = {
+ {
+for (i <- accTypes.indices) yield
+ j"""
+ |java.util.ArrayList<${accTypes(i)}> accList$i;
+ """.stripMargin
+ }.mkString("\n")
+}
+
+def initMergeList(
+accTypes: Array[String],
+aggs: Array[String]): String = {
+ {
+for (i <- accTypes.indices) yield
flink git commit: [FLINK-6090] [table] Add RetractionRules for annotating AccMode to DataStreamRel nodes.
Repository: flink
Updated Branches:
refs/heads/table-retraction 07a59ae0e -> 5ba0f02a6
[FLINK-6090] [table] Add RetractionRules for annotating AccMode to
DataStreamRel nodes.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5ba0f02a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5ba0f02a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5ba0f02a
Branch: refs/heads/table-retraction
Commit: 5ba0f02a6e574daf206a9ee764153782f6401c97
Parents: 07a59ae
Author: hequn.chq
Authored: Fri Apr 7 13:12:04 2017 +0800
Committer: Fabian Hueske
Committed: Mon Apr 17 21:56:50 2017 +0200
--
.../nodes/datastream/DataStreamCorrelate.scala | 6 +-
.../datastream/DataStreamGroupAggregate.scala | 6 +
.../DataStreamGroupWindowAggregate.scala| 4 +
.../datastream/DataStreamOverAggregate.scala| 4 +
.../plan/nodes/datastream/DataStreamRel.scala | 17 +
.../datastream/DataStreamRetractionRules.scala | 247 ++
.../nodes/datastream/retractionTraitDefs.scala | 81 +
.../nodes/datastream/retractionTraits.scala | 100 ++
.../flink/table/plan/rules/FlinkRuleSets.scala | 5 +
.../flink/table/CalciteConfigBuilderTest.scala | 31 +-
.../table/plan/rules/RetractionRulesTest.scala | 334 +++
11 files changed, 817 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/5ba0f02a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
index dd799e6..9ea413a 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCorrelate.scala
@@ -36,14 +36,14 @@ import org.apache.flink.types.Row
class DataStreamCorrelate(
cluster: RelOptCluster,
traitSet: RelTraitSet,
-inputNode: RelNode,
+input: RelNode,
scan: LogicalTableFunctionScan,
condition: Option[RexNode],
relRowType: RelDataType,
joinRowType: RelDataType,
joinType: SemiJoinType,
ruleDescription: String)
- extends SingleRel(cluster, traitSet, inputNode)
+ extends SingleRel(cluster, traitSet, input)
with CommonCorrelate
with DataStreamRel {
@@ -84,7 +84,7 @@ class DataStreamCorrelate(
val config = tableEnv.getConfig
// we do not need to specify input type
-val inputDS =
inputNode.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+val inputDS =
getInput.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
val funcRel = scan.asInstanceOf[LogicalTableFunctionScan]
val rexCall = funcRel.getCall.asInstanceOf[RexCall]
http://git-wip-us.apache.org/repos/asf/flink/blob/5ba0f02a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
index c2d4fb7..d4aa33c 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -57,6 +57,12 @@ class DataStreamGroupAggregate(
override def deriveRowType() = rowRelDataType
+ override def needsUpdatesAsRetraction = true
+
+ override def producesUpdates = true
+
+ override def consumesRetractions = true
+
override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]):
RelNode = {
new DataStreamGroupAggregate(
cluster,
http://git-wip-us.apache.org/repos/asf/flink/blob/5ba0f02a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index a0c1dec..c78e8bd 100644
---
a/flink-libraries/flink-table
flink git commit: [FLINK-6279] [table] Fix toString() of TableSourceScan.
Repository: flink
Updated Branches:
refs/heads/master 0038da415 -> 697cc9610
[FLINK-6279] [table] Fix toString() of TableSourceScan.
- The digest of VolcanoRuleMatch matched different table sources with same
field names as the same.
This closes #3699.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/697cc961
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/697cc961
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/697cc961
Branch: refs/heads/master
Commit: 697cc96106846547ff856aa5e478fee037ffde1a
Parents: 0038da4
Author: godfreyhe
Authored: Fri Apr 7 21:41:29 2017 +0800
Committer: Fabian Hueske
Committed: Fri Apr 7 21:42:52 2017 +0200
--
.../table/plan/nodes/TableSourceScan.scala | 10 ++-
.../apache/flink/table/TableSourceTest.scala| 29
2 files changed, 38 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/697cc961/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/TableSourceScan.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/TableSourceScan.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/TableSourceScan.scala
index e0f7786..7bd5c5b 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/TableSourceScan.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/TableSourceScan.scala
@@ -55,7 +55,15 @@ abstract class TableSourceScan(
}
override def toString: String = {
-s"Source(from: (${getRowType.getFieldNames.asScala.toList.mkString(",
")}))"
+val tableName = getTable.getQualifiedName
+val s = s"table:$tableName,
fields:(${getRowType.getFieldNames.asScala.toList.mkString(", ")})"
+
+val sourceDesc = tableSource.explainSource()
+if (sourceDesc.nonEmpty) {
+ s"Scan($s, source:$sourceDesc)"
+} else {
+ s"Scan($s)"
+}
}
def copy(traitSet: RelTraitSet, tableSource: TableSource[_]): TableSourceScan
http://git-wip-us.apache.org/repos/asf/flink/blob/697cc961/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableSourceTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableSourceTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableSourceTest.scala
index 1866e3c..24a32de 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableSourceTest.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableSourceTest.scala
@@ -31,6 +31,35 @@ class TableSourceTest extends TableTestBase {
private val projectedFields: Array[String] = Array("last", "id", "score")
private val noCalcFields: Array[String] = Array("id", "score", "first")
+ @Test
+ def testTableSourceScanToString(): Unit = {
+val (tableSource1, _) = filterableTableSource
+val (tableSource2, _) = filterableTableSource
+val util = batchTestUtil()
+val tEnv = util.tEnv
+
+tEnv.registerTableSource("table1", tableSource1)
+tEnv.registerTableSource("table2", tableSource2)
+
+val table1 = tEnv.scan("table1").where("amount > 2")
+val table2 = tEnv.scan("table2").where("amount > 2")
+val result = table1.unionAll(table2)
+
+val expected = binaryNode(
+ "DataSetUnion",
+ batchFilterableSourceTableNode(
+"table1",
+Array("name", "id", "amount", "price"),
+"'amount > 2"),
+ batchFilterableSourceTableNode(
+"table2",
+Array("name", "id", "amount", "price"),
+"'amount > 2"),
+ term("union", "name, id, amount, price")
+)
+util.verifyTable(result, expected)
+ }
+
// batch plan
@Test
[2/2] flink git commit: [FLINK-5545] [table] Remove FlinkAggregateExpandDistinctAggregatesRule after bumping Calcite to v1.12.
[FLINK-5545] [table] Remove FlinkAggregateExpandDistinctAggregatesRule after
bumping Calcite to v1.12.
This closes #3695.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0038da41
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0038da41
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0038da41
Branch: refs/heads/master
Commit: 0038da41553908a427dd20be75838cccb48c6bcf
Parents: fa7907a
Author: Kurt Young
Authored: Fri Apr 7 17:46:06 2017 +0800
Committer: Fabian Hueske
Committed: Fri Apr 7 14:09:56 2017 +0200
--
...nkAggregateExpandDistinctAggregatesRule.java | 1158 --
.../flink/table/plan/rules/FlinkRuleSets.scala |8 +-
2 files changed, 3 insertions(+), 1163 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/0038da41/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateExpandDistinctAggregatesRule.java
--
diff --git
a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateExpandDistinctAggregatesRule.java
b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateExpandDistinctAggregatesRule.java
deleted file mode 100644
index 9d4e08e..000
---
a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateExpandDistinctAggregatesRule.java
+++ /dev/null
@@ -1,1158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.calcite.rules;
-
-import org.apache.calcite.plan.Contexts;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.core.JoinRelType;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.rel.logical.LogicalAggregate;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.sql.fun.SqlCountAggFunction;
-import org.apache.calcite.sql.fun.SqlMinMaxAggFunction;
-import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.sql.fun.SqlSumAggFunction;
-import org.apache.calcite.sql.fun.SqlSumEmptyIsZeroAggFunction;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.tools.RelBuilder;
-import org.apache.calcite.tools.RelBuilderFactory;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.apache.calcite.util.ImmutableIntList;
-import org.apache.calcite.util.Pair;
-import org.apache.calcite.util.Util;
-
-import org.apache.flink.util.Preconditions;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-/**
- Copy calcite's AggregateExpandDistinctAggregatesRule to Flink project,
- and do a quick fix to avoid some bad case mentioned in CALCITE-1558.
- Should drop it and use calcite's AggregateExpandDistinctAggregatesRule
- when we upgrade to calcite 1.12(above)
- */
-
-/**
- * Planner rule that expands distinct aggregates
- * (such as {@code COUNT(DISTINCT x)}) from a
- * {@link org.apache.calcite.rel.logical.LogicalAggregate}.
- *
- * How this is done depends upon the arguments to the function. If all
- * functions have the same argument
- * (e.g. {@code COUNT(DISTINCT x), SUM(DISTINCT x)} both
[1/2] flink git commit: [FLINK-6012] [table] Support SQL WindowStart and WindowEnd functions.
Repository: flink
Updated Branches:
refs/heads/master 635394751 -> 0038da415
[FLINK-6012] [table] Support SQL WindowStart and WindowEnd functions.
This closes #3693.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fa7907ab
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fa7907ab
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fa7907ab
Branch: refs/heads/master
Commit: fa7907ab0e90d182b6386802c97f9b4e001dc440
Parents: 6353947
Author: Haohui Mai
Authored: Fri Apr 7 01:08:12 2017 -0700
Committer: Fabian Hueske
Committed: Fri Apr 7 14:07:05 2017 +0200
--
docs/dev/table_api.md | 67 +-
.../flink/table/plan/rules/FlinkRuleSets.scala | 5 +-
.../common/WindowStartEndPropertiesRule.scala | 122 +++
.../flink/table/validate/FunctionCatalog.scala | 8 +-
.../scala/batch/sql/WindowAggregateTest.scala | 38 --
.../scala/stream/sql/WindowAggregateTest.scala | 42 ---
6 files changed, 252 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/fa7907ab/docs/dev/table_api.md
--
diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 6f96920..2a838c7 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1439,7 +1439,7 @@ Group windows are defined in the `GROUP BY` clause of a
SQL query. Just like que
TUMBLE(time_attr, interval)
- Defines are tumbling time window. A tumbling time window assigns
rows to non-overlapping, continuous windows with a fixed duration
(interval). For example, a tumbling window of 5 minutes groups
rows in 5 minutes intervals. Tumbling windows can be defined on event-time
(stream + batch) or processing-time (stream).
+ Defines a tumbling time window. A tumbling time window assigns rows
to non-overlapping, continuous windows with a fixed duration
(interval). For example, a tumbling window of 5 minutes groups
rows in 5 minutes intervals. Tumbling windows can be defined on event-time
(stream + batch) or processing-time (stream).
HOP(time_attr, interval, interval)
@@ -1454,6 +1454,40 @@ Group windows are defined in the `GROUP BY` clause of a
SQL query. Just like que
For SQL queries on streaming tables, the `time_attr` argument of the group
window function must be one of the `rowtime()` or `proctime()` time-indicators,
which distinguish between event or processing time, respectively. For SQL on
batch tables, the `time_attr` argument of the group window function must be an
attribute of type `TIMESTAMP`.
+ Selecting Group Window Start and End Timestamps
+
+The start and end timestamps of group windows can be selected with the
following auxiliary functions:
+
+
+
+
+ Auxiliary Function
+ Description
+
+
+
+
+
+
+TUMBLE_START(time_attr, interval)
+HOP_START(time_attr, interval, interval)
+SESSION_START(time_attr, interval)
+
+ Returns the start timestamp of the corresponding tumbling, hopping,
and session window.
+
+
+
+TUMBLE_END(time_attr, interval)
+HOP_END(time_attr, interval, interval)
+SESSION_END(time_attr, interval)
+
+ Returns the end timestamp of the corresponding tumbling, hopping,
and session window.
+
+
+
+
+Note that the auxiliary functions must be called with exactly same arguments
as the group window function in the `GROUP BY` clause.
+
The following examples show how to specify SQL queries with group windows on
streaming tables.
@@ -1469,7 +1503,10 @@ tableEnv.registerDataStream("Orders", ds, "user,
product, amount");
// compute SUM(amount) per day (in event-time)
Table result1 = tableEnv.sql(
- "SELECT user, SUM(amount) FROM Orders GROUP BY TUMBLE(rowtime(), INTERVAL
'1' DAY), user");
+ "SELECT user, " +
+ " TUMBLE_START(rowtime(), INTERVAL '1' DAY) as wStart, " +
+ " SUM(amount) FROM Orders " +
+ "GROUP BY TUMBLE(rowtime(), INTERVAL '1' DAY), user");
// compute SUM(amount) per day (in processing-time)
Table result2 = tableEnv.sql(
@@ -1481,7 +1518,12 @@ Table result3 = tableEnv.sql(
// compute SUM(amount) per session with 12 hour inactivity gap (in event-time)
Table result4 = tableEnv.sql(
- "SELECT user, SUM(amount) FROM Orders GROUP BY SESSION(rowtime(), INTERVAL
'12' HOUR), user");
+ "SELECT user, " +
+ " SESSION_START(rowtime(), INTERVAL '12' HOUR) AS sStart, " +
+ " SESSION_END(rowtime(), INTERVAL '12' HOUR) AS snd, " +
+ " SUM(amount) " +
+ "FROM Orders " +
+ "GROUP BY SESSION(rowtime(), INTERVAL '12' HOUR), user");
{% endhighlight %}
@@ -1498,7 +1540,14 @@ tableEnv.registerDataStream("Orders", ds, 'user,
'product, 'amount
[05/11] flink git commit: [FLINK-5376] Fix log statement in UnorderedStreamElementQueue
[FLINK-5376] Fix log statement in UnorderedStreamElementQueue
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/153d73a6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/153d73a6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/153d73a6
Branch: refs/heads/table-retraction
Commit: 153d73a6f4ac51dcc02ccdb734d548855f1a1e45
Parents: ae17718
Author: zentol
Authored: Thu Apr 6 13:05:11 2017 +0200
Committer: zentol
Committed: Thu Apr 6 19:35:50 2017 +0200
--
.../api/operators/async/queue/UnorderedStreamElementQueue.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/153d73a6/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
--
diff --git
a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
index 396dbe8..f2c78f8 100644
---
a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
+++
b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java
@@ -123,7 +123,7 @@ public class UnorderedStreamElementQueue implements
StreamElementQueue {
if (numberEntries < capacity) {
addEntry(streamElementQueueEntry);
- LOG.debug("Put element into ordered stream
element queue. New filling degree " +
+ LOG.debug("Put element into unordered stream
element queue. New filling degree " +
"({}/{}).", numberEntries, capacity);
return true;
[11/11] flink git commit: [FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
[FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
This closes #3646.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/07a59ae0
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/07a59ae0
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/07a59ae0
Branch: refs/heads/table-retraction
Commit: 07a59ae0ef0b2343b8e93e8b6c06f6d668206411
Parents: 6353947
Author: shaoxuan-wang
Authored: Thu Mar 30 03:57:58 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 6 23:08:41 2017 +0200
--
.../flink/table/plan/logical/operators.scala| 3 -
.../nodes/datastream/DataStreamAggregate.scala | 267 ---
.../datastream/DataStreamGroupAggregate.scala | 133 +
.../DataStreamGroupWindowAggregate.scala| 267 +++
.../flink/table/plan/rules/FlinkRuleSets.scala | 3 +-
.../datastream/DataStreamAggregateRule.scala| 76 --
.../DataStreamGroupAggregateRule.scala | 77 ++
.../DataStreamGroupWindowAggregateRule.scala| 75 ++
.../table/runtime/aggregate/AggregateUtil.scala | 46 +++-
.../aggregate/GroupAggProcessFunction.scala | 100 +++
.../scala/batch/table/FieldProjectionTest.scala | 4 +-
.../table/api/scala/stream/sql/SqlITCase.scala | 21 ++
.../scala/stream/sql/WindowAggregateTest.scala | 50 ++--
.../scala/stream/table/AggregationsITCase.scala | 167
.../stream/table/GroupAggregationsITCase.scala | 132 +
.../stream/table/GroupAggregationsTest.scala| 218 +++
.../table/GroupWindowAggregationsITCase.scala | 167
.../scala/stream/table/GroupWindowTest.scala| 56 ++--
.../scala/stream/table/UnsupportedOpsTest.scala | 7 -
19 files changed, 1286 insertions(+), 583 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/07a59ae0/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index 559bd75..7438082 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -221,9 +221,6 @@ case class Aggregate(
}
override def validate(tableEnv: TableEnvironment): LogicalNode = {
-if (tableEnv.isInstanceOf[StreamTableEnvironment]) {
- failValidation(s"Aggregate on stream tables is currently not supported.")
-}
val resolvedAggregate = super.validate(tableEnv).asInstanceOf[Aggregate]
val groupingExprs = resolvedAggregate.groupingExpressions
http://git-wip-us.apache.org/repos/asf/flink/blob/07a59ae0/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
deleted file mode 100644
index 50f8281..000
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.plan.nodes.datastream
-
-import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
-import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.core.AggregateCall
-import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
-import org.apache.flink.api.java.tuple.Tuple
-import org.apache.flink.streaming.api.datastream.{AllWindowedStream,
DataStream, KeyedStr
[09/11] flink git commit: [FLINK-6261] [table] Support TUMBLE, HOP, SESSION group window functions for SQL queries on batch tables.
[FLINK-6261] [table] Support TUMBLE, HOP, SESSION group window functions for
SQL queries on batch tables.
- Drop support for group window translation of "GROUP BY FLOOR/CEIL".
This closes #3675.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/63539475
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/63539475
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/63539475
Branch: refs/heads/table-retraction
Commit: 635394751dce6e532fcd5a758c3d1bdb25303712
Parents: e2a4f47
Author: Fabian Hueske
Authored: Tue Apr 4 15:19:25 2017 +0200
Committer: Fabian Hueske
Committed: Thu Apr 6 22:37:45 2017 +0200
--
docs/dev/table_api.md | 90 +--
.../flink/table/plan/rules/FlinkRuleSets.scala | 7 +-
.../common/LogicalWindowAggregateRule.scala | 144 +++
.../DataSetLogicalWindowAggregateRule.scala | 92 +++
.../DataStreamLogicalWindowAggregateRule.scala | 112 +
.../datastream/LogicalWindowAggregateRule.scala | 222 -
.../scala/batch/sql/AggregationsITCase.scala| 92 ++-
.../scala/batch/sql/WindowAggregateTest.scala | 244 +++
.../scala/stream/sql/WindowAggregateTest.scala | 86 ---
9 files changed, 762 insertions(+), 327 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/63539475/docs/dev/table_api.md
--
diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 7156393..6f96920 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1418,45 +1418,103 @@ val result2 = tableEnv.sql(
- Group windows
+ Limitations
+
+Joins, set operations, and non-windowed aggregations are not supported yet.
+
+{% top %}
+
+### Group Windows
-Streaming SQL supports aggregation on group windows by specifying the windows
in the `GROUP BY` clause. The following table describes the syntax of the group
windows:
+Group windows are defined in the `GROUP BY` clause of a SQL query. Just like
queries with regular `GROUP BY` clauses, queries with a `GROUP BY` clause that
includes a group window function compute a single result row per group. The
following group windows functions are supported for SQL on batch and streaming
tables.
- GROUP BY clause
+ Group Window Function
Description
- TUMBLE(mode, interval)
- A tumbling window over the time period specified by
interval.
+ TUMBLE(time_attr, interval)
+ Defines are tumbling time window. A tumbling time window assigns
rows to non-overlapping, continuous windows with a fixed duration
(interval). For example, a tumbling window of 5 minutes groups
rows in 5 minutes intervals. Tumbling windows can be defined on event-time
(stream + batch) or processing-time (stream).
- HOP(mode, slide, size)
- A sliding window with the length of size and moves
every slide.
+ HOP(time_attr, interval, interval)
+ Defines a hopping time window (called sliding window in the Table
API). A hopping time window has a fixed duration (second interval
parameter) and hops by a specified hop interval (first interval
parameter). If the hop interval is smaller than the window size, hopping
windows are overlapping. Thus, rows can be assigned to multiple windows. For
example, a hopping window of 15 minutes size and 5 minute hop interval assigns
each row to 3 different windows of 15 minute size, which are evaluated in an
interval of 5 minutes. Hopping windows can be defined on event-time (stream +
batch) or processing-time (stream).
- SESSION(mode, gap)
- A session window that has gap as the gap between two
windows.
+ SESSION(time_attr, interval)
+ Defines a session time window. Session time windows do not have a
fixed duration but their bounds are defined by a time interval of
inactivity, i.e., a session window is closed if no event appears for a defined
gap period. For example a session window with a 30 minute gap starts when a row
is observed after 30 minutes inactivity (otherwise the row would be added to an
existing window) and is closed if no row is added within 30 minutes. Session
windows can work on event-time (stream + batch) or processing-time
(stream).
-The parameters `interval`, `slide`, `size`, `gap` must be constant time
intervals. The `mode` can be either `proctime()` or `rowtime()`, which
specifies the window is over the processing time or the event time.
+For SQL queries on streaming tables, the `time_attr` argument of the group
window function must be one of the `rowtime()` or `proctime()` time-indicators,
which distinguish between event or processing time, respectively. For SQL on
batch tables,
[01/11] flink git commit: [FLINK-6268] [core] Object reuse for Either type [Forced Update!]
Repository: flink
Updated Branches:
refs/heads/table-retraction ff2625089 -> 07a59ae0e (forced update)
[FLINK-6268] [core] Object reuse for Either type
Implement object reuse for Flink's Either type by storing a reference to
Right in Left and Left in Right. These references are private and remain
null until set by EitherSerializer when copying or deserializing with
object reuse.
This closes #3680
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ae17718f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ae17718f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ae17718f
Branch: refs/heads/table-retraction
Commit: ae17718fb9262f65b89b4551d2aecaab3ee23d9e
Parents: c5173fa
Author: Greg Hogan
Authored: Wed Apr 5 10:12:04 2017 -0400
Committer: Greg Hogan
Committed: Thu Apr 6 12:55:49 2017 -0400
--
.../typeutils/runtime/EitherSerializer.java | 51 +++-
.../java/org/apache/flink/types/Either.java | 87 -
.../typeutils/runtime/EitherSerializerTest.java | 129 +--
3 files changed, 227 insertions(+), 40 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/ae17718f/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
--
diff --git
a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
index 4066e9a..d9018da 100644
---
a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
+++
b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java
@@ -51,7 +51,7 @@ public class EitherSerializer extends
TypeSerializer> {
@Override
public boolean isImmutableType() {
- return leftSerializer.isImmutableType() &&
rightSerializer.isImmutableType();
+ return false;
}
@Override
@@ -91,23 +91,16 @@ public class EitherSerializer extends
TypeSerializer> {
@Override
public Either copy(Either from, Either reuse) {
- if (from.isRight()) {
- final R right = from.right();
- if (reuse.isRight()) {
- R copyRight = rightSerializer.copy(right,
reuse.right());
- return Right(copyRight);
- }
- else {
- // if the reuse record isn't a right value, we
cannot reuse
- R copyRight = rightSerializer.copy(right);
- return Right(copyRight);
- }
- }
- else {
- L left = from.left();
- // reuse record is never a left value because we always
create a right instance
- L copyLeft = leftSerializer.copy(left);
- return Left(copyLeft);
+ if (from.isLeft()) {
+ Left to = Either.obtainLeft(reuse,
leftSerializer);
+ L left = leftSerializer.copy(from.left(), to.left());
+ to.setValue(left);
+ return to;
+ } else {
+ Right to = Either.obtainRight(reuse,
rightSerializer);
+ R right = rightSerializer.copy(from.right(),
to.right());
+ to.setValue(right);
+ return to;
}
}
@@ -142,18 +135,16 @@ public class EitherSerializer extends
TypeSerializer> {
@Override
public Either deserialize(Either reuse, DataInputView
source) throws IOException {
boolean isLeft = source.readBoolean();
- if (!isLeft) {
- if (reuse.isRight()) {
- return
Right(rightSerializer.deserialize(reuse.right(), source));
- }
- else {
- // if the reuse record isn't a right value, we
cannot reuse
- return
Right(rightSerializer.deserialize(source));
- }
- }
- else {
- // reuse record is never a left value because we always
create a right instance
- return Left(leftSerializer.deserialize(source));
+ if (isLeft) {
+ Left to = Either.obtainLeft(reuse,
leftSerializer);
+ L left = leftSerializer.deserialize(to.left(), source);
+ to.setValue(left);
+
[06/11] flink git commit: [FLINK-6270] port some memory and network task manager options to ConfigOption
[FLINK-6270] port some memory and network task manager options to ConfigOption
This closes #3683.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e2a4f47e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e2a4f47e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e2a4f47e
Branch: refs/heads/table-retraction
Commit: e2a4f47ed8c95c7045f79bf9fe59cab39518710b
Parents: d1d761e
Author: Nico Kruber
Authored: Wed Apr 5 11:45:57 2017 +0200
Committer: zentol
Committed: Thu Apr 6 19:35:50 2017 +0200
--
.../kafka/KafkaShortRetentionTestBase.java | 3 +-
.../connectors/kafka/KafkaTestBase.java | 3 +-
.../flink/storm/api/FlinkLocalCluster.java | 3 +-
.../flink/configuration/ConfigConstants.java| 44
.../flink/configuration/TaskManagerOptions.java | 43 +++
.../ContaineredTaskManagerParameters.java | 12 ++
.../io/network/buffer/NetworkBufferPool.java| 4 +-
.../partition/SpillableSubpartition.java| 11 +++--
.../minicluster/MiniClusterConfiguration.java | 32 +-
.../TaskManagerServicesConfiguration.java | 34 +++
.../minicluster/LocalFlinkMiniCluster.scala | 20 -
.../PartialConsumePipelinedResultTest.java | 3 +-
.../runtime/jobmanager/JobManagerTest.java | 4 +-
.../TaskCancelAsyncProducerConsumerITCase.java | 5 ++-
.../TaskManagerProcessReapingTestBase.java | 5 ++-
.../taskmanager/TaskManagerStartupTest.java | 11 ++---
.../runtime/taskmanager/TaskManagerTest.java| 2 +-
.../runtime/testutils/TaskManagerProcess.java | 10 ++---
.../runtime/testingUtils/TestingUtils.scala | 4 +-
.../Flip6LocalStreamEnvironment.java| 4 +-
.../api/environment/LocalStreamEnvironment.java | 3 +-
.../apache/flink/test/util/TestBaseUtils.java | 3 +-
.../accumulators/AccumulatorErrorITCase.java| 3 +-
.../test/cancelling/CancelingTestBase.java | 5 ++-
...tractEventTimeWindowCheckpointingITCase.java | 3 +-
.../EventTimeAllWindowCheckpointingITCase.java | 3 +-
.../test/checkpointing/SavepointITCase.java | 3 +-
.../StreamFaultToleranceTestBase.java | 3 +-
.../WindowCheckpointingITCase.java | 3 +-
.../JobSubmissionFailsITCase.java | 3 +-
.../test/manual/NotSoMiniClusterIterations.java | 7 ++--
.../manual/StreamingScalabilityAndLatency.java | 5 ++-
.../test/misc/CustomSerializationITCase.java| 3 +-
.../test/misc/MiscellaneousIssuesITCase.java| 3 +-
...SuccessAfterNetworkBuffersFailureITCase.java | 5 ++-
.../query/AbstractQueryableStateITCase.java | 3 +-
...ctTaskManagerProcessFailureRecoveryTest.java | 5 ++-
...agerHAProcessFailureBatchRecoveryITCase.java | 5 ++-
.../TaskManagerFailureRecoveryITCase.java | 3 +-
.../flink/test/runtime/IPv6HostnamesITCase.java | 3 +-
.../test/streaming/runtime/TimestampITCase.java | 3 +-
.../flink/test/web/WebFrontendITCase.java | 3 +-
42 files changed, 209 insertions(+), 128 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/e2a4f47e/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java
--
diff --git
a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java
b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java
index 1e85370..954dc7d 100644
---
a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java
+++
b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java
@@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.TypeInfoParser;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.TaskManagerOptions;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
@@ -98,7 +99,7 @@ public class KafkaShortRetentionTestBase implements
Serializable {
// start also a re-usable Flink mini cluster
flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
-
flinkConfig.setInteg
[03/11] flink git commit: [FLINK-6079] [kafka] Provide meaningful error message if TopicPartitions are null
[FLINK-6079] [kafka] Provide meaningful error message if TopicPartitions are
null
This closes #3685.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8890a8db
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8890a8db
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8890a8db
Branch: refs/heads/table-retraction
Commit: 8890a8db41c45504aa658a1942f40bb9af7dcf30
Parents: a6355ed
Author: zentol
Authored: Thu Apr 6 11:55:29 2017 +0200
Committer: zentol
Committed: Thu Apr 6 19:35:50 2017 +0200
--
.../kafka/FlinkKafkaConsumerBase.java | 114 +--
1 file changed, 57 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/8890a8db/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
--
diff --git
a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
index d409027..a35e710 100644
---
a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
+++
b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
@@ -44,6 +44,7 @@ import
org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher;
import
org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition;
import
org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionStateSentinel;
import
org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema;
+import org.apache.flink.util.Preconditions;
import org.apache.flink.util.SerializedValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -348,71 +349,70 @@ public abstract class FlinkKafkaConsumerBase extends
RichParallelSourceFuncti
// initialize subscribed partitions
List kafkaTopicPartitions =
getKafkaPartitions(topics);
+ Preconditions.checkNotNull(kafkaTopicPartitions,
"TopicPartitions must not be null.");
subscribedPartitionsToStartOffsets = new
HashMap<>(kafkaTopicPartitions.size());
- if (kafkaTopicPartitions != null) {
- if (restoredState != null) {
- for (KafkaTopicPartition kafkaTopicPartition :
kafkaTopicPartitions) {
- if
(restoredState.containsKey(kafkaTopicPartition)) {
-
subscribedPartitionsToStartOffsets.put(kafkaTopicPartition,
restoredState.get(kafkaTopicPartition));
- }
+ if (restoredState != null) {
+ for (KafkaTopicPartition kafkaTopicPartition :
kafkaTopicPartitions) {
+ if
(restoredState.containsKey(kafkaTopicPartition)) {
+
subscribedPartitionsToStartOffsets.put(kafkaTopicPartition,
restoredState.get(kafkaTopicPartition));
}
+ }
- LOG.info("Consumer subtask {} will start
reading {} partitions with offsets in restored state: {}",
-
getRuntimeContext().getIndexOfThisSubtask(),
subscribedPartitionsToStartOffsets.size(), subscribedPartitionsToStartOffsets);
- } else {
- initializeSubscribedPartitionsToStartOffsets(
- subscribedPartitionsToStartOffsets,
- kafkaTopicPartitions,
-
getRuntimeContext().getIndexOfThisSubtask(),
-
getRuntimeContext().getNumberOfParallelSubtasks(),
- startupMode,
- specificStartupOffsets);
-
- if (subscribedPartitionsToStartOffsets.size()
!= 0) {
- switch (startupMode) {
- case EARLIEST:
- LOG.info("Consumer
subtask {} will start reading the following {} partitions from the earliest
offsets: {}",
-
getRuntimeContext().getIndexOfThisSubtask(),
-
subscribedPartitionsToStar
[04/11] flink git commit: [FLINK-6269] Make UserCodeClassLoader final
[FLINK-6269] Make UserCodeClassLoader final
This closes #3682.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2cb60043
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2cb60043
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2cb60043
Branch: refs/heads/table-retraction
Commit: 2cb60043d8fce4a291fbefeddce6598b442b8f01
Parents: b3ac693
Author: zcb
Authored: Thu Apr 6 15:31:46 2017 +0800
Committer: zentol
Committed: Thu Apr 6 19:35:50 2017 +0200
--
.../scala/org/apache/flink/runtime/jobmanager/JobManager.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/2cb60043/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
--
diff --git
a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 1e6d8d3..f2ecde5 100644
---
a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++
b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -1250,7 +1250,7 @@ class JobManager(
"Cannot set up the user code libraries: " + t.getMessage, t)
}
-var userCodeLoader =
libraryCacheManager.getClassLoader(jobGraph.getJobID)
+val userCodeLoader =
libraryCacheManager.getClassLoader(jobGraph.getJobID)
if (userCodeLoader == null) {
throw new JobSubmissionException(jobId,
"The user code class loader could not be initialized.")
[08/11] flink git commit: [FLINK-6256] Fix outputTag variable name in Side Output docs
[FLINK-6256] Fix outputTag variable name in Side Output docs
This closes #3684.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a6355edd
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a6355edd
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a6355edd
Branch: refs/heads/table-retraction
Commit: a6355edd904c2efe2ee37118132b0add5a2e5588
Parents: 153d73a
Author: zentol
Authored: Thu Apr 6 11:45:56 2017 +0200
Committer: zentol
Committed: Thu Apr 6 19:35:50 2017 +0200
--
docs/dev/stream/side_output.md | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/a6355edd/docs/dev/stream/side_output.md
--
diff --git a/docs/dev/stream/side_output.md b/docs/dev/stream/side_output.md
index 3633b75..e4c4c19 100644
--- a/docs/dev/stream/side_output.md
+++ b/docs/dev/stream/side_output.md
@@ -41,13 +41,13 @@ side output stream:
{% highlight java %}
// this needs to be an anonymous inner class, so that we can analyze the type
-OutputTag outputTag = new OutputTag("string-side-output") {};
+OutputTag outputTag = new OutputTag("side-output") {};
{% endhighlight %}
{% highlight scala %}
-val outputTag = OutputTag[String]("string-side-output")
+val outputTag = OutputTag[String]("side-output")
{% endhighlight %}
@@ -79,7 +79,7 @@ SingleOutputStreamOperator mainDataStream = input
out.collect(value);
// emit data to side output
-ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
+ctx.output(outputTag, "sideout-" + String.valueOf(value));
}
});
{% endhighlight %}
@@ -90,7 +90,7 @@ SingleOutputStreamOperator mainDataStream = input
{% highlight scala %}
val input: DataStream[Int] = ...
-val outputTag = OutputTag[String]("string-side-output")
+val outputTag = OutputTag[String]("side-output")
val mainDataStream = input
.process(new ProcessFunction[Int, Int] {
@@ -128,7 +128,7 @@ DataStream sideOutputStream =
mainDataStream.getSideOutput(outputTag);
{% highlight scala %}
-val outputTag = OutputTag[String]("string-side-output")
+val outputTag = OutputTag[String]("side-output")
val mainDataStream = ...
[07/11] flink git commit: [FLINK-6270] extend Configuration with contains(configOption)
[FLINK-6270] extend Configuration with contains(configOption)
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d1d761ed
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d1d761ed
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d1d761ed
Branch: refs/heads/table-retraction
Commit: d1d761ed6b4de050402905b8c196e781479245a0
Parents: 2cb6004
Author: Nico Kruber
Authored: Wed Apr 5 10:59:00 2017 +0200
Committer: zentol
Committed: Thu Apr 6 19:35:50 2017 +0200
--
.../flink/configuration/Configuration.java | 30
.../configuration/DelegatingConfiguration.java | 5
2 files changed, 35 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/d1d761ed/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java
--
diff --git
a/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java
b/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java
index 8f23435..ea0c419 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java
@@ -595,6 +595,36 @@ public class Configuration extends
ExecutionConfig.GlobalJobParameters
}
}
+ /**
+* Checks whether there is an entry for the given config option
+*
+* @param configOption The configuration option
+*
+* @return true if a valid (current or deprecated) key of the
config option is stored,
+* false otherwise
+*/
+ @PublicEvolving
+ public boolean contains(ConfigOption configOption) {
+ synchronized (this.confData){
+ // first try the current key
+ if (this.confData.containsKey(configOption.key())) {
+ return true;
+ }
+ else if (configOption.hasDeprecatedKeys()) {
+ // try the deprecated keys
+ for (String deprecatedKey :
configOption.deprecatedKeys()) {
+ if
(this.confData.containsKey(deprecatedKey)) {
+ LOG.warn("Config uses
deprecated configuration key '{}' instead of proper key '{}'",
+ deprecatedKey,
configOption.key());
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+ }
+
//
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/d1d761ed/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
--
diff --git
a/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
b/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
index bd9a962..1b14e9e 100644
---
a/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
+++
b/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
@@ -290,6 +290,11 @@ public final class DelegatingConfiguration extends
Configuration {
return backingConfig.containsKey(prefix + key);
}
+ @Override
+ public boolean contains(ConfigOption configOption) {
+ return backingConfig.contains(prefixOption(configOption,
prefix));
+ }
+
//
@Override
[10/11] flink git commit: [FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
http://git-wip-us.apache.org/repos/asf/flink/blob/07a59ae0/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
index fde7682..c3e 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
@@ -181,11 +181,11 @@ class GroupWindowTest extends TableTestBase {
.select('string.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -222,7 +222,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -251,7 +251,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -279,7 +279,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -299,7 +299,7 @@ class GroupWindowTest extends TableTestBase {
}
@Test
- @Ignore // see comments in DataStreamAggregate
+ @Ignore // see comments in DataStreamGroupWindowAggregate
def testEventTimeTumblingGroupWindowOverCount(): Unit = {
val util = streamTestUtil()
val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
@@ -310,7 +310,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
streamTableNode(0),
term("groupBy", "string"),
term(
@@ -335,7 +335,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -364,7 +364,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -393,7 +393,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -412,7 +412,7 @@ class GroupWindowTest extends TableTestBase {
}
@Test
- @Ignore // see comments in DataStreamAggregate
+ @Ignore // see comments in DataStreamGroupWindowAggregate
def testEventTimeSlidingGroupWindowOverCount(): Unit = {
val util = streamTestUtil()
val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
@@ -423,7 +423,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
streamTableNode(0),
term("groupBy", "string"),
term(
@@ -448,7 +448,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -477,7 +477,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -506,7 +506,7 @@ class GroupWindowTest extends TableTestBase {
.select('int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -534,7 +534,7 @@ class Gr
[02/11] flink git commit: [hotfix] [jdbc] Add generic parameter to ResultTypeQueryable
[hotfix] [jdbc] Add generic parameter to ResultTypeQueryable
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b3ac6932
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b3ac6932
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b3ac6932
Branch: refs/heads/table-retraction
Commit: b3ac693200aacd6f8a581e25056067fabe0ef356
Parents: 8890a8d
Author: zentol
Authored: Thu Apr 6 13:11:35 2017 +0200
Committer: zentol
Committed: Thu Apr 6 19:35:50 2017 +0200
--
.../java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/b3ac6932/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
--
diff --git
a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
index 3153f96..e714867 100644
---
a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
+++
b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormat.java
@@ -95,7 +95,7 @@ import org.slf4j.LoggerFactory;
* @see PreparedStatement
* @see DriverManager
*/
-public class JDBCInputFormat extends RichInputFormat
implements ResultTypeQueryable {
+public class JDBCInputFormat extends RichInputFormat
implements ResultTypeQueryable {
private static final long serialVersionUID = 1L;
private static final Logger LOG =
LoggerFactory.getLogger(JDBCInputFormat.class);
flink git commit: [FLINK-6261] [table] Support TUMBLE, HOP, SESSION group window functions for SQL queries on batch tables.
Repository: flink
Updated Branches:
refs/heads/master e2a4f47ed -> 635394751
[FLINK-6261] [table] Support TUMBLE, HOP, SESSION group window functions for
SQL queries on batch tables.
- Drop support for group window translation of "GROUP BY FLOOR/CEIL".
This closes #3675.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/63539475
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/63539475
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/63539475
Branch: refs/heads/master
Commit: 635394751dce6e532fcd5a758c3d1bdb25303712
Parents: e2a4f47
Author: Fabian Hueske
Authored: Tue Apr 4 15:19:25 2017 +0200
Committer: Fabian Hueske
Committed: Thu Apr 6 22:37:45 2017 +0200
--
docs/dev/table_api.md | 90 +--
.../flink/table/plan/rules/FlinkRuleSets.scala | 7 +-
.../common/LogicalWindowAggregateRule.scala | 144 +++
.../DataSetLogicalWindowAggregateRule.scala | 92 +++
.../DataStreamLogicalWindowAggregateRule.scala | 112 +
.../datastream/LogicalWindowAggregateRule.scala | 222 -
.../scala/batch/sql/AggregationsITCase.scala| 92 ++-
.../scala/batch/sql/WindowAggregateTest.scala | 244 +++
.../scala/stream/sql/WindowAggregateTest.scala | 86 ---
9 files changed, 762 insertions(+), 327 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/63539475/docs/dev/table_api.md
--
diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 7156393..6f96920 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -1418,45 +1418,103 @@ val result2 = tableEnv.sql(
- Group windows
+ Limitations
+
+Joins, set operations, and non-windowed aggregations are not supported yet.
+
+{% top %}
+
+### Group Windows
-Streaming SQL supports aggregation on group windows by specifying the windows
in the `GROUP BY` clause. The following table describes the syntax of the group
windows:
+Group windows are defined in the `GROUP BY` clause of a SQL query. Just like
queries with regular `GROUP BY` clauses, queries with a `GROUP BY` clause that
includes a group window function compute a single result row per group. The
following group windows functions are supported for SQL on batch and streaming
tables.
- GROUP BY clause
+ Group Window Function
Description
- TUMBLE(mode, interval)
- A tumbling window over the time period specified by
interval.
+ TUMBLE(time_attr, interval)
+ Defines are tumbling time window. A tumbling time window assigns
rows to non-overlapping, continuous windows with a fixed duration
(interval). For example, a tumbling window of 5 minutes groups
rows in 5 minutes intervals. Tumbling windows can be defined on event-time
(stream + batch) or processing-time (stream).
- HOP(mode, slide, size)
- A sliding window with the length of size and moves
every slide.
+ HOP(time_attr, interval, interval)
+ Defines a hopping time window (called sliding window in the Table
API). A hopping time window has a fixed duration (second interval
parameter) and hops by a specified hop interval (first interval
parameter). If the hop interval is smaller than the window size, hopping
windows are overlapping. Thus, rows can be assigned to multiple windows. For
example, a hopping window of 15 minutes size and 5 minute hop interval assigns
each row to 3 different windows of 15 minute size, which are evaluated in an
interval of 5 minutes. Hopping windows can be defined on event-time (stream +
batch) or processing-time (stream).
- SESSION(mode, gap)
- A session window that has gap as the gap between two
windows.
+ SESSION(time_attr, interval)
+ Defines a session time window. Session time windows do not have a
fixed duration but their bounds are defined by a time interval of
inactivity, i.e., a session window is closed if no event appears for a defined
gap period. For example a session window with a 30 minute gap starts when a row
is observed after 30 minutes inactivity (otherwise the row would be added to an
existing window) and is closed if no row is added within 30 minutes. Session
windows can work on event-time (stream + batch) or processing-time
(stream).
-The parameters `interval`, `slide`, `size`, `gap` must be constant time
intervals. The `mode` can be either `proctime()` or `rowtime()`, which
specifies the window is over the processing time or the event time.
+For SQL queries on streaming tables, the `time_attr` argument of the group
window function must be one of the `rowtime()` or `proctime()` time-indicators,
which distinguish bet
[03/12] flink git commit: [hotfix] [py] Fix PythonCoGroup useByteArray check
[hotfix] [py] Fix PythonCoGroup useByteArray check
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8f78e9d7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8f78e9d7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8f78e9d7
Branch: refs/heads/table-retraction
Commit: 8f78e9d72f494c4eb773211beb30a7116d048109
Parents: 89d9dec
Author: zentol
Authored: Thu Mar 30 23:13:50 2017 +0200
Committer: zentol
Committed: Thu Apr 6 10:57:10 2017 +0200
--
.../java/org/apache/flink/python/api/functions/PythonCoGroup.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/8f78e9d7/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
--
diff --git
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
index 72d3361..ff5a8d4 100644
---
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
+++
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
@@ -12,6 +12,7 @@
*/
package org.apache.flink.python.api.functions;
+import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.python.api.streaming.data.PythonDualInputStreamer;
@@ -36,7 +37,7 @@ public class PythonCoGroup extends
RichCoGroupFunction
typeInformation) {
this.typeInformation = typeInformation;
- streamer = new PythonDualInputStreamer<>(this, envID, setID,
true);
+ streamer = new PythonDualInputStreamer<>(this, envID, setID,
typeInformation instanceof PrimitiveArrayTypeInfo);
}
/**
[09/12] flink git commit: [FLINK-5435] [table] Remove FlinkAggregateJoinTransposeRule and FlinkRelDecorrelator after bumping Calcite to v1.12.
http://git-wip-us.apache.org/repos/asf/flink/blob/c5173fa2/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/sql2rel/FlinkRelDecorrelator.java -- diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/sql2rel/FlinkRelDecorrelator.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/sql2rel/FlinkRelDecorrelator.java deleted file mode 100644 index 0179192..000 --- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/sql2rel/FlinkRelDecorrelator.java +++ /dev/null @@ -1,2216 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.flink.table.calcite.sql2rel; - -import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; -import com.google.common.collect.Sets; -import com.google.common.collect.SortedSetMultimap; -import org.apache.calcite.linq4j.Ord; -import org.apache.calcite.linq4j.function.Function2; -import org.apache.calcite.plan.Context; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptCostImpl; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.hep.HepPlanner; -import org.apache.calcite.plan.hep.HepProgram; -import org.apache.calcite.plan.hep.HepRelVertex; -import org.apache.calcite.rel.BiRel; -import org.apache.calcite.rel.RelCollation; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelShuttleImpl; -import org.apache.calcite.rel.core.Aggregate; -import org.apache.calcite.rel.core.AggregateCall; -import org.apache.calcite.rel.core.Correlate; -import org.apache.calcite.rel.core.CorrelationId; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.core.Sort; -import org.apache.calcite.rel.core.Values; -import org.apache.calcite.rel.logical.LogicalAggregate; -import org.apache.calcite.rel.logical.LogicalCorrelate; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.logical.LogicalJoin; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.logical.LogicalSort; -import org.apache.calcite.rel.metadata.RelMdUtil; -import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.rules.FilterCorrelateRule; -import org.apache.calcite.rel.rules.FilterJoinRule; -import org.apache.calcite.rel.rules.FilterProjectTransposeRule; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexCorrelVariable; -import org.apache.calcite.rex.RexFieldAccess; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexShuttle; -import org.apache.calcite.rex.RexSubQuery; -import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.rex.RexVisitorImpl; -import org.apache.calcite.sql.SqlExplainLevel; -import org.apache.calcite.sql.SqlFunction; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.fun.SqlCountAggFunction; -import org.apache.calcite.sql.fun.SqlSingleValueAggFunction; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.util.Bug; -import org.apache.calcite.util.Holder; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Litmus; -import org.apache.calcite.util.Pair; -import o
[02/12] flink git commit: [FLINK-6223] [py] Rework PythonPlanBinder generics
[FLINK-6223] [py] Rework PythonPlanBinder generics
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bba49d67
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bba49d67
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bba49d67
Branch: refs/heads/table-retraction
Commit: bba49d673f9a36d4291bc74f07895e957de75bce
Parents: 8f78e9d
Author: zentol
Authored: Thu Mar 30 23:45:14 2017 +0200
Committer: zentol
Committed: Thu Apr 6 10:57:10 2017 +0200
--
.../flink/python/api/PythonPlanBinder.java | 339 +--
.../apache/flink/python/api/util/SetCache.java | 204 +++
2 files changed, 367 insertions(+), 176 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/bba49d67/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
--
diff --git
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
index a3cae4a..7c228e1 100644
---
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
+++
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
@@ -17,11 +17,12 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.Random;
import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.operators.Keys;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.LocalEnvironment;
@@ -29,12 +30,11 @@ import org.apache.flink.api.java.io.PrintingOutputFormat;
import org.apache.flink.api.java.io.TupleCsvInputFormat;
import org.apache.flink.api.java.operators.CoGroupRawOperator;
import org.apache.flink.api.java.operators.CrossOperator.DefaultCross;
-import org.apache.flink.api.java.operators.Grouping;
-import org.apache.flink.api.common.operators.Keys;
import org.apache.flink.api.java.operators.SortedGrouping;
import org.apache.flink.api.java.operators.UdfOperator;
import org.apache.flink.api.java.operators.UnsortedGrouping;
import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.GlobalConfiguration;
@@ -53,6 +53,7 @@ import
org.apache.flink.python.api.functions.util.KeyDiscarder;
import org.apache.flink.python.api.functions.util.SerializerMap;
import org.apache.flink.python.api.functions.util.StringDeserializerMap;
import org.apache.flink.python.api.streaming.plan.PythonPlanStreamer;
+import org.apache.flink.python.api.util.SetCache;
import org.apache.flink.runtime.filecache.FileCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,7 +93,7 @@ public class PythonPlanBinder {
private static String FLINK_HDFS_PATH = "hdfs:/tmp";
public static final String FLINK_TMP_DATA_DIR =
System.getProperty("java.io.tmpdir") + File.separator + "flink_data";
- private HashMap sets = new HashMap<>();
+ private final SetCache sets = new SetCache();
public ExecutionEnvironment env;
private int currentEnvironmentID = 0;
private PythonPlanStreamer streamer;
@@ -242,7 +243,7 @@ public class PythonPlanBinder {
private void receivePlan() throws IOException {
env = ExecutionEnvironment.getExecutionEnvironment();
//IDs used in HashMap of sets are only unique for each
environment
- sets.clear();
+ sets.reset();
receiveParameters();
receiveOperations();
}
@@ -263,18 +264,18 @@ public class PythonPlanBinder {
Tuple value = (Tuple) streamer.getRecord(true);
switch (Parameters.valueOf(((String)
value.getField(0)).toUpperCase())) {
case DOP:
- Integer dop = (Integer)
value.getField(1);
+ Integer dop =
value.getField(1);
env.setParallelism(dop);
break;
case MODE:
- FLINK_HDFS_PATH = (Boolean)
value.getField(1) ? "file:/tmp/flink" : "hdfs:/tmp/flink";
+
[11/12] flink git commit: [FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
http://git-wip-us.apache.org/repos/asf/flink/blob/ff262508/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
index fde7682..c3e 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
@@ -181,11 +181,11 @@ class GroupWindowTest extends TableTestBase {
.select('string.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -222,7 +222,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -251,7 +251,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -279,7 +279,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -299,7 +299,7 @@ class GroupWindowTest extends TableTestBase {
}
@Test
- @Ignore // see comments in DataStreamAggregate
+ @Ignore // see comments in DataStreamGroupWindowAggregate
def testEventTimeTumblingGroupWindowOverCount(): Unit = {
val util = streamTestUtil()
val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
@@ -310,7 +310,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
streamTableNode(0),
term("groupBy", "string"),
term(
@@ -335,7 +335,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -364,7 +364,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -393,7 +393,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -412,7 +412,7 @@ class GroupWindowTest extends TableTestBase {
}
@Test
- @Ignore // see comments in DataStreamAggregate
+ @Ignore // see comments in DataStreamGroupWindowAggregate
def testEventTimeSlidingGroupWindowOverCount(): Unit = {
val util = streamTestUtil()
val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
@@ -423,7 +423,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
streamTableNode(0),
term("groupBy", "string"),
term(
@@ -448,7 +448,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -477,7 +477,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -506,7 +506,7 @@ class GroupWindowTest extends TableTestBase {
.select('int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -534,7 +534,7 @@ class Gr
[10/12] flink git commit: [FLINK-5435] [table] Remove FlinkAggregateJoinTransposeRule and FlinkRelDecorrelator after bumping Calcite to v1.12.
[FLINK-5435] [table] Remove FlinkAggregateJoinTransposeRule and
FlinkRelDecorrelator after bumping Calcite to v1.12.
This closes #3689.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c5173fa2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c5173fa2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c5173fa2
Branch: refs/heads/table-retraction
Commit: c5173fa26d3d8a32b0b182a37d34a8eeff6e36d0
Parents: 07f1b03
Author: Kurt Young
Authored: Thu Apr 6 22:06:51 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 6 16:34:02 2017 +0200
--
.../rules/FlinkAggregateJoinTransposeRule.java | 358 ---
.../calcite/sql2rel/FlinkRelDecorrelator.java | 2216 --
.../flink/table/calcite/FlinkPlannerImpl.scala |8 +-
.../flink/table/plan/rules/FlinkRuleSets.scala |4 +-
.../batch/sql/QueryDecorrelationTest.scala | 123 +-
.../api/scala/batch/sql/SetOperatorsTest.scala | 32 +-
6 files changed, 36 insertions(+), 2705 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/c5173fa2/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
--
diff --git
a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
deleted file mode 100644
index a817c91..000
---
a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.table.calcite.rules;
-
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import org.apache.calcite.linq4j.Ord;
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.JoinRelType;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.rel.logical.LogicalAggregate;
-import org.apache.calcite.rel.logical.LogicalJoin;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.sql.SqlSplittableAggFunction;
-import org.apache.calcite.tools.RelBuilder;
-import org.apache.calcite.tools.RelBuilderFactory;
-import org.apache.calcite.util.Bug;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.apache.calcite.util.Util;
-import org.apache.calcite.util.mapping.Mapping;
-import org.apache.calcite.util.mapping.Mappings;
-
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import org.apache.flink.util.Preconditions;
-
-/**
- * Copied from {@link
org.apache.calcite.rel.rules.AggregateJoinTransposeRule}, should be
- * removed once https://issues.apache.org/jira/browse/CALCITE-1544";>[CALCITE-1544] fixes.
- */
-public class FlinkAggregateJoinTransposeRule extends RelOptRule {
- public static final FlinkAggregateJoinTransposeRule INSTANCE = new
FlinkAggregateJoinTransposeRule(LogicalAggregate.class, LogicalJoin.class,
RelFactories.LOGICAL_BUILDER, false);
-
- /**
-* Extended instance of the rule that can push down aggregate functions.
-*/
- public static final FlinkAggregateJoinTransposeRule EXTE
[06/12] flink git commit: [FLINK-6257] [table] Consistent naming of ProcessFunction and methods for OVER windows.
http://git-wip-us.apache.org/repos/asf/flink/blob/07f1b035/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
new file mode 100644
index 000..525d4d7
--- /dev/null
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.aggregate
+
+import java.util
+import java.util.{List => JList}
+
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.types.Row
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.api.common.state._
+import org.apache.flink.api.java.typeutils.ListTypeInfo
+import org.apache.flink.streaming.api.operators.TimestampedCollector
+import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
+import org.slf4j.LoggerFactory
+
+
+/**
+ * A ProcessFunction to support unbounded event-time over-window
+ *
+ * @param genAggregations Generated aggregate helper function
+ * @param intermediateType the intermediate row tye which the state
saved
+ * @param inputTypethe input row tye which the state saved
+ */
+abstract class RowTimeUnboundedOver(
+genAggregations: GeneratedAggregationsFunction,
+intermediateType: TypeInformation[Row],
+inputType: TypeInformation[Row])
+ extends ProcessFunction[Row, Row]
+with Compiler[GeneratedAggregations] {
+
+ protected var output: Row = _
+ // state to hold the accumulators of the aggregations
+ private var accumulatorState: ValueState[Row] = _
+ // state to hold rows until the next watermark arrives
+ private var rowMapState: MapState[Long, JList[Row]] = _
+ // list to sort timestamps to access rows in timestamp order
+ private var sortedTimestamps: util.LinkedList[Long] = _
+
+ val LOG = LoggerFactory.getLogger(this.getClass)
+ protected var function: GeneratedAggregations = _
+
+ override def open(config: Configuration) {
+LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+s"Code:\n$genAggregations.code")
+val clazz = compile(
+ getRuntimeContext.getUserCodeClassLoader,
+ genAggregations.name,
+ genAggregations.code)
+LOG.debug("Instantiating AggregateHelper.")
+function = clazz.newInstance()
+
+output = function.createOutputRow()
+sortedTimestamps = new util.LinkedList[Long]()
+
+// initialize accumulator state
+val accDescriptor: ValueStateDescriptor[Row] =
+ new ValueStateDescriptor[Row]("accumulatorstate", intermediateType)
+accumulatorState = getRuntimeContext.getState[Row](accDescriptor)
+
+// initialize row state
+val rowListTypeInfo: TypeInformation[JList[Row]] = new
ListTypeInfo[Row](inputType)
+val mapStateDescriptor: MapStateDescriptor[Long, JList[Row]] =
+ new MapStateDescriptor[Long, JList[Row]]("rowmapstate",
+BasicTypeInfo.LONG_TYPE_INFO.asInstanceOf[TypeInformation[Long]],
rowListTypeInfo)
+rowMapState = getRuntimeContext.getMapState(mapStateDescriptor)
+ }
+
+ /**
+* Puts an element from the input stream into state if it is not late.
+* Registers a timer for the next watermark.
+*
+* @param input The input value.
+* @param ctx The ctx to register timer or get current time
+* @param out The collector for returning result values.
+*
+*/
+ override def processElement(
+ input: Row,
+ ctx: ProcessFunction[Row, Row]#Context,
+ out: Collector[Row]): Unit = {
+
+val timestamp = ctx.timestamp()
+val curWatermark = ctx.timerService().currentWatermark()
+
+// discard late record
+if (timestamp >= curWatermark)
[08/12] flink git commit: [FLINK-5435] [table] Remove FlinkAggregateJoinTransposeRule and FlinkRelDecorrelator after bumping Calcite to v1.12.
http://git-wip-us.apache.org/repos/asf/flink/blob/c5173fa2/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
index 3e44526..7496ff8 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
@@ -40,56 +40,6 @@ class QueryDecorrelationTest extends TableTestBase {
"and e1.deptno < 10 and d1.deptno < 15\n" +
"and e1.salary > (select avg(salary) from emp e2 where e1.empno =
e2.empno)"
-// the inner query "select avg(salary) from emp e2 where e1.empno =
e2.empno" will be
-// decorrelated into a join and then groupby. And the filters
-// "e1.deptno < 10 and d1.deptno < 15" will also be pushed down before
join.
-val decorrelatedSubQuery = unaryNode(
- "DataSetAggregate",
- unaryNode(
-"DataSetCalc",
-binaryNode(
- "DataSetJoin",
- unaryNode(
-"DataSetCalc",
-batchTableNode(0),
-term("select", "empno", "salary")
- ),
- unaryNode(
-"DataSetDistinct",
-unaryNode(
- "DataSetCalc",
- binaryNode(
-"DataSetJoin",
-unaryNode(
- "DataSetCalc",
- batchTableNode(0),
- term("select", "empno", "deptno"),
- term("where", "<(deptno, 10)")
-),
-unaryNode(
- "DataSetCalc",
- batchTableNode(1),
- term("select", "deptno"),
- term("where", "<(deptno, 15)")
-),
-term("where", "=(deptno, deptno0)"),
-term("join", "empno", "deptno", "deptno0"),
-term("joinType", "InnerJoin")
- ),
- term("select", "empno")
-),
-term("distinct", "empno")
- ),
- term("where", "=(empno0, empno)"),
- term("join", "empno", "salary", "empno0"),
- term("joinType", "InnerJoin")
-),
-term("select", "empno0", "salary")
- ),
- term("groupBy", "empno0"),
- term("select", "empno0", "AVG(salary) AS EXPR$0")
-)
-
val expectedQuery = unaryNode(
"DataSetCalc",
binaryNode(
@@ -112,7 +62,17 @@ class QueryDecorrelationTest extends TableTestBase {
term("join", "empno", "ename", "job", "salary", "deptno", "deptno0",
"name"),
term("joinType", "InnerJoin")
),
-decorrelatedSubQuery,
+unaryNode(
+ "DataSetAggregate",
+ unaryNode(
+"DataSetCalc",
+batchTableNode(0),
+term("select", "salary", "empno"),
+term("where", "IS NOT NULL(empno)")
+ ),
+ term("groupBy", "empno"),
+ term("select", "empno", "AVG(salary) AS EXPR$0")
+),
term("where", "AND(=(empno, empno0), >(salary, EXPR$0))"),
term("join", "empno", "ename", "job", "salary", "deptno",
"deptno0", "name", "empno0", "EXPR$0"),
@@ -132,51 +92,6 @@ class QueryDecorrelationTest extends TableTestBase {
"select avg(e2.salary) from emp e2 where e2.deptno = d1.deptno" +
")"
-val decorrelatedSubQuery = unaryNode(
- "DataSetAggregate",
- unaryNode(
-"DataSetCalc",
-binaryNode(
- "DataSetJoin",
- unaryNode(
-"DataSetCalc",
-batchTableNode(0),
-term("select", "salary", "deptno")
- ),
- unaryNode(
-"DataSetDistinct",
-unaryNode(
- "DataSetCalc",
- binaryNode(
-"DataSetJoin",
-unaryNode(
- "DataSetCalc",
- batchTableNode(0),
- term("select", "deptno")
-),
-unaryNode(
- "DataSetCalc",
- batchTableNode(1),
- term("select", "deptno")
-),
-term("where", "=(deptno, deptno0)"),
-term("join", "deptno", "deptno0"),
-term("joinType", "InnerJoin")
- ),
- term("select", "deptno0")
-),
-term("distinct", "deptno0")
- ),
- term("where", "=(deptno, deptno0)"),
- term("join", "salary", "deptno", "deptno0"),
- term("joinType", "InnerJoin")
-),
-
[05/12] flink git commit: [FLINK-6229] [py] Rework configuration of PythonPlanBinder/Operators
[FLINK-6229] [py] Rework configuration of PythonPlanBinder/Operators
- unify python2/python3 configuration
- explicitly pass on a configuration to each operator
- port all configuration options to ConfigOptions
- [FLINK-5516] Make all paths explicitly configurable
- [FLINK-6230] Make mmap size configurable
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bdcebfda
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bdcebfda
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bdcebfda
Branch: refs/heads/table-retraction
Commit: bdcebfda06846a1e21bb6a4678909d503ebc6333
Parents: 940d16c
Author: zentol
Authored: Fri Mar 31 11:55:18 2017 +0200
Committer: zentol
Committed: Thu Apr 6 10:57:11 2017 +0200
--
docs/dev/batch/python.md| 10 +-
flink-dist/src/main/flink-bin/bin/pyflink.bat | 25
flink-dist/src/main/flink-bin/bin/pyflink.sh| 25
flink-dist/src/main/flink-bin/bin/pyflink2.bat | 25
flink-dist/src/main/flink-bin/bin/pyflink2.sh | 25
flink-dist/src/main/flink-bin/bin/pyflink3.bat | 25
flink-dist/src/main/flink-bin/bin/pyflink3.sh | 26
.../apache/flink/python/api/PythonOptions.java | 74 ++
.../flink/python/api/PythonPlanBinder.java | 143 +++
.../python/api/functions/PythonCoGroup.java | 4 +-
.../api/functions/PythonMapPartition.java | 4 +-
.../streaming/data/PythonDualInputSender.java | 6 +
.../streaming/data/PythonDualInputStreamer.java | 5 +-
.../api/streaming/data/PythonReceiver.java | 29 ++--
.../python/api/streaming/data/PythonSender.java | 36 ++---
.../streaming/data/PythonSingleInputSender.java | 6 +
.../data/PythonSingleInputStreamer.java | 9 +-
.../api/streaming/data/PythonStreamer.java | 39 ++---
.../api/streaming/plan/PythonPlanStreamer.java | 12 +-
.../python/api/flink/connection/Connection.py | 21 ++-
.../api/flink/functions/CoGroupFunction.py | 4 +-
.../python/api/flink/functions/Function.py | 4 +-
.../api/flink/functions/GroupReduceFunction.py | 4 +-
.../api/flink/functions/ReduceFunction.py | 4 +-
.../flink/python/api/flink/plan/Environment.py | 5 +-
.../flink/python/api/PythonPlanBinderTest.java | 12 +-
26 files changed, 322 insertions(+), 260 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/bdcebfda/docs/dev/batch/python.md
--
diff --git a/docs/dev/batch/python.md b/docs/dev/batch/python.md
index 09a4fa8..c4c2671 100644
--- a/docs/dev/batch/python.md
+++ b/docs/dev/batch/python.md
@@ -149,8 +149,7 @@ Apart from setting up Flink, no additional work is
required. The python package
The Python API was tested on Linux/Windows systems that have Python 2.7 or 3.4
installed.
-By default Flink will start python processes by calling "python" or "python3",
depending on which start-script
-was used. By setting the "python.binary.python[2/3]" key in the
flink-conf.yaml you can modify this behaviour to use a binary of your choice.
+By default Flink will start python processes by calling "python". By setting
the "python.binary.path" key in the flink-conf.yaml you can modify this
behaviour to use a binary of your choice.
{% top %}
@@ -624,12 +623,11 @@ Executing Plans
---
To run the plan with Flink, go to your Flink distribution, and run the
pyflink.sh script from the /bin folder.
-use pyflink2.sh for python 2.7, and pyflink3.sh for python 3.4. The script
containing the plan has to be passed
-as the first argument, followed by a number of additional python packages, and
finally, separated by - additional
-arguments that will be fed to the script.
+The script containing the plan has to be passed as the first argument,
followed by a number of additional python
+packages, and finally, separated by - additional arguments that will be fed to
the script.
{% highlight python %}
-./bin/pyflink<2/3>.sh
[01/12] flink git commit: [hotfix] [py] Code cleanup - PythonPlanBinder [Forced Update!]
Repository: flink
Updated Branches:
refs/heads/table-retraction 448cb333b -> ff2625089 (forced update)
[hotfix] [py] Code cleanup - PythonPlanBinder
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/940d16c7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/940d16c7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/940d16c7
Branch: refs/heads/table-retraction
Commit: 940d16c77731a8558b13cd9b3406862229e4bd4f
Parents: bba49d6
Author: zentol
Authored: Thu Mar 30 23:46:44 2017 +0200
Committer: zentol
Committed: Thu Apr 6 10:57:10 2017 +0200
--
.../flink/python/api/PythonPlanBinder.java | 45 ++--
1 file changed, 22 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/940d16c7/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
--
diff --git
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
index 7c228e1..733a6fb 100644
---
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
+++
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
@@ -12,13 +12,6 @@
*/
package org.apache.flink.python.api;
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.Random;
-
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.operators.Keys;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
@@ -40,24 +33,32 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.GlobalConfiguration;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.python.api.functions.util.NestedKeyDiscarder;
-import org.apache.flink.python.api.functions.util.StringTupleDeserializerMap;
import org.apache.flink.python.api.PythonOperationInfo.DatasizeHint;
-import static
org.apache.flink.python.api.PythonOperationInfo.DatasizeHint.HUGE;
-import static
org.apache.flink.python.api.PythonOperationInfo.DatasizeHint.NONE;
-import static
org.apache.flink.python.api.PythonOperationInfo.DatasizeHint.TINY;
import org.apache.flink.python.api.functions.PythonCoGroup;
-import org.apache.flink.python.api.functions.util.IdentityGroupReduce;
import org.apache.flink.python.api.functions.PythonMapPartition;
+import org.apache.flink.python.api.functions.util.IdentityGroupReduce;
import org.apache.flink.python.api.functions.util.KeyDiscarder;
+import org.apache.flink.python.api.functions.util.NestedKeyDiscarder;
import org.apache.flink.python.api.functions.util.SerializerMap;
import org.apache.flink.python.api.functions.util.StringDeserializerMap;
+import org.apache.flink.python.api.functions.util.StringTupleDeserializerMap;
import org.apache.flink.python.api.streaming.plan.PythonPlanStreamer;
import org.apache.flink.python.api.util.SetCache;
import org.apache.flink.runtime.filecache.FileCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Random;
+
+import static
org.apache.flink.python.api.PythonOperationInfo.DatasizeHint.HUGE;
+import static
org.apache.flink.python.api.PythonOperationInfo.DatasizeHint.NONE;
+import static
org.apache.flink.python.api.PythonOperationInfo.DatasizeHint.TINY;
+
/**
* This class allows the execution of a Flink plan written in python.
*/
@@ -116,7 +117,7 @@ public class PythonPlanBinder {
binder.runPlan(Arrays.copyOfRange(args, 1, args.length));
}
- public PythonPlanBinder() throws IOException {
+ public PythonPlanBinder() {
Configuration conf = GlobalConfiguration.loadConfiguration();
FLINK_PYTHON2_BINARY_PATH =
conf.getString(FLINK_PYTHON2_BINARY_KEY, "python");
FLINK_PYTHON3_BINARY_PATH =
conf.getString(FLINK_PYTHON3_BINARY_KEY, "python3");
@@ -164,6 +165,7 @@ public class PythonPlanBinder {
}
//=Setup
+
/**
* Copies all files to a common directory (FLINK_PYTHON_FILE_PATH).
This allows us to distribute it as one big
* package, and resolves PYTHONPATH issues.
@@ -186,7 +188,7 @@ public class PythonPlanBinder {
}
}
- private static voi
[04/12] flink git commit: [FLINK-6229] [py] Rework setup of PythonPlanBinder
[FLINK-6229] [py] Rework setup of PythonPlanBinder
- make file/argument split more readable
- pass on Paths where applicable instead of recreating them every time
- rename PPB#clearPath to more appropriate deleteIfExists
- simplify PPB#copyFile
- simplify PPB#startPython
- use UUID#randomUUID() instead of Random.nextInt()
- remove several invalid exception declarations
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5ff9c99f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5ff9c99f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5ff9c99f
Branch: refs/heads/table-retraction
Commit: 5ff9c99ff193725c125f2b4c450411db97b6f3b4
Parents: bdcebfd
Author: zentol
Authored: Fri Mar 31 12:11:40 2017 +0200
Committer: zentol
Committed: Thu Apr 6 10:57:11 2017 +0200
--
.../flink/python/api/PythonPlanBinder.java | 283 ---
1 file changed, 124 insertions(+), 159 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/5ff9c99f/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
--
diff --git
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
index b6181b4..2378d60 100644
---
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
+++
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/PythonPlanBinder.java
@@ -49,9 +49,8 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.net.URISyntaxException;
import java.util.Arrays;
-import java.util.Random;
+import java.util.UUID;
import static
org.apache.flink.python.api.PythonOperationInfo.DatasizeHint.HUGE;
import static
org.apache.flink.python.api.PythonOperationInfo.DatasizeHint.NONE;
@@ -68,9 +67,6 @@ public class PythonPlanBinder {
public static final String PLANBINDER_CONFIG_BCVAR_COUNT =
"PLANBINDER_BCVAR_COUNT";
public static final String PLANBINDER_CONFIG_BCVAR_NAME_PREFIX =
"PLANBINDER_BCVAR_";
-
- private static final Random r = new Random();
-
public static final String PLAN_ARGUMENTS_KEY = "python.plan.arguments";
private static final String FLINK_PYTHON_REL_LOCAL_PATH =
File.separator + "resources" + File.separator + "python";
@@ -80,10 +76,9 @@ public class PythonPlanBinder {
private final String pythonLibraryPath;
private final String tmpPlanFilesDir;
- private String tmpDistributedDir;
+ private Path tmpDistributedDir;
private final SetCache sets = new SetCache();
- public ExecutionEnvironment env;
private int currentEnvironmentID = 0;
private PythonPlanStreamer streamer;
@@ -108,9 +103,9 @@ public class PythonPlanBinder {
String configuredPlanTmpPath =
globalConfig.getString(PythonOptions.PLAN_TMP_DIR);
tmpPlanFilesDir = configuredPlanTmpPath != null
? configuredPlanTmpPath
- : System.getProperty("java.io.tmpdir") + File.separator
+ "flink_plan_" + r.nextInt();
+ : System.getProperty("java.io.tmpdir") + File.separator
+ "flink_plan_" + UUID.randomUUID();
- tmpDistributedDir =
globalConfig.getString(PythonOptions.DC_TMP_DIR);
+ tmpDistributedDir = new
Path(globalConfig.getString(PythonOptions.DC_TMP_DIR));
String flinkRootDir = System.getenv("FLINK_ROOT_DIR");
pythonLibraryPath = flinkRootDir != null
@@ -131,121 +126,95 @@ public class PythonPlanBinder {
void runPlan(String[] args) throws Exception {
int split = 0;
for (int x = 0; x < args.length; x++) {
- if (args[x].compareTo("-") == 0) {
+ if (args[x].equals("-")) {
split = x;
+ break;
}
}
try {
- String tmpPath = tmpPlanFilesDir;
- prepareFiles(tmpPath, Arrays.copyOfRange(args, 0, split
== 0 ? args.length : split));
- startPython(tmpPath, Arrays.copyOfRange(args, split ==
0 ? args.length : split + 1, args.length));
+ String planFile = args[0];
+ String[] filesToCopy = Arrays.copyOfRange(args, 1,
split == 0 ? args.length : split);
+ String[] planArgumentsArray = Arrays.copyOfRange(args,
split == 0 ? args.length : split + 1, args.length);
+
+
[07/12] flink git commit: [FLINK-6257] [table] Consistent naming of ProcessFunction and methods for OVER windows.
[FLINK-6257] [table] Consistent naming of ProcessFunction and methods for OVER
windows.
- Add check for sort order of OVER windows.
This closes #3681.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/07f1b035
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/07f1b035
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/07f1b035
Branch: refs/heads/table-retraction
Commit: 07f1b035ffbf07d160503c48e2c58a464ec5d014
Parents: 5ff9c99
Author: sunjincheng121
Authored: Thu Apr 6 10:33:30 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 6 16:33:45 2017 +0200
--
.../datastream/DataStreamOverAggregate.scala| 196 +
.../table/runtime/aggregate/AggregateUtil.scala | 122 +++-
...ndedProcessingOverRangeProcessFunction.scala | 183
...oundedProcessingOverRowProcessFunction.scala | 179
.../aggregate/ProcTimeBoundedRangeOver.scala| 182
.../aggregate/ProcTimeBoundedRowsOver.scala | 179
.../ProcTimeUnboundedNonPartitionedOver.scala | 96 ++
.../ProcTimeUnboundedPartitionedOver.scala | 84 ++
.../RangeClauseBoundedOverProcessFunction.scala | 201 -
.../aggregate/RowTimeBoundedRangeOver.scala | 200 +
.../aggregate/RowTimeBoundedRowsOver.scala | 222 ++
.../aggregate/RowTimeUnboundedOver.scala| 292 +++
.../RowsClauseBoundedOverProcessFunction.scala | 222 --
.../UnboundedEventTimeOverProcessFunction.scala | 292 ---
...rtitionedProcessingOverProcessFunction.scala | 96 --
...UnboundedProcessingOverProcessFunction.scala | 84 --
...ProcessingOverRangeProcessFunctionTest.scala | 2 +-
17 files changed, 1380 insertions(+), 1452 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/07f1b035/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index 947775b..2224752 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -17,20 +17,21 @@
*/
package org.apache.flink.table.plan.nodes.datastream
+import java.util.{List => JList}
+
import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.core.AggregateCall
+import org.apache.calcite.rel.core.{AggregateCall, Window}
+import org.apache.calcite.rel.core.Window.Group
import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
+import org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING
import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.streaming.api.datastream.DataStream
import org.apache.flink.table.api.{StreamTableEnvironment, TableException}
import org.apache.flink.table.calcite.FlinkTypeFactory
-import org.apache.flink.table.runtime.aggregate._
import org.apache.flink.table.plan.nodes.OverAggregate
+import org.apache.flink.table.runtime.aggregate._
import org.apache.flink.types.Row
-import org.apache.calcite.rel.core.Window
-import org.apache.calcite.rel.core.Window.Group
-import java.util.{List => JList}
import org.apache.flink.api.java.functions.NullByteKeySelector
import org.apache.flink.table.codegen.CodeGenerator
@@ -90,12 +91,20 @@ class DataStreamOverAggregate(
val overWindow: org.apache.calcite.rel.core.Window.Group =
logicWindow.groups.get(0)
-val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+val orderKeys = overWindow.orderKeys.getFieldCollations
-if (overWindow.orderKeys.getFieldCollations.size() != 1) {
+if (orderKeys.size() != 1) {
throw new TableException(
-"Unsupported use of OVER windows. The window may only be ordered by a
single time column.")
+"Unsupported use of OVER windows. The window can only be ordered by a
single time column.")
}
+val orderKey = orderKeys.get(0)
+
+if (!orderKey.direction.equals(ASCENDING)) {
+ throw new TableException(
+"Unsupported use of OVER windows. The window can only be ordered in
ASCENDING mode.")
+}
+
+val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
val generator = new CodeGenerator(
tableEnv.getConfig,
@@ -104,78 +113,6
[12/12] flink git commit: [FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
[FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
This closes #3646.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ff262508
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ff262508
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ff262508
Branch: refs/heads/table-retraction
Commit: ff2625089e9f184326c7a8b39cbdbec35ba58869
Parents: c5173fa
Author: shaoxuan-wang
Authored: Thu Mar 30 03:57:58 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 6 21:28:57 2017 +0200
--
.../flink/table/plan/logical/operators.scala| 3 -
.../nodes/datastream/DataStreamAggregate.scala | 267 ---
.../datastream/DataStreamGroupAggregate.scala | 133 +
.../DataStreamGroupWindowAggregate.scala| 267 +++
.../flink/table/plan/rules/FlinkRuleSets.scala | 3 +-
.../datastream/DataStreamAggregateRule.scala| 76 --
.../DataStreamGroupAggregateRule.scala | 77 ++
.../DataStreamGroupWindowAggregateRule.scala| 75 ++
.../table/runtime/aggregate/AggregateUtil.scala | 46 +++-
.../aggregate/GroupAggProcessFunction.scala | 100 +++
.../scala/batch/table/FieldProjectionTest.scala | 4 +-
.../table/api/scala/stream/sql/SqlITCase.scala | 21 ++
.../scala/stream/sql/WindowAggregateTest.scala | 47 ++--
.../scala/stream/table/AggregationsITCase.scala | 167
.../stream/table/GroupAggregationsITCase.scala | 132 +
.../stream/table/GroupAggregationsTest.scala| 218 +++
.../table/GroupWindowAggregationsITCase.scala | 167
.../scala/stream/table/GroupWindowTest.scala| 56 ++--
.../scala/stream/table/UnsupportedOpsTest.scala | 7 -
19 files changed, 1288 insertions(+), 578 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/ff262508/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index 559bd75..7438082 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -221,9 +221,6 @@ case class Aggregate(
}
override def validate(tableEnv: TableEnvironment): LogicalNode = {
-if (tableEnv.isInstanceOf[StreamTableEnvironment]) {
- failValidation(s"Aggregate on stream tables is currently not supported.")
-}
val resolvedAggregate = super.validate(tableEnv).asInstanceOf[Aggregate]
val groupingExprs = resolvedAggregate.groupingExpressions
http://git-wip-us.apache.org/repos/asf/flink/blob/ff262508/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
deleted file mode 100644
index 50f8281..000
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.plan.nodes.datastream
-
-import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
-import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.core.AggregateCall
-import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
-import org.apache.flink.api.java.tuple.Tuple
-import org.apache.flink.streaming.api.datastream.{AllWindowedStream,
DataStream, KeyedStr
[4/5] flink git commit: [FLINK-5435] [table] Remove FlinkAggregateJoinTransposeRule and FlinkRelDecorrelator after bumping Calcite to v1.12.
http://git-wip-us.apache.org/repos/asf/flink/blob/c5173fa2/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/sql2rel/FlinkRelDecorrelator.java -- diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/sql2rel/FlinkRelDecorrelator.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/sql2rel/FlinkRelDecorrelator.java deleted file mode 100644 index 0179192..000 --- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/sql2rel/FlinkRelDecorrelator.java +++ /dev/null @@ -1,2216 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.flink.table.calcite.sql2rel; - -import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; -import com.google.common.collect.Sets; -import com.google.common.collect.SortedSetMultimap; -import org.apache.calcite.linq4j.Ord; -import org.apache.calcite.linq4j.function.Function2; -import org.apache.calcite.plan.Context; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptCostImpl; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.hep.HepPlanner; -import org.apache.calcite.plan.hep.HepProgram; -import org.apache.calcite.plan.hep.HepRelVertex; -import org.apache.calcite.rel.BiRel; -import org.apache.calcite.rel.RelCollation; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelShuttleImpl; -import org.apache.calcite.rel.core.Aggregate; -import org.apache.calcite.rel.core.AggregateCall; -import org.apache.calcite.rel.core.Correlate; -import org.apache.calcite.rel.core.CorrelationId; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.core.Sort; -import org.apache.calcite.rel.core.Values; -import org.apache.calcite.rel.logical.LogicalAggregate; -import org.apache.calcite.rel.logical.LogicalCorrelate; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.logical.LogicalJoin; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.logical.LogicalSort; -import org.apache.calcite.rel.metadata.RelMdUtil; -import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.rules.FilterCorrelateRule; -import org.apache.calcite.rel.rules.FilterJoinRule; -import org.apache.calcite.rel.rules.FilterProjectTransposeRule; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexCorrelVariable; -import org.apache.calcite.rex.RexFieldAccess; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexShuttle; -import org.apache.calcite.rex.RexSubQuery; -import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.rex.RexVisitorImpl; -import org.apache.calcite.sql.SqlExplainLevel; -import org.apache.calcite.sql.SqlFunction; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.fun.SqlCountAggFunction; -import org.apache.calcite.sql.fun.SqlSingleValueAggFunction; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.util.Bug; -import org.apache.calcite.util.Holder; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.Litmus; -import org.apache.calcite.util.Pair; -import o
[2/5] flink git commit: [FLINK-6257] [table] Consistent naming of ProcessFunction and methods for OVER windows.
[FLINK-6257] [table] Consistent naming of ProcessFunction and methods for OVER
windows.
- Add check for sort order of OVER windows.
This closes #3681.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/07f1b035
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/07f1b035
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/07f1b035
Branch: refs/heads/master
Commit: 07f1b035ffbf07d160503c48e2c58a464ec5d014
Parents: 5ff9c99
Author: sunjincheng121
Authored: Thu Apr 6 10:33:30 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 6 16:33:45 2017 +0200
--
.../datastream/DataStreamOverAggregate.scala| 196 +
.../table/runtime/aggregate/AggregateUtil.scala | 122 +++-
...ndedProcessingOverRangeProcessFunction.scala | 183
...oundedProcessingOverRowProcessFunction.scala | 179
.../aggregate/ProcTimeBoundedRangeOver.scala| 182
.../aggregate/ProcTimeBoundedRowsOver.scala | 179
.../ProcTimeUnboundedNonPartitionedOver.scala | 96 ++
.../ProcTimeUnboundedPartitionedOver.scala | 84 ++
.../RangeClauseBoundedOverProcessFunction.scala | 201 -
.../aggregate/RowTimeBoundedRangeOver.scala | 200 +
.../aggregate/RowTimeBoundedRowsOver.scala | 222 ++
.../aggregate/RowTimeUnboundedOver.scala| 292 +++
.../RowsClauseBoundedOverProcessFunction.scala | 222 --
.../UnboundedEventTimeOverProcessFunction.scala | 292 ---
...rtitionedProcessingOverProcessFunction.scala | 96 --
...UnboundedProcessingOverProcessFunction.scala | 84 --
...ProcessingOverRangeProcessFunctionTest.scala | 2 +-
17 files changed, 1380 insertions(+), 1452 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/07f1b035/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index 947775b..2224752 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -17,20 +17,21 @@
*/
package org.apache.flink.table.plan.nodes.datastream
+import java.util.{List => JList}
+
import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.core.AggregateCall
+import org.apache.calcite.rel.core.{AggregateCall, Window}
+import org.apache.calcite.rel.core.Window.Group
import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
+import org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING
import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.streaming.api.datastream.DataStream
import org.apache.flink.table.api.{StreamTableEnvironment, TableException}
import org.apache.flink.table.calcite.FlinkTypeFactory
-import org.apache.flink.table.runtime.aggregate._
import org.apache.flink.table.plan.nodes.OverAggregate
+import org.apache.flink.table.runtime.aggregate._
import org.apache.flink.types.Row
-import org.apache.calcite.rel.core.Window
-import org.apache.calcite.rel.core.Window.Group
-import java.util.{List => JList}
import org.apache.flink.api.java.functions.NullByteKeySelector
import org.apache.flink.table.codegen.CodeGenerator
@@ -90,12 +91,20 @@ class DataStreamOverAggregate(
val overWindow: org.apache.calcite.rel.core.Window.Group =
logicWindow.groups.get(0)
-val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
+val orderKeys = overWindow.orderKeys.getFieldCollations
-if (overWindow.orderKeys.getFieldCollations.size() != 1) {
+if (orderKeys.size() != 1) {
throw new TableException(
-"Unsupported use of OVER windows. The window may only be ordered by a
single time column.")
+"Unsupported use of OVER windows. The window can only be ordered by a
single time column.")
}
+val orderKey = orderKeys.get(0)
+
+if (!orderKey.direction.equals(ASCENDING)) {
+ throw new TableException(
+"Unsupported use of OVER windows. The window can only be ordered in
ASCENDING mode.")
+}
+
+val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(tableEnv)
val generator = new CodeGenerator(
tableEnv.getConfig,
@@ -104,78 +113,69 @@ class
[3/5] flink git commit: [FLINK-5435] [table] Remove FlinkAggregateJoinTransposeRule and FlinkRelDecorrelator after bumping Calcite to v1.12.
http://git-wip-us.apache.org/repos/asf/flink/blob/c5173fa2/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
index 3e44526..7496ff8 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/batch/sql/QueryDecorrelationTest.scala
@@ -40,56 +40,6 @@ class QueryDecorrelationTest extends TableTestBase {
"and e1.deptno < 10 and d1.deptno < 15\n" +
"and e1.salary > (select avg(salary) from emp e2 where e1.empno =
e2.empno)"
-// the inner query "select avg(salary) from emp e2 where e1.empno =
e2.empno" will be
-// decorrelated into a join and then groupby. And the filters
-// "e1.deptno < 10 and d1.deptno < 15" will also be pushed down before
join.
-val decorrelatedSubQuery = unaryNode(
- "DataSetAggregate",
- unaryNode(
-"DataSetCalc",
-binaryNode(
- "DataSetJoin",
- unaryNode(
-"DataSetCalc",
-batchTableNode(0),
-term("select", "empno", "salary")
- ),
- unaryNode(
-"DataSetDistinct",
-unaryNode(
- "DataSetCalc",
- binaryNode(
-"DataSetJoin",
-unaryNode(
- "DataSetCalc",
- batchTableNode(0),
- term("select", "empno", "deptno"),
- term("where", "<(deptno, 10)")
-),
-unaryNode(
- "DataSetCalc",
- batchTableNode(1),
- term("select", "deptno"),
- term("where", "<(deptno, 15)")
-),
-term("where", "=(deptno, deptno0)"),
-term("join", "empno", "deptno", "deptno0"),
-term("joinType", "InnerJoin")
- ),
- term("select", "empno")
-),
-term("distinct", "empno")
- ),
- term("where", "=(empno0, empno)"),
- term("join", "empno", "salary", "empno0"),
- term("joinType", "InnerJoin")
-),
-term("select", "empno0", "salary")
- ),
- term("groupBy", "empno0"),
- term("select", "empno0", "AVG(salary) AS EXPR$0")
-)
-
val expectedQuery = unaryNode(
"DataSetCalc",
binaryNode(
@@ -112,7 +62,17 @@ class QueryDecorrelationTest extends TableTestBase {
term("join", "empno", "ename", "job", "salary", "deptno", "deptno0",
"name"),
term("joinType", "InnerJoin")
),
-decorrelatedSubQuery,
+unaryNode(
+ "DataSetAggregate",
+ unaryNode(
+"DataSetCalc",
+batchTableNode(0),
+term("select", "salary", "empno"),
+term("where", "IS NOT NULL(empno)")
+ ),
+ term("groupBy", "empno"),
+ term("select", "empno", "AVG(salary) AS EXPR$0")
+),
term("where", "AND(=(empno, empno0), >(salary, EXPR$0))"),
term("join", "empno", "ename", "job", "salary", "deptno",
"deptno0", "name", "empno0", "EXPR$0"),
@@ -132,51 +92,6 @@ class QueryDecorrelationTest extends TableTestBase {
"select avg(e2.salary) from emp e2 where e2.deptno = d1.deptno" +
")"
-val decorrelatedSubQuery = unaryNode(
- "DataSetAggregate",
- unaryNode(
-"DataSetCalc",
-binaryNode(
- "DataSetJoin",
- unaryNode(
-"DataSetCalc",
-batchTableNode(0),
-term("select", "salary", "deptno")
- ),
- unaryNode(
-"DataSetDistinct",
-unaryNode(
- "DataSetCalc",
- binaryNode(
-"DataSetJoin",
-unaryNode(
- "DataSetCalc",
- batchTableNode(0),
- term("select", "deptno")
-),
-unaryNode(
- "DataSetCalc",
- batchTableNode(1),
- term("select", "deptno")
-),
-term("where", "=(deptno, deptno0)"),
-term("join", "deptno", "deptno0"),
-term("joinType", "InnerJoin")
- ),
- term("select", "deptno0")
-),
-term("distinct", "deptno0")
- ),
- term("where", "=(deptno, deptno0)"),
- term("join", "salary", "deptno", "deptno0"),
- term("joinType", "InnerJoin")
-),
-
[5/5] flink git commit: [FLINK-5435] [table] Remove FlinkAggregateJoinTransposeRule and FlinkRelDecorrelator after bumping Calcite to v1.12.
[FLINK-5435] [table] Remove FlinkAggregateJoinTransposeRule and
FlinkRelDecorrelator after bumping Calcite to v1.12.
This closes #3689.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c5173fa2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c5173fa2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c5173fa2
Branch: refs/heads/master
Commit: c5173fa26d3d8a32b0b182a37d34a8eeff6e36d0
Parents: 07f1b03
Author: Kurt Young
Authored: Thu Apr 6 22:06:51 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 6 16:34:02 2017 +0200
--
.../rules/FlinkAggregateJoinTransposeRule.java | 358 ---
.../calcite/sql2rel/FlinkRelDecorrelator.java | 2216 --
.../flink/table/calcite/FlinkPlannerImpl.scala |8 +-
.../flink/table/plan/rules/FlinkRuleSets.scala |4 +-
.../batch/sql/QueryDecorrelationTest.scala | 123 +-
.../api/scala/batch/sql/SetOperatorsTest.scala | 32 +-
6 files changed, 36 insertions(+), 2705 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/c5173fa2/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
--
diff --git
a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
deleted file mode 100644
index a817c91..000
---
a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/calcite/rules/FlinkAggregateJoinTransposeRule.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.table.calcite.rules;
-
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import org.apache.calcite.linq4j.Ord;
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.JoinRelType;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.rel.logical.LogicalAggregate;
-import org.apache.calcite.rel.logical.LogicalJoin;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.sql.SqlSplittableAggFunction;
-import org.apache.calcite.tools.RelBuilder;
-import org.apache.calcite.tools.RelBuilderFactory;
-import org.apache.calcite.util.Bug;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.apache.calcite.util.Util;
-import org.apache.calcite.util.mapping.Mapping;
-import org.apache.calcite.util.mapping.Mappings;
-
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import org.apache.flink.util.Preconditions;
-
-/**
- * Copied from {@link
org.apache.calcite.rel.rules.AggregateJoinTransposeRule}, should be
- * removed once https://issues.apache.org/jira/browse/CALCITE-1544";>[CALCITE-1544] fixes.
- */
-public class FlinkAggregateJoinTransposeRule extends RelOptRule {
- public static final FlinkAggregateJoinTransposeRule INSTANCE = new
FlinkAggregateJoinTransposeRule(LogicalAggregate.class, LogicalJoin.class,
RelFactories.LOGICAL_BUILDER, false);
-
- /**
-* Extended instance of the rule that can push down aggregate functions.
-*/
- public static final FlinkAggregateJoinTransposeRule EXTENDED = new
[1/5] flink git commit: [FLINK-6257] [table] Consistent naming of ProcessFunction and methods for OVER windows.
Repository: flink
Updated Branches:
refs/heads/master 5ff9c99ff -> c5173fa26
http://git-wip-us.apache.org/repos/asf/flink/blob/07f1b035/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
new file mode 100644
index 000..525d4d7
--- /dev/null
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.aggregate
+
+import java.util
+import java.util.{List => JList}
+
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.types.Row
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.util.{Collector, Preconditions}
+import org.apache.flink.api.common.state._
+import org.apache.flink.api.java.typeutils.ListTypeInfo
+import org.apache.flink.streaming.api.operators.TimestampedCollector
+import org.apache.flink.table.codegen.{GeneratedAggregationsFunction, Compiler}
+import org.slf4j.LoggerFactory
+
+
+/**
+ * A ProcessFunction to support unbounded event-time over-window
+ *
+ * @param genAggregations Generated aggregate helper function
+ * @param intermediateType the intermediate row tye which the state
saved
+ * @param inputTypethe input row tye which the state saved
+ */
+abstract class RowTimeUnboundedOver(
+genAggregations: GeneratedAggregationsFunction,
+intermediateType: TypeInformation[Row],
+inputType: TypeInformation[Row])
+ extends ProcessFunction[Row, Row]
+with Compiler[GeneratedAggregations] {
+
+ protected var output: Row = _
+ // state to hold the accumulators of the aggregations
+ private var accumulatorState: ValueState[Row] = _
+ // state to hold rows until the next watermark arrives
+ private var rowMapState: MapState[Long, JList[Row]] = _
+ // list to sort timestamps to access rows in timestamp order
+ private var sortedTimestamps: util.LinkedList[Long] = _
+
+ val LOG = LoggerFactory.getLogger(this.getClass)
+ protected var function: GeneratedAggregations = _
+
+ override def open(config: Configuration) {
+LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
+s"Code:\n$genAggregations.code")
+val clazz = compile(
+ getRuntimeContext.getUserCodeClassLoader,
+ genAggregations.name,
+ genAggregations.code)
+LOG.debug("Instantiating AggregateHelper.")
+function = clazz.newInstance()
+
+output = function.createOutputRow()
+sortedTimestamps = new util.LinkedList[Long]()
+
+// initialize accumulator state
+val accDescriptor: ValueStateDescriptor[Row] =
+ new ValueStateDescriptor[Row]("accumulatorstate", intermediateType)
+accumulatorState = getRuntimeContext.getState[Row](accDescriptor)
+
+// initialize row state
+val rowListTypeInfo: TypeInformation[JList[Row]] = new
ListTypeInfo[Row](inputType)
+val mapStateDescriptor: MapStateDescriptor[Long, JList[Row]] =
+ new MapStateDescriptor[Long, JList[Row]]("rowmapstate",
+BasicTypeInfo.LONG_TYPE_INFO.asInstanceOf[TypeInformation[Long]],
rowListTypeInfo)
+rowMapState = getRuntimeContext.getMapState(mapStateDescriptor)
+ }
+
+ /**
+* Puts an element from the input stream into state if it is not late.
+* Registers a timer for the next watermark.
+*
+* @param input The input value.
+* @param ctx The ctx to register timer or get current time
+* @param out The collector for returning result values.
+*
+*/
+ override def processElement(
+ input: Row,
+ ctx: ProcessFunction[Row, Row]#Context,
+ out: Collector[Row]): Unit = {
+
+val timestamp = ctx.timestamp()
+val curWatermark = ctx.timerService().cur
[2/2] flink git commit: [FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
[FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
This closes #3646.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/448cb333
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/448cb333
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/448cb333
Branch: refs/heads/table-retraction
Commit: 448cb333b446e3ad17e77116cd6f692b9068bb68
Parents: 89d9dec
Author: shaoxuan-wang
Authored: Thu Mar 30 03:57:58 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 6 09:29:32 2017 +0200
--
.../flink/table/plan/logical/operators.scala| 3 -
.../nodes/datastream/DataStreamAggregate.scala | 267 ---
.../datastream/DataStreamGroupAggregate.scala | 133 +
.../DataStreamGroupWindowAggregate.scala| 267 +++
.../flink/table/plan/rules/FlinkRuleSets.scala | 3 +-
.../datastream/DataStreamAggregateRule.scala| 76 --
.../DataStreamGroupAggregateRule.scala | 77 ++
.../DataStreamGroupWindowAggregateRule.scala| 75 ++
.../table/runtime/aggregate/AggregateUtil.scala | 46 +++-
.../aggregate/GroupAggProcessFunction.scala | 100 +++
.../scala/batch/table/FieldProjectionTest.scala | 4 +-
.../table/api/scala/stream/sql/SqlITCase.scala | 21 ++
.../scala/stream/sql/WindowAggregateTest.scala | 47 ++--
.../scala/stream/table/AggregationsITCase.scala | 167
.../stream/table/GroupAggregationsITCase.scala | 132 +
.../stream/table/GroupAggregationsTest.scala| 218 +++
.../table/GroupWindowAggregationsITCase.scala | 167
.../scala/stream/table/GroupWindowTest.scala| 56 ++--
.../scala/stream/table/UnsupportedOpsTest.scala | 7 -
19 files changed, 1288 insertions(+), 578 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/448cb333/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
index 559bd75..7438082 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/logical/operators.scala
@@ -221,9 +221,6 @@ case class Aggregate(
}
override def validate(tableEnv: TableEnvironment): LogicalNode = {
-if (tableEnv.isInstanceOf[StreamTableEnvironment]) {
- failValidation(s"Aggregate on stream tables is currently not supported.")
-}
val resolvedAggregate = super.validate(tableEnv).asInstanceOf[Aggregate]
val groupingExprs = resolvedAggregate.groupingExpressions
http://git-wip-us.apache.org/repos/asf/flink/blob/448cb333/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
deleted file mode 100644
index 50f8281..000
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamAggregate.scala
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.plan.nodes.datastream
-
-import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
-import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rel.core.AggregateCall
-import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
-import org.apache.flink.api.java.tuple.Tuple
-import org.apache.flink.streaming.api.datastream.{AllWindowedStream,
DataStream, KeyedStr
[1/2] flink git commit: [FLINK-6216] [table] Add non-windowed GroupBy aggregation for streams.
Repository: flink
Updated Branches:
refs/heads/table-retraction 89d9dec38 -> 448cb333b
http://git-wip-us.apache.org/repos/asf/flink/blob/448cb333/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
--
diff --git
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
index fde7682..c3e 100644
---
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
+++
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
@@ -181,11 +181,11 @@ class GroupWindowTest extends TableTestBase {
.select('string.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -222,7 +222,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -251,7 +251,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -279,7 +279,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -299,7 +299,7 @@ class GroupWindowTest extends TableTestBase {
}
@Test
- @Ignore // see comments in DataStreamAggregate
+ @Ignore // see comments in DataStreamGroupWindowAggregate
def testEventTimeTumblingGroupWindowOverCount(): Unit = {
val util = streamTestUtil()
val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
@@ -310,7 +310,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
streamTableNode(0),
term("groupBy", "string"),
term(
@@ -335,7 +335,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -364,7 +364,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -393,7 +393,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -412,7 +412,7 @@ class GroupWindowTest extends TableTestBase {
}
@Test
- @Ignore // see comments in DataStreamAggregate
+ @Ignore // see comments in DataStreamGroupWindowAggregate
def testEventTimeSlidingGroupWindowOverCount(): Unit = {
val util = streamTestUtil()
val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
@@ -423,7 +423,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
streamTableNode(0),
term("groupBy", "string"),
term(
@@ -448,7 +448,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -477,7 +477,7 @@ class GroupWindowTest extends TableTestBase {
.select('string, 'int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unaryNode(
"DataStreamCalc",
streamTableNode(0),
@@ -506,7 +506,7 @@ class GroupWindowTest extends TableTestBase {
.select('int.count)
val expected = unaryNode(
- "DataStreamAggregate",
+ "DataStreamGroupWindowAggregate",
unar
[01/37] flink git commit: [FLINK-5625] [kinesis] Configurable date format for timestamp-based start position in FlinkKinesisConsumer
Repository: flink
Updated Branches:
refs/heads/table-retraction ee033c903 -> 89d9dec38
[FLINK-5625] [kinesis] Configurable date format for timestamp-based start
position in FlinkKinesisConsumer
This closes #3651.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a119a30d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a119a30d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a119a30d
Branch: refs/heads/table-retraction
Commit: a119a30da91bc88b6b0242622adff2189b0a8fa4
Parents: ee033c9
Author: Tony Wei
Authored: Thu Mar 30 09:48:43 2017 +0800
Committer: Tzu-Li (Gordon) Tai
Committed: Fri Mar 31 12:32:19 2017 +0800
--
docs/dev/connectors/kinesis.md | 8 ++-
.../kinesis/config/ConsumerConfigConstants.java | 7 +-
.../kinesis/internals/ShardConsumer.java| 12 +++-
.../kinesis/util/KinesisConfigUtil.java | 23 +++---
.../kinesis/FlinkKinesisConsumerTest.java | 75
5 files changed, 95 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/a119a30d/docs/dev/connectors/kinesis.md
--
diff --git a/docs/dev/connectors/kinesis.md b/docs/dev/connectors/kinesis.md
index 59f3d61..ef1afca 100644
--- a/docs/dev/connectors/kinesis.md
+++ b/docs/dev/connectors/kinesis.md
@@ -121,9 +121,11 @@ one of the following values in the provided configuration
properties (the naming
- `LATEST`: read all shards of all streams starting from the latest record.
- `TRIM_HORIZON`: read all shards of all streams starting from the earliest
record possible (data may be trimmed by Kinesis depending on the retention
settings).
- `AT_TIMESTAMP`: read all shards of all streams starting from a specified
timestamp. The timestamp must also be specified in the configuration
-properties by providing a value for
`ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP`, either in the date pattern
-`-MM-dd'T'HH:mm:ss.SSSXXX` (for example, `2016-04-04T19:58:46.480-00:00`),
or a non-negative double value representing the number of seconds
-that has elapsed since the Unix epoch (for example, `1459799926.480`).
+properties by providing a value for
`ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP`, in one of the following
date pattern :
+- a non-negative double value representing the number of seconds that has
elapsed since the Unix epoch (for example, `1459799926.480`).
+- a user defined pattern, which is a valid pattern for `SimpleDateFormat`
provided by `ConsumerConfigConstants.STREAM_TIMESTAMP_DATE_FORMAT`.
+If `ConsumerConfigConstants.STREAM_TIMESTAMP_DATE_FORMAT` is not defined
then the default pattern will be `-MM-dd'T'HH:mm:ss.SSSXXX`
+(for example, timestamp value is `2016-04-04` and pattern is `-MM-dd`
given by user or timestamp value is `2016-04-04T19:58:46.480-00:00` without
given a pattern).
Fault Tolerance for Exactly-Once User-Defined State Update Semantics
http://git-wip-us.apache.org/repos/asf/flink/blob/a119a30d/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/config/ConsumerConfigConstants.java
--
diff --git
a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/config/ConsumerConfigConstants.java
b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/config/ConsumerConfigConstants.java
index 4ffe0ad..7c31af4 100644
---
a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/config/ConsumerConfigConstants.java
+++
b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/config/ConsumerConfigConstants.java
@@ -56,9 +56,12 @@ public class ConsumerConfigConstants extends
AWSConfigConstants {
/** The initial position to start reading Kinesis streams from (LATEST
is used if not set) */
public static final String STREAM_INITIAL_POSITION =
"flink.stream.initpos";
- /** The initial timestamp to start reading Kinesis stream from (when
AT_TIMESTAMP is set for STREAM_INITIAL_POSITION */
+ /** The initial timestamp to start reading Kinesis stream from (when
AT_TIMESTAMP is set for STREAM_INITIAL_POSITION) */
public static final String STREAM_INITIAL_TIMESTAMP =
"flink.stream.initpos.timestamp";
+ /** The date format of initial timestamp to start reading Kinesis
stream from (when AT_TIMESTAMP is set for STREAM_INITIAL_POSITION) */
+ public static final String STREAM_TIMESTAMP_DATE_FORMAT =
"flink.stream.initpos.timestamp.format";
+
/** The base ba
[20/37] flink git commit: [FLINK-6265] [cep] Fix consecutive() for times()
[FLINK-6265] [cep] Fix consecutive() for times()
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ffca0e76
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ffca0e76
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ffca0e76
Branch: refs/heads/table-retraction
Commit: ffca0e76eaaefe6251a7664b93c4ec007e4305e9
Parents: 1649f35
Author: kl0u
Authored: Tue Apr 4 18:23:50 2017 +0200
Committer: kl0u
Committed: Wed Apr 5 17:20:01 2017 +0200
--
.../flink/cep/nfa/compiler/NFACompiler.java | 1 -
.../org/apache/flink/cep/nfa/NFAITCase.java | 136 ++-
2 files changed, 133 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/ffca0e76/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
--
diff --git
a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
index e441c4b..e8077fa 100644
---
a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
+++
b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/compiler/NFACompiler.java
@@ -266,7 +266,6 @@ public class NFACompiler {
for (int i = 0; i < times - 1; i++) {
lastSink = createSingletonState(
lastSink,
- currentPattern instanceof
FollowedByPattern &&
!currentPattern.getQuantifier().hasProperty(QuantifierProperty.STRICT));
}
return createSingletonState(lastSink, currentPattern
instanceof FollowedByPattern);
http://git-wip-us.apache.org/repos/asf/flink/blob/ffca0e76/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
--
diff --git
a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
index da5f413..7359ca8 100644
---
a/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
+++
b/flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/nfa/NFAITCase.java
@@ -1336,9 +1336,10 @@ public class NFAITCase extends TestLogger {
}
}
- assertEquals(1, allPatterns.size());
+ assertEquals(2, allPatterns.size());
assertEquals(Sets.>newHashSet(
- Sets.newHashSet(startEvent, middleEvent1, middleEvent2,
end1)
+ Sets.newHashSet(startEvent, middleEvent1, middleEvent2,
end1),
+ Sets.newHashSet(startEvent, middleEvent1, middleEvent3,
end1)
), resultingPatterns);
}
@@ -1363,7 +1364,7 @@ public class NFAITCase extends TestLogger {
public boolean filter(Event value) throws Exception {
return value.getName().equals("a");
}
- }).times(2).followedBy("end1").where(new
SimpleCondition() {
+ }).times(2).consecutive().followedBy("end1").where(new
SimpleCondition() {
private static final long serialVersionUID =
5726188262756267490L;
@Override
@@ -1396,6 +1397,135 @@ public class NFAITCase extends TestLogger {
}
@Test
+ public void testTimesNonStrictWithNext() {
+ List> inputEvents = new ArrayList<>();
+
+ inputEvents.add(new StreamRecord<>(ConsecutiveData.startEvent,
1));
+ inputEvents.add(new
StreamRecord<>(ConsecutiveData.middleEvent1, 2));
+ inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 3));
+ inputEvents.add(new
StreamRecord<>(ConsecutiveData.middleEvent2, 4));
+ inputEvents.add(new StreamRecord<>(new Event(23, "f", 1.0), 5));
+ inputEvents.add(new
StreamRecord<>(ConsecutiveData.middleEvent3, 6));
+ inputEvents.add(new StreamRecord<>(ConsecutiveData.end, 7));
+
+ Pattern pattern =
Pattern.begin("start").where(new SimpleCondition() {
+ private static final long serialVersionUID =
5726188262756267490L;
+
+ @Override
+ public boolean filter(Event value) throws Exception {
+ return value.getName().equals("c");
+ }
+ }).next("middle").where(new SimpleCon
[34/37] flink git commit: [FLINK-6183] [metrics] Prevent some cases of TaskMG not being closed
[FLINK-6183] [metrics] Prevent some cases of TaskMG not being closed
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dabb0bac
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dabb0bac
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dabb0bac
Branch: refs/heads/table-retraction
Commit: dabb0bac0f724d50dcab5b3b767f38dc5feeb407
Parents: dc13500
Author: zentol
Authored: Fri Mar 24 19:39:31 2017 +0100
Committer: zentol
Committed: Wed Apr 5 23:18:08 2017 +0200
--
.../groups/TaskManagerJobMetricGroup.java | 25
.../apache/flink/runtime/taskmanager/Task.java | 9 +++
2 files changed, 24 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/dabb0bac/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskManagerJobMetricGroup.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskManagerJobMetricGroup.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskManagerJobMetricGroup.java
index 1ac8140..79a87d0 100644
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskManagerJobMetricGroup.java
+++
b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskManagerJobMetricGroup.java
@@ -72,16 +72,21 @@ public class TaskManagerJobMetricGroup extends
JobMetricGrouphttp://git-wip-us.apache.org/repos/asf/flink/blob/dabb0bac/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
--
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index b0f0eb8..ef934de 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -526,16 +526,25 @@ public class Task implements Runnable, TaskActions {
else if (current == ExecutionState.FAILED) {
// we were immediately failed. tell the
TaskManager that we reached our final state
notifyFinalState();
+ if (metrics != null) {
+ metrics.close();
+ }
return;
}
else if (current == ExecutionState.CANCELING) {
if (transitionState(ExecutionState.CANCELING,
ExecutionState.CANCELED)) {
// we were immediately canceled. tell
the TaskManager that we reached our final state
notifyFinalState();
+ if (metrics != null) {
+ metrics.close();
+ }
return;
}
}
else {
+ if (metrics != null) {
+ metrics.close();
+ }
throw new IllegalStateException("Invalid state
for beginning of operation of task " + this + '.');
}
}
[10/37] flink git commit: [FLINK-5913] [gelly] Example drivers
http://git-wip-us.apache.org/repos/asf/flink/blob/a48357db/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/TriangleListing.java
--
diff --git
a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/TriangleListing.java
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/TriangleListing.java
index 93a96c4..ca0c167 100644
---
a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/TriangleListing.java
+++
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/TriangleListing.java
@@ -18,315 +18,125 @@
package org.apache.flink.graph.drivers;
-import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.text.StrBuilder;
import org.apache.commons.lang3.text.WordUtils;
-import org.apache.commons.math3.random.JDKRandomGenerator;
-import org.apache.flink.api.common.JobExecutionResult;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.CsvOutputFormat;
-import org.apache.flink.api.java.utils.DataSetUtils;
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.client.program.ProgramParametrizationException;
import org.apache.flink.graph.Graph;
import org.apache.flink.graph.GraphAnalytic;
-import org.apache.flink.graph.GraphCsvReader;
-import org.apache.flink.graph.asm.translate.TranslateGraphIds;
-import
org.apache.flink.graph.asm.translate.translators.LongValueToUnsignedIntValue;
-import org.apache.flink.graph.generator.RMatGraph;
-import org.apache.flink.graph.generator.random.JDKRandomGeneratorFactory;
-import org.apache.flink.graph.generator.random.RandomGenerableFactory;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.apache.flink.types.NullValue;
-import org.apache.flink.types.StringValue;
-
-import java.text.NumberFormat;
+import org.apache.flink.graph.asm.result.PrintableResult;
+import org.apache.flink.graph.drivers.output.CSV;
+import org.apache.flink.graph.drivers.output.Hash;
+import org.apache.flink.graph.drivers.output.Print;
+import org.apache.flink.graph.drivers.parameter.BooleanParameter;
+import org.apache.flink.graph.drivers.parameter.ChoiceParameter;
+import org.apache.flink.graph.drivers.parameter.LongParameter;
+import org.apache.flink.types.CopyableValue;
import static org.apache.flink.api.common.ExecutionConfig.PARALLELISM_DEFAULT;
/**
- * Driver for the library implementation of Triangle Listing.
- *
- * This example reads a simple directed or undirected graph from a CSV file or
- * generates an RMat graph with the given scale and edge factor then lists
- * all triangles.
+ * Driver for directed and undirected triangle listing algorithm and analytic.
*
* @see org.apache.flink.graph.library.clustering.directed.TriangleListing
+ * @see org.apache.flink.graph.library.clustering.directed.TriadicCensus
* @see org.apache.flink.graph.library.clustering.undirected.TriangleListing
+ * @see org.apache.flink.graph.library.clustering.undirected.TriadicCensus
*/
-public class TriangleListing {
-
- private static final int DEFAULT_SCALE = 10;
-
- private static final int DEFAULT_EDGE_FACTOR = 16;
+public class TriangleListing & CopyableValue, VV,
EV>
+extends SimpleDriver
+implements Driver, CSV, Hash, Print {
- private static final boolean DEFAULT_TRIADIC_CENSUS = true;
-
- private static final boolean DEFAULT_CLIP_AND_FLIP = true;
-
- private static String getUsage(String message) {
- return new StrBuilder()
- .appendNewLine()
- .appendln(WordUtils.wrap("Lists all triangles in a
graph.", 80))
- .appendNewLine()
- .appendln(WordUtils.wrap("This algorithm returns tuples
containing the vertex IDs for each triangle and" +
- " for directed graphs a bitmask indicating the
presence of the six potential connecting edges.", 80))
- .appendNewLine()
- .appendln("usage: TriangleListing --directed [--triadic_census ] --input --output ")
- .appendNewLine()
- .appendln("options:")
- .appendln(" --input csv --type
[--simplify ] --input_filename FILENAME [--input_line_delimiter
LINE_DELIMITER] [--input_field_delimiter FIELD_DELIMITER]")
- .appendln(" --input rmat [--scale SCALE]
[--edge_factor EDGE_FACTOR]")
- .appendNewLine()
- .appendln(" --output print")
- .appendln(" --output hash")
- .appendln(" --output csv --output_filename FILENAME
[--output_line_delimiter LINE_DELIMITER] [--output_field_delimiter
FI
[11/37] flink git commit: [FLINK-5913] [gelly] Example drivers
[FLINK-5913] [gelly] Example drivers
Replace existing and create new algorithm Driver implementations for
each of the library methods.
This closes #3635
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a48357db
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a48357db
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a48357db
Branch: refs/heads/table-retraction
Commit: a48357db8c4187fd08f3b17880899ebbcb5d3b5e
Parents: ded25be
Author: Greg Hogan
Authored: Wed Oct 26 15:18:50 2016 -0400
Committer: Greg Hogan
Committed: Fri Mar 31 11:17:26 2017 -0400
--
.../main/java/org/apache/flink/graph/Usage.java | 2 -
.../apache/flink/graph/drivers/AdamicAdar.java | 71
.../graph/drivers/ClusteringCoefficient.java| 378 +--
.../graph/drivers/ConnectedComponents.java | 105 ++
.../apache/flink/graph/drivers/EdgeList.java| 92 +
.../apache/flink/graph/drivers/Graph500.java| 165
.../flink/graph/drivers/GraphMetrics.java | 265 -
.../org/apache/flink/graph/drivers/HITS.java| 188 ++---
.../flink/graph/drivers/JaccardIndex.java | 224 ++-
.../apache/flink/graph/drivers/PageRank.java| 74
.../flink/graph/drivers/SimpleDriver.java | 65
.../flink/graph/drivers/TriangleListing.java| 362 +-
.../drivers/parameter/IterationConvergence.java | 89 +
.../graph/examples/ConnectedComponents.java | 141 ---
.../examples/GSASingleSourceShortestPaths.java | 4 +-
.../parameter/IterationConvergenceTest.java | 66
.../examples/ConnectedComponentsITCase.java | 72
.../main/java/org/apache/flink/graph/Graph.java | 16 +-
.../graph/library/ConnectedComponents.java | 5 +-
.../graph/library/GSAConnectedComponents.java | 8 +-
.../flink/graph/library/LabelPropagation.java | 5 +-
.../clustering/directed/TriangleListing.java| 2 +-
.../undirected/LocalClusteringCoefficient.java | 2 +-
.../graph/library/link_analysis/PageRank.java | 8 +-
.../graph/library/similarity/AdamicAdar.java| 2 +-
.../graph/library/similarity/JaccardIndex.java | 2 +-
.../apache/flink/graph/utils/GraphUtils.java| 10 +-
.../flink/graph/utils/NullValueEdgeMapper.java | 32 --
28 files changed, 919 insertions(+), 1536 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/a48357db/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Usage.java
--
diff --git
a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Usage.java
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Usage.java
index d923bf0..642fe5b 100644
---
a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Usage.java
+++
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/Usage.java
@@ -28,7 +28,6 @@ public class Usage {
private static final Class[] DRIVERS = new Class[]{
org.apache.flink.graph.drivers.ClusteringCoefficient.class,
- org.apache.flink.graph.drivers.Graph500.class,
org.apache.flink.graph.drivers.GraphMetrics.class,
org.apache.flink.graph.drivers.HITS.class,
org.apache.flink.graph.drivers.JaccardIndex.class,
@@ -36,7 +35,6 @@ public class Usage {
};
private static final Class[] EXAMPLES = new Class[]{
- org.apache.flink.graph.examples.ConnectedComponents.class,
org.apache.flink.graph.examples.EuclideanGraphWeighing.class,
org.apache.flink.graph.examples.GSASingleSourceShortestPaths.class,
org.apache.flink.graph.examples.IncrementalSSSP.class,
http://git-wip-us.apache.org/repos/asf/flink/blob/a48357db/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/AdamicAdar.java
--
diff --git
a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/AdamicAdar.java
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/AdamicAdar.java
new file mode 100644
index 000..742c1de
--- /dev/null
+++
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/AdamicAdar.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this f
[27/37] flink git commit: [FLINK-6246] Fix generic type of OutputTag in operator Output
[FLINK-6246] Fix generic type of OutputTag in operator Output
This closes #3662.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9bdbe607
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9bdbe607
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9bdbe607
Branch: refs/heads/table-retraction
Commit: 9bdbe6071f1946391598709bfa637fd76a8c7396
Parents: 48ce77c
Author: Aljoscha Krettek
Authored: Mon Apr 3 16:10:14 2017 +0200
Committer: zentol
Committed: Wed Apr 5 20:43:00 2017 +0200
--
.../api/collector/selector/DirectedOutput.java | 2 +-
.../api/operators/AbstractStreamOperator.java | 2 +-
.../flink/streaming/api/operators/Output.java | 2 +-
.../streaming/runtime/io/RecordWriterOutput.java | 2 +-
.../streaming/runtime/tasks/OperatorChain.java | 17 ++---
.../runtime/tasks/StreamIterationTail.java | 2 +-
.../util/AbstractStreamOperatorTestHarness.java| 2 +-
.../flink/streaming/util/CollectorOutput.java | 2 +-
.../apache/flink/streaming/util/MockOutput.java| 2 +-
9 files changed, 14 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
--
diff --git
a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
index dabe804..6339506 100644
---
a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
+++
b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/collector/selector/DirectedOutput.java
@@ -140,7 +140,7 @@ public class DirectedOutput implements
Output> {
}
@Override
- public void collect(OutputTag outputTag, StreamRecord record)
{
+ public void collect(OutputTag outputTag, StreamRecord record)
{
throw new UnsupportedOperationException("Cannot use
split/select with side outputs.");
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
--
diff --git
a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index cc81a0e..7569170 100644
---
a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++
b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -872,7 +872,7 @@ public abstract class AbstractStreamOperator
}
@Override
- public void collect(OutputTag outputTag, StreamRecord
record) {
+ public void collect(OutputTag outputTag, StreamRecord
record) {
numRecordsOut.inc();
output.collect(outputTag, record);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java
--
diff --git
a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java
b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java
index eb10d8d..7035d28 100644
---
a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java
+++
b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/Output.java
@@ -48,7 +48,7 @@ public interface Output extends Collector {
*
* @param record The record to collect.
*/
-void collect(OutputTag outputTag, StreamRecord record);
+void collect(OutputTag outputTag, StreamRecord record);
void emitLatencyMarker(LatencyMarker latencyMarker);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9bdbe607/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
--
diff --git
a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
index d22c60d..365c78e 100644
---
a/flink-streaming-java/src/main/java/org/apache/
[25/37] flink git commit: [FLINK-6127] [checkstyle] Add MissingDeprecation check
[FLINK-6127] [checkstyle] Add MissingDeprecation check This closes #3572. Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/68605d05 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/68605d05 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/68605d05 Branch: refs/heads/table-retraction Commit: 68605d05107e2c70c12178b1db6a9e49641dbfe4 Parents: 6c22404 Author: Ufuk Celebi Authored: Mon Mar 20 11:45:52 2017 +0100 Committer: zentol Committed: Wed Apr 5 20:41:14 2017 +0200 -- .../connectors/kafka/FlinkKafkaConsumer081.java | 4 ++ .../connectors/kafka/FlinkKafkaConsumer082.java | 4 ++ .../connectors/kafka/FlinkKafkaProducer.java| 22 +++- .../state/RocksDBKeyedStateBackend.java | 2 + .../streaming/state/RocksDBStateBackend.java| 3 + .../flink/api/common/JobExecutionResult.java| 1 + .../api/common/io/DelimitedInputFormat.java | 2 +- .../flink/api/common/state/StateDescriptor.java | 12 +++- .../flink/configuration/ConfigConstants.java| 58 ++-- .../flink/migration/util/SerializedValue.java | 1 + .../org/apache/flink/api/java/io/CsvReader.java | 2 + .../flink/api/java/operators/CrossOperator.java | 2 +- .../flink/api/java/operators/DataSink.java | 4 ++ .../flink/api/java/operators/JoinOperator.java | 2 +- .../api/java/operators/ProjectOperator.java | 2 +- ...nkAggregateExpandDistinctAggregatesRule.java | 10 +++- .../rules/FlinkAggregateJoinTransposeRule.java | 20 +-- .../MigrationNamespaceSerializerProxy.java | 4 +- .../api/common/state/ListStateDescriptor.java | 2 + .../runtime/checkpoint/KeyGroupState.java | 2 + .../runtime/checkpoint/SubtaskState.java| 3 + .../migration/runtime/checkpoint/TaskState.java | 3 + .../runtime/state/AbstractCloseableHandle.java | 2 + .../runtime/state/AbstractStateBackend.java | 2 + .../runtime/state/KvStateSnapshot.java | 3 + .../migration/runtime/state/StateHandle.java| 2 + .../migration/runtime/state/StateObject.java| 2 + .../runtime/state/StreamStateHandle.java| 2 + .../filesystem/AbstractFileStateHandle.java | 2 + .../filesystem/AbstractFsStateSnapshot.java | 2 + .../filesystem/FileSerializableStateHandle.java | 2 + .../state/filesystem/FileStreamStateHandle.java | 2 + .../state/filesystem/FsFoldingState.java| 3 + .../runtime/state/filesystem/FsListState.java | 3 + .../state/filesystem/FsReducingState.java | 3 + .../state/filesystem/FsStateBackend.java| 3 + .../runtime/state/filesystem/FsValueState.java | 3 + .../state/memory/AbstractMemStateSnapshot.java | 3 + .../AbstractMigrationRestoreStrategy.java | 4 +- .../state/memory/ByteStreamStateHandle.java | 3 + .../runtime/state/memory/MemFoldingState.java | 3 + .../runtime/state/memory/MemListState.java | 3 + .../runtime/state/memory/MemReducingState.java | 2 + .../runtime/state/memory/MemValueState.java | 2 + .../state/memory/MigrationRestoreSnapshot.java | 3 + .../state/MigrationKeyGroupStateHandle.java | 2 + .../state/MigrationStreamStateHandle.java | 2 + .../runtime/tasks/StreamTaskState.java | 3 + .../runtime/tasks/StreamTaskStateList.java | 3 + .../flink/runtime/checkpoint/SubtaskState.java | 7 +++ .../StandaloneLeaderRetrievalService.java | 1 + .../operators/hash/InMemoryPartition.java | 2 + .../flink/runtime/state/TaskStateHandles.java | 11 +++- .../state/heap/HeapKeyedStateBackend.java | 5 +- .../api/checkpoint/CheckpointedRestoring.java | 2 + .../environment/StreamExecutionEnvironment.java | 5 ++ .../functions/AscendingTimestampExtractor.java | 2 + .../source/FileMonitoringFunction.java | 5 ++ .../api/functions/source/FileReadFunction.java | 5 ++ .../api/operators/AbstractStreamOperator.java | 8 +++ .../CheckpointedRestoringOperator.java | 4 ++ .../operators/StreamCheckpointedOperator.java | 2 +- .../api/windowing/triggers/Trigger.java | 2 + ...ractAlignedProcessingTimeWindowOperator.java | 4 ++ ...ccumulatingProcessingTimeWindowOperator.java | 4 ++ ...AggregatingProcessingTimeWindowOperator.java | 4 ++ .../flink/yarn/cli/FlinkYarnSessionCli.java | 6 ++ tools/maven/checkstyle.xml | 2 + 68 files changed, 269 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/flink/blob/68605d05/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer081.java -- diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/k
[07/37] flink git commit: [FLINK-3695] [gelly] ValueArray types
http://git-wip-us.apache.org/repos/asf/flink/blob/963f46e7/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayFactory.java
--
diff --git
a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayFactory.java
b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayFactory.java
new file mode 100644
index 000..b7b6282
--- /dev/null
+++
b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayFactory.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.types.valuearray;
+
+import org.apache.flink.types.CopyableValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+import org.apache.flink.types.StringValue;
+import org.apache.flink.types.Value;
+
+/**
+ * A factory generator for {@link ValueArray} types is necessary because the
+ * contained {@link Value} types do not currently implement a common interface
+ * for creating a {@link ValueArray}. Algorithms must instantiate classes at
+ * runtime when the type information has been erased.
+ *
+ * This mirrors creating {@link Value} using {@link CopyableValue#copy()}.
+ */
+public class ValueArrayFactory {
+
+ /**
+* Produce a {@code ValueArray} for the given {@code Value} type.
+*
+* @param cls {@code Value} class
+* @return {@code ValueArray} for given {@code Value} class
+*/
+ @SuppressWarnings("unchecked")
+ public static ValueArray createValueArray(Class
cls) {
+ if (IntValue.class.isAssignableFrom(cls)) {
+ return (ValueArray) new IntValueArray();
+ } else if (LongValue.class.isAssignableFrom(cls)) {
+ return (ValueArray) new LongValueArray();
+ } else if (NullValue.class.isAssignableFrom(cls)) {
+ return (ValueArray) new NullValueArray();
+ } else if (StringValue.class.isAssignableFrom(cls)) {
+ return (ValueArray) new StringValueArray();
+ } else {
+ throw new IllegalArgumentException("Unable to create
unbounded ValueArray for type " + cls);
+ }
+ }
+
+ /**
+* Produce a {@code ValueArray} for the given {@code Value} type with
the
+* given bounded size.
+*
+* @param cls {@code Value} class
+* @param bytes limit the array to the given number of bytes
+* @return {@code ValueArray} for given {@code Value} class
+*/
+ @SuppressWarnings("unchecked")
+ public static ValueArray createValueArray(Class
cls, int bytes) {
+ if (IntValue.class.isAssignableFrom(cls)) {
+ return (ValueArray) new IntValueArray(bytes);
+ } else if (LongValue.class.isAssignableFrom(cls)) {
+ return (ValueArray) new LongValueArray(bytes);
+ } else if (NullValue.class.isAssignableFrom(cls)) {
+ return (ValueArray) new NullValueArray(bytes);
+ } else if (StringValue.class.isAssignableFrom(cls)) {
+ return (ValueArray) new StringValueArray(bytes);
+ } else {
+ throw new IllegalArgumentException("Unable to create
bounded ValueArray for type " + cls);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/963f46e7/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfo.java
--
diff --git
a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfo.java
b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/ValueArrayTypeInfo.java
new file mode 100644
index 000..ee9b770
--- /dev/null
+++
b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valu
[18/37] flink git commit: [FLINK-6267] [table] Remove unused imports in FlinkRelBuilder.
[FLINK-6267] [table] Remove unused imports in FlinkRelBuilder.
This closes #3671.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a6add620
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a6add620
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a6add620
Branch: refs/heads/table-retraction
Commit: a6add620060f94b0ff7d3264a09dc8145d9fb819
Parents: cae4976
Author: sunjincheng121
Authored: Wed Apr 5 15:10:28 2017 +0800
Committer: Fabian Hueske
Committed: Wed Apr 5 11:09:18 2017 +0200
--
.../scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala | 2 --
1 file changed, 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/a6add620/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala
index 6430ce0..366de86 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala
@@ -27,13 +27,11 @@ import org.apache.calcite.jdbc.CalciteSchema
import org.apache.calcite.plan._
import org.apache.calcite.prepare.CalciteCatalogReader
import org.apache.calcite.rel.logical.LogicalAggregate
-import org.apache.calcite.rel.metadata.{JaninoRelMetadataProvider,
RelMetadataQuery}
import org.apache.calcite.rex.RexBuilder
import org.apache.calcite.tools.RelBuilder.{AggCall, GroupKey}
import org.apache.calcite.tools.{FrameworkConfig, RelBuilder}
import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
import org.apache.flink.table.expressions.WindowProperty
-import org.apache.flink.table.plan.cost.FlinkDefaultRelMetadataProvider
import org.apache.flink.table.plan.logical.LogicalWindow
import org.apache.flink.table.plan.logical.rel.LogicalWindowAggregate
[02/37] flink git commit: [FLINK-4577] [kinesis] Transparent reshard handling for FlinkKinesisConsumer
[FLINK-4577] [kinesis] Transparent reshard handling for FlinkKinesisConsumer
This closes #3458.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/19322401
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/19322401
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/19322401
Branch: refs/heads/table-retraction
Commit: 1932240179189a88273f52d3d93f277e7bf604de
Parents: a119a30
Author: Tzu-Li (Gordon) Tai
Authored: Thu Mar 2 18:56:39 2017 +0800
Committer: Tzu-Li (Gordon) Tai
Committed: Fri Mar 31 12:33:48 2017 +0800
--
docs/dev/connectors/kinesis.md | 36 +
.../kinesis/internals/KinesisDataFetcher.java | 77
2 files changed, 34 insertions(+), 79 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/19322401/docs/dev/connectors/kinesis.md
--
diff --git a/docs/dev/connectors/kinesis.md b/docs/dev/connectors/kinesis.md
index ef1afca..1fcc529 100644
--- a/docs/dev/connectors/kinesis.md
+++ b/docs/dev/connectors/kinesis.md
@@ -23,6 +23,9 @@ specific language governing permissions and limitations
under the License.
-->
+* This will be replaced by the TOC
+{:toc}
+
The Kinesis connector provides access to [Amazon AWS Kinesis
Streams](http://aws.amazon.com/kinesis/streams/).
To use the connector, add the following Maven dependency to your project:
@@ -53,14 +56,14 @@ mvn clean install -Pinclude-kinesis -DskipTests
The streaming connectors are not part of the binary distribution. See how to
link with them for cluster
execution [here]({{site.baseurl}}/dev/linking.html).
-### Using the Amazon Kinesis Streams Service
+## Using the Amazon Kinesis Streams Service
Follow the instructions from the [Amazon Kinesis Streams Developer
Guide](https://docs.aws.amazon.com/streams/latest/dev/learning-kinesis-module-one-create-stream.html)
to setup Kinesis streams. Make sure to create the appropriate IAM policy and
user to read / write to the Kinesis streams.
-### Kinesis Consumer
+## Kinesis Consumer
The `FlinkKinesisConsumer` is an exactly-once parallel streaming data source
that subscribes to multiple AWS Kinesis
-streams within the same AWS service region, and can handle resharding of
streams. Each subtask of the consumer is
+streams within the same AWS service region, and can transparently handle
resharding of streams while the job is running. Each subtask of the consumer is
responsible for fetching data records from multiple Kinesis shards. The number
of shards fetched by each subtask will
change as shards are closed and created by Kinesis.
@@ -107,13 +110,16 @@ to `TRIM_HORIZON`, which lets the consumer start reading
the Kinesis stream from
Other optional configuration keys for the consumer can be found in
`ConsumerConfigConstants`.
-**NOTE:** Currently, resharding can not be handled transparently (i.e.,
without failing and restarting jobs) if there are idle consumer
-subtasks, which occur when the total number of shards is lower than the
configured consumer parallelism. The job must be
-configured to enable checkpointing, so that the new shards due to resharding
can be correctly picked up and consumed by the
-Kinesis consumer after the job is restored. This is a temporary limitation
that will be resolved in future versions.
-Please see [FLINK-4341](https://issues.apache.org/jira/browse/FLINK-4341) for
more detail.
+Note that the configured parallelism of the Flink Kinesis Consumer source
+can be completely independent of the total number of shards in the Kinesis
streams.
+When the number of shards is larger than the parallelism of the consumer,
+then each consumer subtask can subscribe to multiple shards; otherwise
+if the number of shards is smaller than the parallelism of the consumer,
+then some consumer subtasks will simply be idle and wait until it gets assigned
+new shards (i.e., when the streams are resharded to increase the
+number of shards for higher provisioned Kinesis service throughput).
- Configuring Starting Position
+### Configuring Starting Position
The Flink Kinesis Consumer currently provides the following options to
configure where to start reading Kinesis streams, simply by setting
`ConsumerConfigConstants.STREAM_INITIAL_POSITION` to
one of the following values in the provided configuration properties (the
naming of the options identically follows [the namings used by the AWS Kinesis
Streams
service](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax)):
@@ -127,7 +133,7 @@ properties by providing a value for
`ConsumerConfigConstants.STREAM_INITIAL_TIME
If `ConsumerConfigConstants.STREAM_TIMESTAMP_DATE_FORMAT` is not defined
then th
[15/37] flink git commit: [FLINK-6124] [table] Add min/max string aggregation with retracion
[FLINK-6124] [table] Add min/max string aggregation with retracion
This closes #3593.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c5282cbc
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c5282cbc
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c5282cbc
Branch: refs/heads/table-retraction
Commit: c5282cbcf898b99593ca753fce7557bae1ae09aa
Parents: 05ceec0
Author: twalthr
Authored: Wed Mar 22 10:03:56 2017 +0100
Committer: twalthr
Committed: Mon Apr 3 18:21:53 2017 +0200
--
.../functions/aggfunctions/MaxAggFunction.scala | 2 +-
.../MaxAggFunctionWithRetract.scala | 12 ++--
.../functions/aggfunctions/MinAggFunction.scala | 2 +-
.../MinAggFunctionWithRetract.scala | 8 +
.../table/runtime/aggregate/AggregateUtil.scala | 4 +++
.../aggfunctions/AggFunctionTestBase.scala | 4 +--
.../MaxWithRetractAggFunctionTest.scala | 32
.../MinWithRetractAggFunctionTest.scala | 32
8 files changed, 90 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/c5282cbc/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunction.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunction.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunction.scala
index 3793434..2e666fa 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunction.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunction.scala
@@ -160,6 +160,6 @@ class DecimalMaxAggFunction extends
MaxAggFunction[BigDecimal] {
* Built-in String Max aggregate function
*/
class StringMaxAggFunction extends MaxAggFunction[String] {
- override def getInitValue = "".toString
+ override def getInitValue = ""
override def getValueTypeInfo = BasicTypeInfo.STRING_TYPE_INFO
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c5282cbc/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionWithRetract.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionWithRetract.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionWithRetract.scala
index eb6e7dc..14ceba2 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionWithRetract.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MaxAggFunctionWithRetract.scala
@@ -201,7 +201,7 @@ class DoubleMaxWithRetractAggFunction extends
MaxWithRetractAggFunction[Double]
* Built-in Boolean Max with retraction aggregate function
*/
class BooleanMaxWithRetractAggFunction extends
MaxWithRetractAggFunction[Boolean] {
- override def getInitValue = false
+ override def getInitValue: Boolean = false
override def getValueTypeInfo = BasicTypeInfo.BOOLEAN_TYPE_INFO
}
@@ -209,6 +209,14 @@ class BooleanMaxWithRetractAggFunction extends
MaxWithRetractAggFunction[Boolean
* Built-in Big Decimal Max with retraction aggregate function
*/
class DecimalMaxWithRetractAggFunction extends
MaxWithRetractAggFunction[BigDecimal] {
- override def getInitValue = BigDecimal.ZERO
+ override def getInitValue: BigDecimal = BigDecimal.ZERO
override def getValueTypeInfo = BasicTypeInfo.BIG_DEC_TYPE_INFO
}
+
+/**
+ * Built-in String Max with retraction aggregate function
+ */
+class StringMaxWithRetractAggFunction extends
MaxWithRetractAggFunction[String] {
+ override def getInitValue: String = ""
+ override def getValueTypeInfo = BasicTypeInfo.STRING_TYPE_INFO
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/c5282cbc/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunction.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunction.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunction.scala
index 41361fd..75a8ebc 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunction.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/MinAggFunc
[37/37] flink git commit: [FLINK-6241] [table] Add codeGen for streaming OVER aggregations.
[FLINK-6241] [table] Add codeGen for streaming OVER aggregations.
This closes #3676.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/89d9dec3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/89d9dec3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/89d9dec3
Branch: refs/heads/table-retraction
Commit: 89d9dec388ccd8d7d9860d7b550f29a083d2605d
Parents: 80800d8
Author: shaoxuan-wang
Authored: Wed Apr 5 22:34:43 2017 +0800
Committer: Fabian Hueske
Committed: Thu Apr 6 09:27:25 2017 +0200
--
.../flink/table/codegen/CodeGenerator.scala | 210 ++-
.../apache/flink/table/codegen/generated.scala | 10 +
.../datastream/DataStreamOverAggregate.scala| 29 ++-
.../table/runtime/aggregate/AggregateUtil.scala | 112 ++
...ndedProcessingOverRangeProcessFunction.scala | 96 -
...oundedProcessingOverRowProcessFunction.scala | 78 ---
.../aggregate/GeneratedAggregations.scala | 78 +++
.../RangeClauseBoundedOverProcessFunction.scala | 84 +++-
.../RowsClauseBoundedOverProcessFunction.scala | 83 +++-
.../UnboundedEventTimeOverProcessFunction.scala | 129 +---
...rtitionedProcessingOverProcessFunction.scala | 66 +++---
...UnboundedProcessingOverProcessFunction.scala | 63 +++---
...ProcessingOverRangeProcessFunctionTest.scala | 124 ++-
13 files changed, 762 insertions(+), 400 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/89d9dec3/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 46b1dcf..ee6fced 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -21,6 +21,7 @@ package org.apache.flink.table.codegen
import java.math.{BigDecimal => JBigDecimal}
import org.apache.calcite.avatica.util.DateTimeUtils
+import org.apache.calcite.rel.`type`.RelDataType
import org.apache.calcite.rex._
import org.apache.calcite.sql.SqlOperator
import org.apache.calcite.sql.`type`.SqlTypeName._
@@ -39,7 +40,7 @@ import
org.apache.flink.table.codegen.GeneratedExpression.{NEVER_NULL, NO_CODE}
import org.apache.flink.table.codegen.Indenter.toISC
import org.apache.flink.table.codegen.calls.FunctionGenerator
import org.apache.flink.table.codegen.calls.ScalarOperators._
-import org.apache.flink.table.functions.{FunctionContext, UserDefinedFunction}
+import org.apache.flink.table.functions.{AggregateFunction, FunctionContext,
UserDefinedFunction}
import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
import org.apache.flink.table.runtime.TableFunctionCollector
import org.apache.flink.table.typeutils.TypeCheckUtils._
@@ -239,6 +240,213 @@ class CodeGenerator(
}
/**
+* Generates a
[[org.apache.flink.table.runtime.aggregate.GeneratedAggregations]] that can be
+* passed to a Java compiler.
+*
+* @param nameClass name of the function.
+*Does not need to be unique but has to be a valid Java
class identifier.
+* @param generator The code generator instance
+* @param inputType Input row type
+* @param aggregates All aggregate functions
+* @param aggFields Indexes of the input fields for all aggregate
functions
+* @param aggMapping The mapping of aggregates to output fields
+* @param fwdMapping The mapping of input fields to output fields
+* @param outputArity The number of fields in the output row.
+*
+* @return A GeneratedAggregationsFunction
+*/
+ def generateAggregations(
+ name: String,
+ generator: CodeGenerator,
+ inputType: RelDataType,
+ aggregates: Array[AggregateFunction[_ <: Any]],
+ aggFields: Array[Array[Int]],
+ aggMapping: Array[Int],
+ fwdMapping: Array[(Int, Int)],
+ outputArity: Int)
+ : GeneratedAggregationsFunction = {
+
+def generateSetAggregationResults(
+ accTypes: Array[String],
+ aggs: Array[String],
+ aggMapping: Array[Int]): String = {
+
+ val sig: String =
+j"""
+| public void setAggregationResults(
+|org.apache.flink.types.Row accs,
+|org.apache.flink.types.Row output)""".stripMargin
+
+ val setAggs: String = {
+for (i <- aggs.indices) yield
+ j"""
+ |org.apache.flink.table.functions.AggregateFunction
baseClass$i =
+ | (org.apa
[08/37] flink git commit: [FLINK-3695] [gelly] ValueArray types
[FLINK-3695] [gelly] ValueArray types
Provide compact and efficiently serializable and comparable array
implementations for Flink mutable Value types and Java primitives.
This cloeses #3382
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/963f46e7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/963f46e7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/963f46e7
Branch: refs/heads/table-retraction
Commit: 963f46e7179db034fd1d69f4af58eac87409
Parents: 43158a8
Author: Greg Hogan
Authored: Tue Feb 21 11:40:22 2017 -0500
Committer: Greg Hogan
Committed: Fri Mar 31 11:14:36 2017 -0400
--
flink-libraries/flink-gelly/pom.xml | 12 +-
.../graph/types/valuearray/IntValueArray.java | 398 ++
.../valuearray/IntValueArrayComparator.java | 156 ++
.../valuearray/IntValueArraySerializer.java | 85 +++
.../graph/types/valuearray/LongValueArray.java | 399 ++
.../valuearray/LongValueArrayComparator.java| 156 ++
.../valuearray/LongValueArraySerializer.java| 85 +++
.../graph/types/valuearray/NullValueArray.java | 267 ++
.../valuearray/NullValueArrayComparator.java| 147 ++
.../valuearray/NullValueArraySerializer.java| 85 +++
.../types/valuearray/StringValueArray.java | 518 +++
.../valuearray/StringValueArrayComparator.java | 217
.../valuearray/StringValueArraySerializer.java | 85 +++
.../graph/types/valuearray/ValueArray.java | 97
.../types/valuearray/ValueArrayFactory.java | 81 +++
.../types/valuearray/ValueArrayTypeInfo.java| 159 ++
.../valuearray/ValueArrayTypeInfoFactory.java | 41 ++
.../valuearray/IntValueArrayComparatorTest.java | 51 ++
.../valuearray/IntValueArraySerializerTest.java | 93
.../types/valuearray/IntValueArrayTest.java | 123 +
.../LongValueArrayComparatorTest.java | 51 ++
.../LongValueArraySerializerTest.java | 93
.../types/valuearray/LongValueArrayTest.java| 123 +
.../NullValueArrayComparatorTest.java | 51 ++
.../NullValueArraySerializerTest.java | 68 +++
.../types/valuearray/NullValueArrayTest.java| 80 +++
.../StringValueArrayComparatorTest.java | 51 ++
.../StringValueArraySerializerTest.java | 93
.../types/valuearray/StringValueArrayTest.java | 168 ++
.../valuearray/ValueArrayTypeInfoTest.java | 64 +++
30 files changed, 4095 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/963f46e7/flink-libraries/flink-gelly/pom.xml
--
diff --git a/flink-libraries/flink-gelly/pom.xml
b/flink-libraries/flink-gelly/pom.xml
index f773c70..fa09102 100644
--- a/flink-libraries/flink-gelly/pom.xml
+++ b/flink-libraries/flink-gelly/pom.xml
@@ -28,7 +28,7 @@ under the License.
1.3-SNAPSHOT
..
-
+
flink-gelly_2.10
flink-gelly
@@ -59,7 +59,7 @@ under the License.
-
+
org.apache.flink
flink-test-utils_2.10
@@ -69,6 +69,14 @@ under the License.
org.apache.flink
+ flink-core
+ ${project.version}
+ test-jar
+ test
+
+
+
+ org.apache.flink
flink-optimizer_2.10
${project.version}
test-jar
http://git-wip-us.apache.org/repos/asf/flink/blob/963f46e7/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/IntValueArray.java
--
diff --git
a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/IntValueArray.java
b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/IntValueArray.java
new file mode 100644
index 000..0e3812d
--- /dev/null
+++
b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/types/valuearray/IntValueArray.java
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+
[09/37] flink git commit: [FLINK-5912] [gelly] Inputs for CSV and graph generators
[FLINK-5912] [gelly] Inputs for CSV and graph generators
Create Input classes for reading graphs from CSV as well as for each of
the graph generators.
This closes #3626
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ded25be4
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ded25be4
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ded25be4
Branch: refs/heads/table-retraction
Commit: ded25be4d8fc51f2a58cbd3264daffe48dca6b04
Parents: 963f46e
Author: Greg Hogan
Authored: Mon Mar 27 10:08:59 2017 -0400
Committer: Greg Hogan
Committed: Fri Mar 31 11:16:05 2017 -0400
--
.../apache/flink/graph/drivers/input/CSV.java | 115
.../graph/drivers/input/CompleteGraph.java | 60 +++
.../flink/graph/drivers/input/CycleGraph.java | 60 +++
.../flink/graph/drivers/input/EmptyGraph.java | 55 ++
.../flink/graph/drivers/input/GridGraph.java| 144 +++
.../graph/drivers/input/HypercubeGraph.java | 60 +++
.../flink/graph/drivers/input/PathGraph.java| 60 +++
.../flink/graph/drivers/input/RMatGraph.java| 174 +++
.../graph/drivers/input/SingletonEdgeGraph.java | 60 +++
.../flink/graph/drivers/input/StarGraph.java| 60 +++
.../graph/drivers/parameter/Parameterized.java | 2 +-
.../drivers/parameter/ParameterizedBase.java| 2 +-
.../flink/graph/drivers/parameter/Simplify.java | 137 +++
.../graph/drivers/parameter/SimplifyTest.java | 56 ++
14 files changed, 1043 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/ded25be4/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CSV.java
--
diff --git
a/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CSV.java
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CSV.java
new file mode 100644
index 000..58b65b6
--- /dev/null
+++
b/flink-libraries/flink-gelly-examples/src/main/java/org/apache/flink/graph/drivers/input/CSV.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.graph.drivers.input;
+
+import org.apache.commons.lang3.text.WordUtils;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.CsvInputFormat;
+import org.apache.flink.client.program.ProgramParametrizationException;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.GraphCsvReader;
+import org.apache.flink.graph.drivers.parameter.ChoiceParameter;
+import org.apache.flink.graph.drivers.parameter.ParameterizedBase;
+import org.apache.flink.graph.drivers.parameter.Simplify;
+import org.apache.flink.graph.drivers.parameter.StringParameter;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.NullValue;
+import org.apache.flink.types.StringValue;
+
+/**
+ * Read a {@link Graph} from a CSV file using {@link IntValue},
+ * {@link LongValue}, or {@link StringValue} keys.
+ *
+ * @param key type
+ */
+public class CSV>
+extends ParameterizedBase
+implements Input {
+
+ private static final String INTEGER = "integer";
+
+ private static final String LONG = "long";
+
+ private static final String STRING = "string";
+
+ private ChoiceParameter type = new ChoiceParameter(this, "type")
+ .setDefaultValue(INTEGER)
+ .addChoices(LONG, STRING);
+
+ private StringParameter inputFilename = new StringParameter(this,
"input_filename");
+
+ private StringParameter commentPrefix = new StringParameter(this,
"comment_prefix")
+ .setDefaultValue("#");
+
+ private StringParameter lineDelimiter = new StringParameter(this,
"input_line_delimiter")
+ .setDefaultValue(CsvInputFormat.DEFAULT_LINE_DELIMITER);
+
+ private StringParameter fieldDelimite
[32/37] flink git commit: [hotfix] Fix shading checks
[hotfix] Fix shading checks
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5c44f539
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5c44f539
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5c44f539
Branch: refs/heads/table-retraction
Commit: 5c44f53966a0ec5e96bc94b09f6067acf24cc969
Parents: dd588ef
Author: Robert Metzger
Authored: Thu Mar 23 15:13:25 2017 +0100
Committer: zentol
Committed: Wed Apr 5 20:48:30 2017 +0200
--
tools/travis_mvn_watchdog.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/5c44f539/tools/travis_mvn_watchdog.sh
--
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index d1b396c..b4c8285 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -164,7 +164,7 @@ watchdog () {
# Check the final fat jar for illegal artifacts
check_shaded_artifacts() {
- jar tf build-target/lib/flink-dist-*.jar > allClasses
+ jar tf build-target/lib/flink-dist*.jar > allClasses
ASM=`cat allClasses | grep '^org/objectweb/asm/' | wc -l`
if [ $ASM != "0" ]; then
echo
"=="
[03/37] flink git commit: [FLINK-6205] [FLINK-6069] [cep] Correct watermark/late events in side output.
[FLINK-6205] [FLINK-6069] [cep] Correct watermark/late events in side output.
With this, the CEP library assumes correctness of the watermark
and considers as late, events that arrive having a timestamp
smaller than that of the last seen watermark. Late events are not
silently dropped, but the user can specify to send them to a side
output.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/48890285
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/48890285
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/48890285
Branch: refs/heads/table-retraction
Commit: 48890285d4b1c285bebb971ae0dbfc310c6fcc0e
Parents: 1932240
Author: kl0u
Authored: Thu Mar 23 19:01:15 2017 +0100
Committer: kl0u
Committed: Fri Mar 31 11:16:50 2017 +0200
--
docs/dev/libs/cep.md| 53 ++
.../apache/flink/cep/scala/PatternStream.scala | 37 ++-
.../org/apache/flink/cep/PatternStream.java | 72 +++--
.../AbstractKeyedCEPPatternOperator.java| 69 ++---
.../flink/cep/operator/CEPOperatorUtils.java| 17 ++-
.../cep/operator/KeyedCEPPatternOperator.java | 4 +-
.../TimeoutKeyedCEPPatternOperator.java | 4 +-
.../java/org/apache/flink/cep/CEPITCase.java| 103 +++
.../cep/operator/CEPMigration11to13Test.java| 2 +
.../cep/operator/CEPMigration12to13Test.java| 3 +
.../flink/cep/operator/CEPOperatorTest.java | 2 +
.../flink/cep/operator/CEPRescalingTest.java| 1 +
12 files changed, 333 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/48890285/docs/dev/libs/cep.md
--
diff --git a/docs/dev/libs/cep.md b/docs/dev/libs/cep.md
index bb704c7..643d6ee 100644
--- a/docs/dev/libs/cep.md
+++ b/docs/dev/libs/cep.md
@@ -777,6 +777,59 @@ DataStream[Either[TimeoutEvent, ComplexEvent]] result =
patternStream.flatSelect
+### Handling Lateness in Event Time
+
+In `CEP` the order in which elements are processed matters. To guarantee that
elements are processed in the correct order
+when working in event time, an incoming element is initially put in a buffer
where elements are *sorted in ascending
+order based on their timestamp*, and when a watermark arrives, all the
elements in this buffer with timestamps smaller
+than that of the watermark are processed. This implies that elements between
watermarks are processed in event-time order.
+
+Attention The library assumes
correctness of the watermark when working
+in event time.
+
+To also guarantee that elements across watermarks are processed in event-time
order, Flink's CEP library assumes
+*correctness of the watermark*, and considers as *late* elements whose
timestamp is smaller than that of the last
+seen watermark. Late elements are not further processed but they can be
redirected to a [side output]
+({{ site.baseurl }}/dev/stream/side_output.html), dedicated to them.
+
+To access the stream of late elements, you first need to specify that you want
to get the late data using
+`.withLateDataOutputTag(OutputTag)` on the `PatternStream` returned using the
`CEP.pattern(...)` call. If you do not do
+so, the late elements will be silently dropped. Then, you can get the
side-output stream using the
+`.getSideOutput(OutputTag)` on the aforementioned `PatternStream`, and
providing as argument the output tag used in
+the `.withLateDataOutputTag(OutputTag)`:
+
+
+
+{% highlight java %}
+final OutputTag lateOutputTag = new OutputTag("late-data"){};
+
+PatternStream patternStream = CEP.pattern(...)
+.withLateDataOutputTag(lateOutputTag);
+
+// main output with matches
+DataStream result = patternStream.select(...)
+
+// side output containing the late events
+DataStream lateStream = patternStream.getSideOutput(lateOutputTag);
+{% endhighlight %}
+
+
+
+{% highlight scala %}
+val lateOutputTag = OutputTag[T]("late-data")
+
+val patternStream: PatternStream[T] = CEP.pattern(...)
+.withLateDataOutputTag(lateOutputTag)
+
+// main output with matches
+val result = patternStream.select(...)
+
+// side output containing the late events
+val lateStream = patternStream.getSideOutput(lateOutputTag)
+{% endhighlight %}
+
+
+
## Examples
The following example detects the pattern `start, middle(name = "error") ->
end(name = "critical")` on a keyed data stream of `Events`.
http://git-wip-us.apache.org/repos/asf/flink/blob/48890285/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
--
diff --git
a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
b/flink-libraries/flink-cep-scala/s
[28/37] flink git commit: [FLINK-6086] [py] Clean up PythonSender/-Streamer generics
[FLINK-6086] [py] Clean up PythonSender/-Streamer generics
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a7251c56
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a7251c56
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a7251c56
Branch: refs/heads/table-retraction
Commit: a7251c56d0d03293f95f0e3b084326a96fe50ce3
Parents: 9bdbe60
Author: zentol
Authored: Thu Mar 16 23:26:09 2017 +0100
Committer: zentol
Committed: Wed Apr 5 20:43:30 2017 +0200
--
.../python/api/functions/PythonCoGroup.java | 6 +-
.../api/functions/PythonMapPartition.java | 6 +-
.../streaming/data/PythonDualInputSender.java | 70 ++
.../streaming/data/PythonDualInputStreamer.java | 97 ++
.../api/streaming/data/PythonReceiver.java | 2 +
.../python/api/streaming/data/PythonSender.java | 100 +++
.../streaming/data/PythonSingleInputSender.java | 50
.../data/PythonSingleInputStreamer.java | 89 +
.../api/streaming/data/PythonStreamer.java | 128 ++-
.../data/SingleElementPushBackIterator.java | 57 +
.../data/SingleElementPushBackIteratorTest.java | 76 +++
11 files changed, 479 insertions(+), 202 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
--
diff --git
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
index 2065b98..72d3361 100644
---
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
+++
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonCoGroup.java
@@ -14,7 +14,7 @@ package org.apache.flink.python.api.functions;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.python.api.streaming.data.PythonStreamer;
+import org.apache.flink.python.api.streaming.data.PythonDualInputStreamer;
import org.apache.flink.util.Collector;
import java.io.IOException;
import org.apache.flink.api.common.functions.RichCoGroupFunction;
@@ -31,12 +31,12 @@ public class PythonCoGroup extends
RichCoGroupFunction streamer;
+ private final PythonDualInputStreamer streamer;
private final transient TypeInformation typeInformation;
public PythonCoGroup(int envID, int setID, TypeInformation
typeInformation) {
this.typeInformation = typeInformation;
- streamer = new PythonStreamer<>(this, envID, setID, true);
+ streamer = new PythonDualInputStreamer<>(this, envID, setID,
true);
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/a7251c56/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java
--
diff --git
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java
index dc21c7c..9142581 100644
---
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java
+++
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/functions/PythonMapPartition.java
@@ -18,7 +18,7 @@ import
org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.python.api.streaming.data.PythonStreamer;
+import org.apache.flink.python.api.streaming.data.PythonSingleInputStreamer;
import org.apache.flink.util.Collector;
/**
@@ -32,12 +32,12 @@ public class PythonMapPartition extends
RichMapPartitionFunction streamer;
+ private final PythonSingleInputStreamer streamer;
private final transient TypeInformation typeInformation;
public PythonMapPartition(int envId, int setId, TypeInformation
typeInformation) {
this.typeInformation = typeInformation;
- streamer = new PythonStreamer(this, envId, setId,
typeInformation instanceof PrimitiveArrayTypeInfo);
+ streamer = new PythonSingleInputStreamer<>(this, envId, setId,
typeInformation instanceof PrimitiveArrayTypeInfo);
}
/**
http://git-wip-us.apac
[17/37] flink git commit: [FLINK-6259] Fix a small spelling error from har-with-dependencies to jar-with-dependencies
[FLINK-6259] Fix a small spelling error from har-with-dependencies to jar-with-dependencies this closes #3667 Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cae4976a Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cae4976a Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cae4976a Branch: refs/heads/table-retraction Commit: cae4976a4b9d4fa67f207dd08b8c9480c6f8989b Parents: 1b6badd Author: sunjincheng121 Authored: Tue Apr 4 08:55:57 2017 +0800 Committer: Jark Wu Committed: Wed Apr 5 09:56:19 2017 +0800 -- flink-libraries/flink-gelly-scala/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/flink/blob/cae4976a/flink-libraries/flink-gelly-scala/pom.xml -- diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml index d6fcb6a..dad7237 100644 --- a/flink-libraries/flink-gelly-scala/pom.xml +++ b/flink-libraries/flink-gelly-scala/pom.xml @@ -216,7 +216,7 @@ under the License. - + maven-assembly-plugin
[05/37] flink git commit: [FLINK-5903] [yarn] Respect taskmanager.numberOfTaskSlots config value in Yarn mode
[FLINK-5903] [yarn] Respect taskmanager.numberOfTaskSlots config value in Yarn
mode
This closes #3408.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2313a74e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2313a74e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2313a74e
Branch: refs/heads/table-retraction
Commit: 2313a74e2726167f9cb586a298d4c397f3b82b69
Parents: 91c90c5
Author: WangTaoTheTonic
Authored: Fri Feb 24 16:11:47 2017 +0800
Committer: Till Rohrmann
Committed: Fri Mar 31 15:37:59 2017 +0200
--
.../java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/2313a74e/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
--
diff --git
a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
index 8928f0a..b383b59 100644
---
a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
+++
b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
@@ -164,6 +164,8 @@ public abstract class AbstractYarnClusterDescriptor
implements ClusterDescriptor
throw new RuntimeException("Unable to locate
configuration file in " + confFile);
}
flinkConfigurationPath = new
Path(confFile.getAbsolutePath());
+
+ slots =
flinkConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
} catch (Exception e) {
LOG.debug("Config couldn't be loaded from environment
variable.");
}
[31/37] flink git commit: [hotfix] [py] Code cleanup - PythonStreamer
[hotfix] [py] Code cleanup - PythonStreamer
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dd588efe
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dd588efe
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dd588efe
Branch: refs/heads/table-retraction
Commit: dd588efe6470dfdcad51249ff36f12cfde7b75da
Parents: 258ed17
Author: zentol
Authored: Thu Mar 30 23:13:07 2017 +0200
Committer: zentol
Committed: Wed Apr 5 20:44:30 2017 +0200
--
.../python/api/streaming/data/PythonStreamer.java | 18 +++---
1 file changed, 15 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/dd588efe/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
--
diff --git
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
index 3c79b1f..006a1b2 100644
---
a/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
+++
b/flink-libraries/flink-python/src/main/java/org/apache/flink/python/api/streaming/data/PythonStreamer.java
@@ -84,7 +84,7 @@ public class PythonStreamer
implements Serializable
this.setID = setID;
this.usePython3 = PythonPlanBinder.usePython3;
planArguments = PythonPlanBinder.arguments.toString();
- receiver = new PythonReceiver(usesByteArray);
+ receiver = new PythonReceiver<>(usesByteArray);
this.function = function;
this.sender = sender;
}
@@ -162,10 +162,22 @@ public class PythonStreamer
implements Serializable
private void checkPythonProcessHealth() {
try {
int value = process.exitValue();
+ try {
+ outPrinter.join();
+ } catch (InterruptedException ignored) {
+ outPrinter.interrupt();
+ Thread.interrupted();
+ }
+ try {
+ errorPrinter.join();
+ } catch (InterruptedException ignored) {
+ errorPrinter.interrupt();
+ Thread.interrupted();
+ }
if (value != 0) {
- throw new RuntimeException("Plan file caused an
error. Check log-files for details.");
+ throw new RuntimeException("Plan file caused an
error. Check log-files for details." + msg.get());
} else {
- throw new RuntimeException("Plan file exited
prematurely without an error.");
+ throw new RuntimeException("Plan file exited
prematurely without an error." + msg.get());
}
} catch (IllegalThreadStateException ignored) {//Process still
running
}
[33/37] flink git commit: [FLINK-6181][Start scripts] Fix regex in start scripts
[FLINK-6181][Start scripts] Fix regex in start scripts
This closes #3605.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dc13500d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dc13500d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dc13500d
Branch: refs/heads/table-retraction
Commit: dc13500dd5c8061eb75a64a62db6aa13f8fd5408
Parents: 5c44f53
Author: Robert Metzger
Authored: Fri Mar 24 11:50:11 2017 +0100
Committer: zentol
Committed: Wed Apr 5 20:48:44 2017 +0200
--
flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh | 2 +-
flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/dc13500d/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
--
diff --git a/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
b/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
index 05a5208..cc8917b 100755
--- a/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
+++ b/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
@@ -35,7 +35,7 @@ while read server ; do
server=$(echo -e "${server}" | sed -e 's/^[[:space:]]*//' -e
's/[[:space:]]*$//') # trim
# match server.id=address[:port[:port]]
-if [[ $server =~ ^server\.([0-9]+)[[:space:]]*\=([^: \#]+) ]]; then
+if [[ $server =~ ^server\.([0-9]+)[[:space:]]*\=[[:space:]]*([^: \#]+) ]];
then
id=${BASH_REMATCH[1]}
address=${BASH_REMATCH[2]}
http://git-wip-us.apache.org/repos/asf/flink/blob/dc13500d/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
--
diff --git a/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
b/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
index b306648..29ddae4 100755
--- a/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
+++ b/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
@@ -35,7 +35,7 @@ while read server ; do
server=$(echo -e "${server}" | sed -e 's/^[[:space:]]*//' -e
's/[[:space:]]*$//') # trim
# match server.id=address[:port[:port]]
-if [[ $server =~ ^server\.([0-9])+[[:space:]]*\=([^: \#]+) ]]; then
+if [[ $server =~ ^server\.([0-9]+)[[:space:]]*\=[[:space:]]*([^: \#]+) ]];
then
id=${BASH_REMATCH[1]}
server=${BASH_REMATCH[2]}
[06/37] flink git commit: [FLINK-2814] [optimizer] DualInputPlanNode cannot be cast to SingleInputPlanNode
[FLINK-2814] [optimizer] DualInputPlanNode cannot be cast to SingleInputPlanNode
WorksetIterationNode#instantiate loops over all solution and work set
candidates. Since the solution set reference is modified in place when
the predecessor node can be used in its place, swith this variable to
the inner loop.
This closes #3563
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/43158a86
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/43158a86
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/43158a86
Branch: refs/heads/table-retraction
Commit: 43158a86d9c3f3d6e3d612c8871625e9d22e7ba3
Parents: 2313a74
Author: Greg Hogan
Authored: Fri Mar 17 12:09:34 2017 -0400
Committer: Greg Hogan
Committed: Fri Mar 31 11:05:58 2017 -0400
--
.../org/apache/flink/optimizer/dag/WorksetIterationNode.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/43158a86/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
--
diff --git
a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
index d7ccaca..e62e727 100644
---
a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
+++
b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
@@ -431,8 +431,8 @@ public class WorksetIterationNode extends TwoInputNode
implements IterationNode
LocalProperties lp =
LocalProperties.EMPTY.addUniqueFields(this.solutionSetKeyFields);
// take all combinations of solution set delta and workset plans
- for (PlanNode solutionSetCandidate :
solutionSetDeltaCandidates) {
- for (PlanNode worksetCandidate : worksetCandidates) {
+ for (PlanNode worksetCandidate : worksetCandidates) {
+ for (PlanNode solutionSetCandidate :
solutionSetDeltaCandidates) {
// check whether they have the same operator at
their latest branching point
if
(this.singleRoot.areBranchCompatible(solutionSetCandidate, worksetCandidate)) {
[35/37] flink git commit: [FLINK-6203] [docs] [batch] Fix scala GroupReduce example
[FLINK-6203] [docs] [batch] Fix scala GroupReduce example
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cf28a562
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cf28a562
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cf28a562
Branch: refs/heads/table-retraction
Commit: cf28a562159a7a73deb2c214c221bd935250387b
Parents: dabb0ba
Author: zentol
Authored: Wed Apr 5 23:40:32 2017 +0200
Committer: zentol
Committed: Wed Apr 5 23:42:07 2017 +0200
--
docs/dev/batch/dataset_transformations.md | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/cf28a562/docs/dev/batch/dataset_transformations.md
--
diff --git a/docs/dev/batch/dataset_transformations.md
b/docs/dev/batch/dataset_transformations.md
index 1f942d4..fb85b07 100644
--- a/docs/dev/batch/dataset_transformations.md
+++ b/docs/dev/batch/dataset_transformations.md
@@ -620,6 +620,7 @@ val output = input.groupBy(0).sortGroup(1,
Order.ASCENDING).reduceGroup {
for (t <- in) {
if (prev == null || prev != t)
out.collect(t)
+prev = t
}
}
[21/37] flink git commit: [FLINK-6201] [py] [dist] move python example files from resources to the examples
[FLINK-6201] [py] [dist] move python example files from resources to the examples This closes #3628. Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/995422c4 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/995422c4 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/995422c4 Branch: refs/heads/table-retraction Commit: 995422c461bc2e58bc1590c313b940e05c0fabef Parents: c8f8a41 Author: Jinkui Shi Authored: Tue Mar 28 17:51:12 2017 +0800 Committer: zentol Committed: Wed Apr 5 20:41:13 2017 +0200 -- flink-dist/src/main/assemblies/bin.xml | 10 ++ 1 file changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/flink/blob/995422c4/flink-dist/src/main/assemblies/bin.xml -- diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml index 901cac9..e1ee00e 100644 --- a/flink-dist/src/main/assemblies/bin.xml +++ b/flink-dist/src/main/assemblies/bin.xml @@ -178,7 +178,17 @@ under the License. ../flink-libraries/flink-python/src/main/python/org/apache/flink/python/api resources/python 0755 + + **/example/** + + + + + ../flink-libraries/flink-python/src/main/python/org/apache/flink/python/api/flink/example + examples/python + 0755 +
[04/37] flink git commit: [FLINK-5915] [table] Forward the complete aggregate ArgList to aggregate runtime functions.
[FLINK-5915] [table] Forward the complete aggregate ArgList to aggregate
runtime functions.
This closes #3647.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/91c90c5d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/91c90c5d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/91c90c5d
Branch: refs/heads/table-retraction
Commit: 91c90c5d2d38c286e128777aa7b8e14fc5321575
Parents: 4889028
Author: shaoxuan-wang
Authored: Fri Mar 31 17:07:09 2017 +0800
Committer: Fabian Hueske
Committed: Fri Mar 31 15:14:37 2017 +0200
--
.../table/runtime/aggregate/AggregateAggFunction.scala | 4 ++--
.../flink/table/runtime/aggregate/AggregateUtil.scala | 12 +++-
.../BoundedProcessingOverRowProcessFunction.scala | 6 +++---
.../table/runtime/aggregate/DataSetAggFunction.scala| 4 ++--
.../table/runtime/aggregate/DataSetPreAggFunction.scala | 4 ++--
.../runtime/aggregate/DataSetWindowAggMapFunction.scala | 4 ++--
.../RangeClauseBoundedOverProcessFunction.scala | 6 +++---
.../RowsClauseBoundedOverProcessFunction.scala | 6 +++---
.../UnboundedEventTimeOverProcessFunction.scala | 10 +-
...dedNonPartitionedProcessingOverProcessFunction.scala | 4 ++--
.../UnboundedProcessingOverProcessFunction.scala| 4 ++--
11 files changed, 33 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/91c90c5d/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
index 11d55e5..c608b97 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
@@ -33,7 +33,7 @@ import org.apache.flink.types.Row
*/
class AggregateAggFunction(
private val aggregates: Array[AggregateFunction[_]],
-private val aggFields: Array[Int])
+private val aggFields: Array[Array[Int]])
extends DataStreamAggFunc[Row, Row, Row] {
override def createAccumulator(): Row = {
@@ -51,7 +51,7 @@ class AggregateAggFunction(
var i = 0
while (i < aggregates.length) {
val acc = accumulatorRow.getField(i).asInstanceOf[Accumulator]
- val v = value.getField(aggFields(i))
+ val v = value.getField(aggFields(i)(0))
aggregates(i).accumulate(acc, v)
i += 1
}
http://git-wip-us.apache.org/repos/asf/flink/blob/91c90c5d/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 88e9d68..74dc5cd 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -45,6 +45,7 @@ import org.apache.flink.table.typeutils.{RowIntervalTypeInfo,
TimeIntervalTypeIn
import org.apache.flink.types.Row
import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
object AggregateUtil {
@@ -886,10 +887,10 @@ object AggregateUtil {
aggregateCalls: Seq[AggregateCall],
inputType: RelDataType,
needRetraction: Boolean)
- : (Array[Int], Array[TableAggregateFunction[_ <: Any]]) = {
+ : (Array[Array[Int]], Array[TableAggregateFunction[_ <: Any]]) = {
// store the aggregate fields of each aggregate function, by the same
order of aggregates.
-val aggFieldIndexes = new Array[Int](aggregateCalls.size)
+val aggFieldIndexes = new Array[Array[Int]](aggregateCalls.size)
val aggregates = new Array[TableAggregateFunction[_ <:
Any]](aggregateCalls.size)
// create aggregate function instances by function type and aggregate
field data type.
@@ -897,7 +898,7 @@ object AggregateUtil {
val argList: util.List[Integer] = aggregateCall.getArgList
if (argList.isEmpty) {
if (aggregateCall.getAggregation.isInstanceOf[SqlCountAggFunction]) {
- aggFieldIndexes(index) = 0
+ aggFieldIndexes(index) = Array[Int](0)
} else {
thro
[12/37] flink git commit: [FLINK-5654] [table] Add processing-time OVER RANGE BETWEEN x PRECEDING aggregation to SQL.
[FLINK-5654] [table] Add processing-time OVER RANGE BETWEEN x PRECEDING
aggregation to SQL.
This closes #3641.
This closes #3590.
This closes #3550.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/31e120a9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/31e120a9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/31e120a9
Branch: refs/heads/table-retraction
Commit: 31e120a98da673ee12ae5879d95243fa0b555e00
Parents: a48357d
Author: rtudoran
Authored: Wed Mar 29 12:02:11 2017 +0200
Committer: Fabian Hueske
Committed: Fri Mar 31 21:42:48 2017 +0200
--
flink-libraries/flink-table/pom.xml | 14 ++
.../datastream/DataStreamOverAggregate.scala| 10 +-
.../table/runtime/aggregate/AggregateUtil.scala | 25 ++-
...ndedProcessingOverRangeProcessFunction.scala | 203 ++
.../scala/stream/sql/WindowAggregateTest.scala | 53 +
...ProcessingOverRangeProcessFunctionTest.scala | 204 +++
6 files changed, 497 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/31e120a9/flink-libraries/flink-table/pom.xml
--
diff --git a/flink-libraries/flink-table/pom.xml
b/flink-libraries/flink-table/pom.xml
index a2945e8..6bcddc2 100644
--- a/flink-libraries/flink-table/pom.xml
+++ b/flink-libraries/flink-table/pom.xml
@@ -140,6 +140,20 @@ under the License.
${project.version}
test
+
+ org.apache.flink
+ flink-streaming-java_2.10
+ ${project.version}
+ test-jar
+ test
+
+
+ org.apache.flink
+ flink-runtime_2.10
+ ${project.version}
+ test
+ test-jar
+
http://git-wip-us.apache.org/repos/asf/flink/blob/31e120a9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index 2df4e02..e15db01 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -114,12 +114,14 @@ class DataStreamOverAggregate(
// ROWS clause bounded OVER window
createBoundedAndCurrentRowOverWindow(
inputDS,
- isRangeClause = true,
+ isRangeClause = false,
isRowTimeType = false)
} else {
// RANGE clause bounded OVER window
-throw new TableException(
- "processing-time OVER RANGE PRECEDING window is not supported
yet.")
+createBoundedAndCurrentRowOverWindow(
+ inputDS,
+ isRangeClause = true,
+ isRowTimeType = false)
}
} else {
throw new TableException(
@@ -206,7 +208,7 @@ class DataStreamOverAggregate(
val namedAggregates: Seq[CalcitePair[AggregateCall, String]] =
generateNamedAggregates
val precedingOffset =
- getLowerBoundary(logicWindow, overWindow, getInput()) + 1
+ getLowerBoundary(logicWindow, overWindow, getInput()) + (if
(isRangeClause) 0 else 1)
// get the output types
val rowTypeInfo =
FlinkTypeFactory.toInternalRowTypeInfo(getRowType).asInstanceOf[RowTypeInfo]
http://git-wip-us.apache.org/repos/asf/flink/blob/31e120a9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
--
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 74dc5cd..caa2818 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -139,13 +139,23 @@ object AggregateUtil {
)
}
} else {
- new BoundedProcessingOverRowProcessFunction(
-aggre
[13/37] flink git commit: [FLINK-4949] [gelly] Refactor Gelly driver inputs
[FLINK-4949] [gelly] Refactor Gelly driver inputs The Gelly drivers started as simple wrappers around library algorithms but have grown to handle a matrix of input sources while often running multiple algorithms and analytics with custom parameterization. The monolithic drivers are replaced with separate inputs and algorithms. Command-line parameter parsers are shared and reusable across inputs and algorithms. Algorithm results now implement a common AlgorithmResult interface. Drivers are now tested with integration tests. This closes #3294 Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f1ff99fd Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f1ff99fd Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f1ff99fd Branch: refs/heads/table-retraction Commit: f1ff99fdc1e228acd936f5684832d5cf49bdbe04 Parents: 31e120a Author: Greg Hogan Authored: Wed Oct 26 15:18:50 2016 -0400 Committer: Greg Hogan Committed: Fri Mar 31 15:57:54 2017 -0400 -- docs/dev/libs/gelly/index.md| 40 ++- flink-libraries/flink-gelly-examples/pom.xml| 9 +- .../java/org/apache/flink/graph/Runner.java | 357 +++ .../main/java/org/apache/flink/graph/Usage.java | 71 .../org/apache/flink/graph/RunnerITCase.java| 122 +++ .../flink/graph/drivers/AdamicAdarITCase.java | 52 +++ .../drivers/ClusteringCoefficientITCase.java| 89 + .../drivers/ConnectedComponentsITCase.java | 65 .../flink/graph/drivers/DriverBaseITCase.java | 185 ++ .../flink/graph/drivers/EdgeListITCase.java | 240 + .../flink/graph/drivers/GraphMetricsITCase.java | 100 ++ .../apache/flink/graph/drivers/HITSITCase.java | 52 +++ .../flink/graph/drivers/JaccardIndexITCase.java | 63 .../flink/graph/drivers/PageRankITCase.java | 52 +++ .../graph/drivers/TriangleListingITCase.java| 107 ++ 15 files changed, 1518 insertions(+), 86 deletions(-) -- http://git-wip-us.apache.org/repos/asf/flink/blob/f1ff99fd/docs/dev/libs/gelly/index.md -- diff --git a/docs/dev/libs/gelly/index.md b/docs/dev/libs/gelly/index.md index 8f8c6de..40018e8 100644 --- a/docs/dev/libs/gelly/index.md +++ b/docs/dev/libs/gelly/index.md @@ -73,7 +73,7 @@ Running Gelly Examples The Gelly library and examples jars are provided in the [Flink distribution](https://flink.apache.org/downloads.html "Apache Flink: Downloads") in the folder **opt** (for versions older than Flink 1.2 these can be manually downloaded from -[Maven Central](http://search.maven.org/#search|ga|1|flink%20gelly). +[Maven Central](http://search.maven.org/#search|ga|1|flink%20gelly)). To run the Gelly examples the **flink-gelly** (for Java) or **flink-gelly-scala** (for Scala) jar must be copied to Flink's **lib** directory. @@ -83,21 +83,29 @@ cp opt/flink-gelly_*.jar lib/ cp opt/flink-gelly-scala_*.jar lib/ ~~~ -Gelly's examples jar includes both drivers for the library methods as well as additional example algorithms. After -configuring and starting the cluster, list the available algorithm classes: +Gelly's examples jar includes drivers for each of the library methods. After configuring and starting the cluster, list +the available algorithm classes: ~~~bash ./bin/start-cluster.sh ./bin/flink run opt/flink-gelly-examples_*.jar ~~~ -The Gelly drivers can generate [RMat](http://www.cs.cmu.edu/~christos/PUBLICATIONS/siam04.pdf) graph data or read the -edge list from a CSV file. Each node in a cluster must have access to the input file. Calculate graph metrics on a -directed generated graph: +The Gelly drivers can generate graph data or read the edge list from a CSV file (each node in a cluster must have access +to the input file). The algorithm description, available inputs and outputs, and configuration are displayed when an +algorithm is selected. Print usage for [JaccardIndex](./library_methods.html#jaccard-index): ~~~bash -./bin/flink run -c org.apache.flink.graph.drivers.GraphMetrics opt/flink-gelly-examples_*.jar \ ---directed true --input rmat +./bin/flink run opt/flink-gelly-examples_*.jar --algorithm JaccardIndex +~~~ + +Display [graph metrics](./library_methods.html#metric) for a million vertex graph: + +~~~bash +./bin/flink run opt/flink-gelly-examples_*.jar \ +--algorithm GraphMetrics --order directed \ +--input RMatGraph --type integer --scale 20 --simplify directed \ +--output print ~~~ The size of the graph is adjusted by the *\-\-scale* and *\-\-edge_factor* parameters. The @@ -111,15 +119,19 @@ Run a few algorithms and monitor the job progress in Flink's Web UI: ~~~bash wget -O - http://snap.stanford.edu/data/bigdata/communities/com-lj.ungrap
[16/37] flink git commit: [FLINK-6254] [cep] Same method name for late data outputs on PatternStream and WindowedStream
[FLINK-6254] [cep] Same method name for late data outputs on PatternStream and
WindowedStream
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1b6baddc
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1b6baddc
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1b6baddc
Branch: refs/heads/table-retraction
Commit: 1b6baddca07bfba6093951e82ac9108cf4728f2a
Parents: c5282cb
Author: kl0u
Authored: Mon Apr 3 17:39:02 2017 +0200
Committer: kl0u
Committed: Tue Apr 4 11:29:09 2017 +0200
--
docs/dev/libs/cep.md | 10 +-
.../scala/org/apache/flink/cep/scala/PatternStream.scala | 4 ++--
.../src/main/java/org/apache/flink/cep/PatternStream.java | 2 +-
.../src/test/java/org/apache/flink/cep/CEPITCase.java | 2 +-
4 files changed, 9 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/flink/blob/1b6baddc/docs/dev/libs/cep.md
--
diff --git a/docs/dev/libs/cep.md b/docs/dev/libs/cep.md
index 643d6ee..15afdf5 100644
--- a/docs/dev/libs/cep.md
+++ b/docs/dev/libs/cep.md
@@ -790,13 +790,13 @@ in event time.
To also guarantee that elements across watermarks are processed in event-time
order, Flink's CEP library assumes
*correctness of the watermark*, and considers as *late* elements whose
timestamp is smaller than that of the last
seen watermark. Late elements are not further processed but they can be
redirected to a [side output]
-({{ site.baseurl }}/dev/stream/side_output.html), dedicated to them.
+({{ site.baseurl }}/dev/stream/side_output.html) dedicated to them.
To access the stream of late elements, you first need to specify that you want
to get the late data using
-`.withLateDataOutputTag(OutputTag)` on the `PatternStream` returned using the
`CEP.pattern(...)` call. If you do not do
+`.sideOutputLateData(OutputTag)` on the `PatternStream` returned using the
`CEP.pattern(...)` call. If you do not do
so, the late elements will be silently dropped. Then, you can get the
side-output stream using the
`.getSideOutput(OutputTag)` on the aforementioned `PatternStream`, and
providing as argument the output tag used in
-the `.withLateDataOutputTag(OutputTag)`:
+the `.sideOutputLateData(OutputTag)`:
@@ -804,7 +804,7 @@ the `.withLateDataOutputTag(OutputTag)`:
final OutputTag lateOutputTag = new OutputTag("late-data"){};
PatternStream patternStream = CEP.pattern(...)
-.withLateDataOutputTag(lateOutputTag);
+.sideOutputLateData(lateOutputTag);
// main output with matches
DataStream result = patternStream.select(...)
@@ -819,7 +819,7 @@ DataStream lateStream =
patternStream.getSideOutput(lateOutputTag);
val lateOutputTag = OutputTag[T]("late-data")
val patternStream: PatternStream[T] = CEP.pattern(...)
-.withLateDataOutputTag(lateOutputTag)
+.sideOutputLateData(lateOutputTag)
// main output with matches
val result = patternStream.select(...)
http://git-wip-us.apache.org/repos/asf/flink/blob/1b6baddc/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
--
diff --git
a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
index fb09c15..7c92886 100644
---
a/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
+++
b/flink-libraries/flink-cep-scala/src/main/scala/org/apache/flink/cep/scala/PatternStream.scala
@@ -57,8 +57,8 @@ class PatternStream[T](jPatternStream: JPatternStream[T]) {
* timestamp is smaller than the last received watermark.
*/
@PublicEvolving
- def withLateDataOutputTag(outputTag: OutputTag[T]): PatternStream[T] = {
-jPatternStream.withLateDataOutputTag(outputTag)
+ def sideOutputLateData(outputTag: OutputTag[T]): PatternStream[T] = {
+jPatternStream.sideOutputLateData(outputTag)
lateDataOutputTag = outputTag
this
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1b6baddc/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/PatternStream.java
--
diff --git
a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/PatternStream.java
b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/PatternStream.java
index 87666a5..5f2327c 100644
---
a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/PatternStream.java
+++
b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/PatternStream.java
@@ -84,7 +84,7 @@ public class PatternStream {
* CEP library assumes
