[1/2] flink git commit: [FLINK-6228] [table] Add support for OVER windows to streaming Table API.

[fhueske]

Repository: flink
Updated Branches:
  refs/heads/master 166444767 -> fe018921e


http://git-wip-us.apache.org/repos/asf/flink/blob/fe018921/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
--
diff --git 
a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
 
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
new file mode 100644
index 000..0a5e001
--- /dev/null
+++ 
b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/stringexpr/OverWindowStringExpressionTest.scala
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.scala.stream.table.stringexpr
+
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.java.{Over => JOver}
+import org.apache.flink.table.api.scala.{Over => SOver, _}
+import org.apache.flink.table.utils.TableTestBase
+import org.junit.Test
+
+class OverWindowStringExpressionTest extends TableTestBase {
+
+  @Test
+  def testPartitionedUnboundedOverRow(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+  .window(SOver partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_ROW as 
'w)
+  .select('a, 'b.sum over 'w)
+val resJava = t
+  
.window(JOver.partitionBy("a").orderBy("rowtime").preceding("unbounded_row").as("w"))
+  .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testUnboundedOverRow(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+  .window(SOver orderBy 'rowtime preceding UNBOUNDED_ROW following 
CURRENT_ROW as 'w)
+  .select('a, 'b.sum over 'w)
+val resJava = t
+  
.window(JOver.orderBy("rowtime").preceding("unbounded_row").following("current_row").as("w"))
+  .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testPartitionedBoundedOverRow(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+  .window(SOver partitionBy('a, 'd) orderBy 'rowtime preceding 10.rows as 
'w)
+  .select('a, 'b.sum over 'w)
+val resJava = t
+  .window(JOver.partitionBy("a, 
d").orderBy("rowtime").preceding("10.rows").as("w"))
+  .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testBoundedOverRow(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+  .window(SOver orderBy 'rowtime preceding 10.rows following CURRENT_ROW 
as 'w)
+  .select('a, 'b.sum over 'w)
+val resJava = t
+  
.window(JOver.orderBy("rowtime").preceding("10.rows").following("current_row").as("w"))
+  .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testPartitionedUnboundedOverRange(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+  .window(SOver partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_RANGE 
as 'w)
+  .select('a, 'b.sum over 'w)
+val resJava = t
+  
.window(JOver.partitionBy("a").orderBy("rowtime").preceding("unbounded_range").as("w"))
+  .select("a, SUM(b) OVER w")
+
+verifyTableEquals(resScala, resJava)
+  }
+
+  @Test
+  def testUnboundedOverRange(): Unit = {
+val util = streamTestUtil()
+val t = util.addTable[(Long, Int, String, Int, Long)]('a, 'b, 'c, 'd, 'e)
+
+val resScala = t
+  .window(SOver orderBy 'rowtime preceding UNBOUNDED_RANGE following 
CURRENT_RANGE as 'w)
+  .select('a, 'b.sum over 'w)
+val resJava = t
+  .window(
+

[2/2] flink git commit: [FLINK-6228] [table] Add support for OVER windows to streaming Table API.

[fhueske]

[FLINK-6228] [table] Add support for OVER windows to streaming Table API.

This closes #3743.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fe018921
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fe018921
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fe018921

Branch: refs/heads/master
Commit: fe018921ed0b24f94ab2139f04293d6074ce4fba
Parents: 1664447
Author: sunjincheng121 
Authored: Thu Apr 13 17:36:18 2017 +0800
Committer: Fabian Hueske 
Committed: Mon Apr 24 18:39:30 2017 +0200

--
 .../flink/table/api/java/groupWindows.scala |  84 ---
 .../apache/flink/table/api/java/windows.scala   | 129 
 .../flink/table/api/scala/expressionDsl.scala   |  27 +-
 .../flink/table/api/scala/groupWindows.scala|  85 ---
 .../apache/flink/table/api/scala/windows.scala  | 126 
 .../org/apache/flink/table/api/table.scala  |  62 ++
 .../org/apache/flink/table/api/windows.scala| 118 +++-
 .../table/expressions/ExpressionParser.scala|  64 +-
 .../flink/table/expressions/aggregations.scala  |  36 +-
 .../apache/flink/table/expressions/call.scala   | 211 ++-
 .../flink/table/plan/ProjectionTranslator.scala |  51 +-
 .../scala/stream/table/OverWindowITCase.scala   | 331 ++
 .../api/scala/stream/table/OverWindowTest.scala | 596 +++
 .../OverWindowStringExpressionTest.scala| 151 +
 14 files changed, 1880 insertions(+), 191 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/flink/blob/fe018921/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
--
diff --git 
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
 
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
deleted file mode 100644
index 9c82e9b..000
--- 
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/java/groupWindows.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.api.java
-
-import org.apache.flink.table.api.{SessionWindow, SlideWithSize, 
TumblingWindow}
-
-/**
-  * Helper class for creating a tumbling window. Tumbling windows are 
consecutive, non-overlapping
-  * windows of a specified fixed length. For example, a tumbling window of 5 
minutes size groups
-  * elements in 5 minutes intervals.
-  */
-object Tumble {
-
-  /**
-* Creates a tumbling window. Tumbling windows are consecutive, 
non-overlapping
-* windows of a specified fixed length. For example, a tumbling window of 5 
minutes size groups
-* elements in 5 minutes intervals.
-*
-* @param size the size of the window as time or row-count interval.
-* @return a tumbling window
-*/
-  def over(size: String): TumblingWindow = new TumblingWindow(size)
-}
-
-/**
-  * Helper class for creating a sliding window. Sliding windows have a fixed 
size and slide by
-  * a specified slide interval. If the slide interval is smaller than the 
window size, sliding
-  * windows are overlapping. Thus, an element can be assigned to multiple 
windows.
-  *
-  * For example, a sliding window of size 15 minutes with 5 minutes sliding 
interval groups elements
-  * of 15 minutes and evaluates every five minutes. Each element is contained 
in three consecutive
-  * window evaluations.
-  */
-object Slide {
-
-  /**
-* Creates a sliding window. Sliding windows have a fixed size and slide by
-* a specified slide interval. If the slide interval is smaller than the 
window size, sliding
-* windows are overlapping. Thus, an element can be assigned to multiple 
windows.
-*
-* For example, a sliding window of size 15 minutes with 5 minutes sliding 
interval groups
-* elements of 15 minutes and evaluates every five minutes. Each element is 
contained in three
-* consecutive window evaluations.
-  

flink git commit: [FLINK-6280] [scripts] Allow logging with Java flags

[greg]

Repository: flink
Updated Branches:
  refs/heads/master aa21f853a -> 166444767


[FLINK-6280] [scripts] Allow logging with Java flags

Evaluate user-defined Java options immediately before starting services
and rotate all log files.

This closes #3701


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/16644476
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/16644476
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/16644476

Branch: refs/heads/master
Commit: 1664447677404e66ef842350d30469c82286cb41
Parents: aa21f85
Author: Greg Hogan 
Authored: Fri Apr 7 13:34:16 2017 -0400
Committer: Greg Hogan 
Committed: Mon Apr 24 10:12:50 2017 -0400

--
 docs/monitoring/application_profiling.md| 54 
 docs/setup/config.md|  8 ++-
 flink-dist/src/main/flink-bin/bin/config.sh | 11 +++-
 .../src/main/flink-bin/bin/flink-daemon.sh  | 11 ++--
 4 files changed, 78 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/flink/blob/16644476/docs/monitoring/application_profiling.md
--
diff --git a/docs/monitoring/application_profiling.md 
b/docs/monitoring/application_profiling.md
new file mode 100644
index 000..65ef45e
--- /dev/null
+++ b/docs/monitoring/application_profiling.md
@@ -0,0 +1,54 @@
+---
+title: "Application Profiling"
+nav-parent_id: monitoring
+nav-pos: 15
+---
+
+
+* ToC
+{:toc}
+
+## Overview of Custom Logging with Apache Flink
+
+Each standalone JobManager, TaskManager, HistoryServer, and ZooKeeper daemon 
redirects `stdout` and `stderr` to a file
+with a `.out` filename suffix and writes internal logging to a file with a 
`.log` suffix. Java options configured by the
+user in `env.java.opts`, `env.java.opts.jobmanager`, and 
`env.java.opts.taskmanager` can likewise define log files with
+use of the script variable `FLINK_LOG_PREFIX` and by enclosing the options in 
double quotes for late evaluation. Log files
+using `FLINK_LOG_PREFIX` are rotated along with the default `.out` and `.log` 
files.
+
+# Profiling with Java Flight Recorder
+
+Java Flight Recorder is a profiling and event collection framework built into 
the Oracle JDK.
+[Java Mission 
Control](http://www.oracle.com/technetwork/java/javaseproducts/mission-control/java-mission-control-1998576.html)
+is an advanced set of tools that enables efficient and detailed analysis of 
the extensive of data collected by Java
+Flight Recorder. Example configuration:
+
+~~~
+env.java.opts: "-XX:+UnlockCommercialFeatures -XX:+UnlockDiagnosticVMOptions 
-XX:+FlightRecorder -XX:+DebugNonSafepoints 
-XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=${FLINK_LOG_PREFIX}.jfr"
+~~~
+
+# Profiling with JITWatch
+
+[JITWatch](https://github.com/AdoptOpenJDK/jitwatch/wiki) is a log analyser 
and visualizer for the Java HotSpot JIT
+compiler used to inspect inlining decisions, hot methods, bytecode, and 
assembly. Example configuration:
+
+~~~
+env.java.opts: "-XX:+UnlockDiagnosticVMOptions -XX:+TraceClassLoading 
-XX:+LogCompilation -XX:LogFile=${FLINK_LOG_PREFIX}.jit -XX:+PrintAssembly"
+~~~

http://git-wip-us.apache.org/repos/asf/flink/blob/16644476/docs/setup/config.md
--
diff --git a/docs/setup/config.md b/docs/setup/config.md
index 6a5c1ff..e36f149 100644
--- a/docs/setup/config.md
+++ b/docs/setup/config.md
@@ -42,7 +42,11 @@ The configuration files for the TaskManagers can be 
different, Flink does not as
 
 - `env.java.home`: The path to the Java installation to use (DEFAULT: system's 
default Java installation, if found). Needs to be specified if the startup 
scripts fail to automatically resolve the java home directory. Can be specified 
to point to a specific java installation or version. If this option is not 
specified, the startup scripts also evaluate the `$JAVA_HOME` environment 
variable.
 
-- `env.java.opts`: Set custom JVM options. This value is respected by Flink's 
start scripts, both JobManager and TaskManager, and Flink's YARN client. This 
can be used to set different garbage collectors or to include remote debuggers 
into the JVMs running Flink's services. Use `env.java.opts.jobmanager` and 
`env.java.opts.taskmanager` for JobManager or TaskManager-specific options, 
respectively.
+- `env.java.opts`: Set custom JVM options. This value is respected by Flink's 
start scripts, both JobManager and
+TaskManager, and Flink's YARN client. This can be used to set different 
garbage collectors or to include remote
+debuggers into the JVMs running Flink's services. Enclosing options in double 
quotes delays parameter substitution
+allowing access to