[GitHub] johnyangk closed pull request #137: [NEMO-232] Implement InputWatermarkManager
johnyangk closed pull request #137: [NEMO-232] Implement InputWatermarkManager
URL: https://github.com/apache/incubator-nemo/pull/137
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pom.xml b/pom.xml
index 1675e9c0b..3a81df7f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -235,17 +235,17 @@ under the License.
maven-javadoc-plugin
3.0.0
-
*.org.apache.nemo.runtime.common.comm
-docs/apidocs
-docs/apidocs
+
*.org.apache.nemo.runtime.common.comm
+ docs/apidocs
+ docs/apidocs
-aggregate
-
-aggregate
-
-site
+ aggregate
+
+ aggregate
+
+ site
test-javadoc
diff --git
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/PipeManagerWorker.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/PipeManagerWorker.java
index ffbbe56b1..a433f3a2c 100644
---
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/PipeManagerWorker.java
+++
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/PipeManagerWorker.java
@@ -141,8 +141,8 @@ public void notifyMaster(final String runtimeEdgeId, final
long srcTaskIndex) {
/**
* (SYNCHRONIZATION) Called by task threads.
*
- * @param runtimeEdge
- * @param srcTaskIndex
+ * @param runtimeEdge runtime edge
+ * @param srcTaskIndex source task index
* @return output contexts.
*/
public List getOutputContexts(final RuntimeEdge
runtimeEdge,
@@ -163,8 +163,8 @@ public Serializer getSerializer(final String runtimeEdgeId)
{
/**
* (SYNCHRONIZATION) Called by network threads.
*
- * @param outputContext
- * @throws InvalidProtocolBufferException
+ * @param outputContext output context
+ * @throws InvalidProtocolBufferException protobuf exception
*/
public void onOutputContext(final ByteOutputContext outputContext) throws
InvalidProtocolBufferException {
final ControlMessage.PipeTransferContextDescriptor descriptor =
diff --git
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/DataFetcherOutputCollector.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/DataFetcherOutputCollector.java
index 3f1bc9074..56c754038 100644
---
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/DataFetcherOutputCollector.java
+++
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/DataFetcherOutputCollector.java
@@ -34,6 +34,7 @@
/**
* It forwards output to the next operator.
+ * @param nextOperatorVertex next operator to emit data and watermark
*/
public DataFetcherOutputCollector(final OperatorVertex nextOperatorVertex) {
this.nextOperatorVertex = nextOperatorVertex;
diff --git
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/InputWatermarkManager.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/InputWatermarkManager.java
new file mode 100644
index 0..66fb7aa81
--- /dev/null
+++
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/InputWatermarkManager.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.nemo.runtime.executor.datatransfer;
+
+import org.apache.nemo.common.punctuation.Watermark;
+
+
+/**
+ * An interface for tracking input watermarks among multiple input streams.
+ * --edge 1--
+ * --edge 2-- watermarkManager --(emitWatermark)-- nextOperator
+ * --edge 3--
+ */
+public
[GitHub] johnyangk closed pull request #137: [NEMO-232] Implement InputWatermarkManager
johnyangk closed pull request #137: [NEMO-232] Implement InputWatermarkManager
URL: https://github.com/apache/incubator-nemo/pull/137
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pom.xml b/pom.xml
index 1675e9c0b..3a81df7f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -235,17 +235,17 @@ under the License.
maven-javadoc-plugin
3.0.0
-
*.org.apache.nemo.runtime.common.comm
-docs/apidocs
-docs/apidocs
+
*.org.apache.nemo.runtime.common.comm
+ docs/apidocs
+ docs/apidocs
-aggregate
-
-aggregate
-
-site
+ aggregate
+
+ aggregate
+
+ site
test-javadoc
diff --git
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/PipeManagerWorker.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/PipeManagerWorker.java
index ffbbe56b1..a433f3a2c 100644
---
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/PipeManagerWorker.java
+++
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/PipeManagerWorker.java
@@ -141,8 +141,8 @@ public void notifyMaster(final String runtimeEdgeId, final
long srcTaskIndex) {
/**
* (SYNCHRONIZATION) Called by task threads.
*
- * @param runtimeEdge
- * @param srcTaskIndex
+ * @param runtimeEdge runtime edge
+ * @param srcTaskIndex source task index
* @return output contexts.
*/
public List getOutputContexts(final RuntimeEdge
runtimeEdge,
@@ -163,8 +163,8 @@ public Serializer getSerializer(final String runtimeEdgeId)
{
/**
* (SYNCHRONIZATION) Called by network threads.
*
- * @param outputContext
- * @throws InvalidProtocolBufferException
+ * @param outputContext output context
+ * @throws InvalidProtocolBufferException protobuf exception
*/
public void onOutputContext(final ByteOutputContext outputContext) throws
InvalidProtocolBufferException {
final ControlMessage.PipeTransferContextDescriptor descriptor =
diff --git
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/DataFetcherOutputCollector.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/DataFetcherOutputCollector.java
index 3f1bc9074..56c754038 100644
---
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/DataFetcherOutputCollector.java
+++
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/DataFetcherOutputCollector.java
@@ -34,6 +34,7 @@
/**
* It forwards output to the next operator.
+ * @param nextOperatorVertex next operator to emit data and watermark
*/
public DataFetcherOutputCollector(final OperatorVertex nextOperatorVertex) {
this.nextOperatorVertex = nextOperatorVertex;
diff --git
a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/InputWatermarkManager.java
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/InputWatermarkManager.java
new file mode 100644
index 0..66fb7aa81
--- /dev/null
+++
b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/datatransfer/InputWatermarkManager.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.nemo.runtime.executor.datatransfer;
+
+import org.apache.nemo.common.punctuation.Watermark;
+
+
+/**
+ * An interface for tracking input watermarks among multiple input streams.
+ * --edge 1--
+ * --edge 2-- watermarkManager --(emitWatermark)-- nextOperator
+ * --edge 3--
+ */
+public
