[GitHub] twill pull request #58: [TWILL-240] EventHandler Improvement
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132029365
--- Diff:
twill-yarn/src/test/java/org/apache/twill/yarn/EventHandlerTest.java ---
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests {@link EventHandler} methods
+ */
+public final class EventHandlerTest extends BaseYarnTest {
--- End diff --
Rename the test class to `EventHandlerTestRun` and add it to
`YarnTestSuite`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] twill pull request #58: [TWILL-240] EventHandler Improvement
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132029465
--- Diff:
twill-yarn/src/test/java/org/apache/twill/yarn/EventHandlerTest.java ---
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests {@link EventHandler} methods
+ */
+public final class EventHandlerTest extends BaseYarnTest {
+ private static final Logger LOG =
LoggerFactory.getLogger(EventHandlerTest.class);
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+ public static final String STARTED_FILE = "started_file";
+ public static final String RUN_FILE = "run_file";
+ public static final String CONTAINER_LAUNCHED_FOLDER = "launched_folder";
+ public static final String CONTAINER_STOPPED_FOLDER = "stopped_folder";
+ public static final String COMPLETED_FILE = "completed_file";
+ public static final String KILLED_FILE = "killed_file";
+ public static final String ABORTED_FILE = "aborted_file";
+
+ @Test
+ public void testComplete() throws InterruptedException,
ExecutionException, TimeoutException, IOException {
+// Create a parent folder to be written by EventHandler
+File parentFolder = TMP_FOLDER.newFolder();
+parentFolder.setWritable(true, false);
+TwillController controller = getTwillRunner().prepare(new
CompleteApplication(parentFolder.getAbsolutePath()))
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out,
true)))
+ .start();
+
+// Wait for the app to complete within 120 seconds.
+try {
+ controller.awaitTerminated(120, TimeUnit.SECONDS);
+ Set handlerFiles = new
HashSet<>(Arrays.asList(parentFolder.list()));
+ Assert.assertEquals(5, handlerFiles.size());
+ // EventHandler#started() method should be called to create a file
+ Assert.assertTrue(handlerFiles.contains(STARTED_FILE));
+ // CompleteRunnable#run() method should be called to create a file
after EventHandler#started() method is called
+ Assert.assertTrue(handlerFiles.contains(RUN_FILE));
+ // EventHandler#containerLaunched(String, int, String) method should
be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_LAUNCHED_FOLDER));
+ // EventHandler#containerStopped(String, int, String, int) method
should be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_STOPPED_FOLDER));
+ // Assert that containerLaunched and containerStopped are called for
the same containers
+ // for the same number of times
+ String[] containerLaunchedFiles = new
File(parentFolder.getAbsolutePath(), CONTAINER_LAUNCHED_FOLDER).list();
+ String[] containerStoppedFiles = new
File(paren
[GitHub] twill pull request #58: [TWILL-240] EventHandler Improvement
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132029508
--- Diff:
twill-yarn/src/test/java/org/apache/twill/yarn/EventHandlerTest.java ---
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests {@link EventHandler} methods
+ */
+public final class EventHandlerTest extends BaseYarnTest {
+ private static final Logger LOG =
LoggerFactory.getLogger(EventHandlerTest.class);
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+ public static final String STARTED_FILE = "started_file";
+ public static final String RUN_FILE = "run_file";
+ public static final String CONTAINER_LAUNCHED_FOLDER = "launched_folder";
+ public static final String CONTAINER_STOPPED_FOLDER = "stopped_folder";
+ public static final String COMPLETED_FILE = "completed_file";
+ public static final String KILLED_FILE = "killed_file";
+ public static final String ABORTED_FILE = "aborted_file";
+
+ @Test
+ public void testComplete() throws InterruptedException,
ExecutionException, TimeoutException, IOException {
+// Create a parent folder to be written by EventHandler
+File parentFolder = TMP_FOLDER.newFolder();
+parentFolder.setWritable(true, false);
+TwillController controller = getTwillRunner().prepare(new
CompleteApplication(parentFolder.getAbsolutePath()))
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out,
true)))
+ .start();
+
+// Wait for the app to complete within 120 seconds.
+try {
+ controller.awaitTerminated(120, TimeUnit.SECONDS);
+ Set handlerFiles = new
HashSet<>(Arrays.asList(parentFolder.list()));
+ Assert.assertEquals(5, handlerFiles.size());
+ // EventHandler#started() method should be called to create a file
+ Assert.assertTrue(handlerFiles.contains(STARTED_FILE));
+ // CompleteRunnable#run() method should be called to create a file
after EventHandler#started() method is called
+ Assert.assertTrue(handlerFiles.contains(RUN_FILE));
+ // EventHandler#containerLaunched(String, int, String) method should
be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_LAUNCHED_FOLDER));
+ // EventHandler#containerStopped(String, int, String, int) method
should be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_STOPPED_FOLDER));
+ // Assert that containerLaunched and containerStopped are called for
the same containers
+ // for the same number of times
+ String[] containerLaunchedFiles = new
File(parentFolder.getAbsolutePath(), CONTAINER_LAUNCHED_FOLDER).list();
+ String[] containerStoppedFiles = new
File(paren
[jira] [Commented] (TWILL-240) Improve EventHandler to handle more application lifecycle events
[
https://issues.apache.org/jira/browse/TWILL-240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16119021#comment-16119021
]
ASF GitHub Bot commented on TWILL-240:
--
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132029365
--- Diff:
twill-yarn/src/test/java/org/apache/twill/yarn/EventHandlerTest.java ---
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests {@link EventHandler} methods
+ */
+public final class EventHandlerTest extends BaseYarnTest {
--- End diff --
Rename the test class to `EventHandlerTestRun` and add it to
`YarnTestSuite`.
> Improve EventHandler to handle more application lifecycle events
>
>
> Key: TWILL-240
> URL: https://issues.apache.org/jira/browse/TWILL-240
> Project: Apache Twill
> Issue Type: New Feature
>Reporter: Chengfeng Mao
>Assignee: Chengfeng Mao
> Fix For: 0.12.0
>
>
> Application Master should be able to run application specific code when
> certain lifecycle events happen by calling methods from EventHandler. For
> instance, when the app first starts, completes, aborts and etc.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (TWILL-240) Improve EventHandler to handle more application lifecycle events
[
https://issues.apache.org/jira/browse/TWILL-240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16119022#comment-16119022
]
ASF GitHub Bot commented on TWILL-240:
--
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132029465
--- Diff:
twill-yarn/src/test/java/org/apache/twill/yarn/EventHandlerTest.java ---
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests {@link EventHandler} methods
+ */
+public final class EventHandlerTest extends BaseYarnTest {
+ private static final Logger LOG =
LoggerFactory.getLogger(EventHandlerTest.class);
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+ public static final String STARTED_FILE = "started_file";
+ public static final String RUN_FILE = "run_file";
+ public static final String CONTAINER_LAUNCHED_FOLDER = "launched_folder";
+ public static final String CONTAINER_STOPPED_FOLDER = "stopped_folder";
+ public static final String COMPLETED_FILE = "completed_file";
+ public static final String KILLED_FILE = "killed_file";
+ public static final String ABORTED_FILE = "aborted_file";
+
+ @Test
+ public void testComplete() throws InterruptedException,
ExecutionException, TimeoutException, IOException {
+// Create a parent folder to be written by EventHandler
+File parentFolder = TMP_FOLDER.newFolder();
+parentFolder.setWritable(true, false);
+TwillController controller = getTwillRunner().prepare(new
CompleteApplication(parentFolder.getAbsolutePath()))
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out,
true)))
+ .start();
+
+// Wait for the app to complete within 120 seconds.
+try {
+ controller.awaitTerminated(120, TimeUnit.SECONDS);
+ Set handlerFiles = new
HashSet<>(Arrays.asList(parentFolder.list()));
+ Assert.assertEquals(5, handlerFiles.size());
+ // EventHandler#started() method should be called to create a file
+ Assert.assertTrue(handlerFiles.contains(STARTED_FILE));
+ // CompleteRunnable#run() method should be called to create a file
after EventHandler#started() method is called
+ Assert.assertTrue(handlerFiles.contains(RUN_FILE));
+ // EventHandler#containerLaunched(String, int, String) method should
be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_LAUNCHED_FOLDER));
+ // EventHandler#containerStopped(String, int, String, int) method
should be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_STOPPED_FOLDER));
+ // Assert that containerLaunched and containerStopped are cal
[jira] [Commented] (TWILL-240) Improve EventHandler to handle more application lifecycle events
[
https://issues.apache.org/jira/browse/TWILL-240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16119023#comment-16119023
]
ASF GitHub Bot commented on TWILL-240:
--
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132029508
--- Diff:
twill-yarn/src/test/java/org/apache/twill/yarn/EventHandlerTest.java ---
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests {@link EventHandler} methods
+ */
+public final class EventHandlerTest extends BaseYarnTest {
+ private static final Logger LOG =
LoggerFactory.getLogger(EventHandlerTest.class);
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+ public static final String STARTED_FILE = "started_file";
+ public static final String RUN_FILE = "run_file";
+ public static final String CONTAINER_LAUNCHED_FOLDER = "launched_folder";
+ public static final String CONTAINER_STOPPED_FOLDER = "stopped_folder";
+ public static final String COMPLETED_FILE = "completed_file";
+ public static final String KILLED_FILE = "killed_file";
+ public static final String ABORTED_FILE = "aborted_file";
+
+ @Test
+ public void testComplete() throws InterruptedException,
ExecutionException, TimeoutException, IOException {
+// Create a parent folder to be written by EventHandler
+File parentFolder = TMP_FOLDER.newFolder();
+parentFolder.setWritable(true, false);
+TwillController controller = getTwillRunner().prepare(new
CompleteApplication(parentFolder.getAbsolutePath()))
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out,
true)))
+ .start();
+
+// Wait for the app to complete within 120 seconds.
+try {
+ controller.awaitTerminated(120, TimeUnit.SECONDS);
+ Set handlerFiles = new
HashSet<>(Arrays.asList(parentFolder.list()));
+ Assert.assertEquals(5, handlerFiles.size());
+ // EventHandler#started() method should be called to create a file
+ Assert.assertTrue(handlerFiles.contains(STARTED_FILE));
+ // CompleteRunnable#run() method should be called to create a file
after EventHandler#started() method is called
+ Assert.assertTrue(handlerFiles.contains(RUN_FILE));
+ // EventHandler#containerLaunched(String, int, String) method should
be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_LAUNCHED_FOLDER));
+ // EventHandler#containerStopped(String, int, String, int) method
should be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_STOPPED_FOLDER));
+ // Assert that containerLaunched and containerStopped are cal
[GitHub] twill pull request #58: [TWILL-240] EventHandler Improvement
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132040093
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
---
@@ -267,6 +280,14 @@ void stopByIdAndWait(String runnableName, int
instanceId) {
resourceReport.removeRunnableResources(runnableName, containerId);
containerChange.signalAll();
+if (eventHandler != null) {
+ Integer exitStatus = containerExitStatus.get(containerId);
+ if (exitStatus == null) {
--- End diff --
So this is for handling the case when `handleCompleted` was not called
(e.g. the runnable container is not stopping after receiving the "stop"
command), right? If that's the case, isn't that the exit status is always not
there?
Also, do we need to do the similar logic to call event handler in the
`stopAll` method?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[jira] [Commented] (TWILL-240) Improve EventHandler to handle more application lifecycle events
[
https://issues.apache.org/jira/browse/TWILL-240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16119074#comment-16119074
]
ASF GitHub Bot commented on TWILL-240:
--
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132040093
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
---
@@ -267,6 +280,14 @@ void stopByIdAndWait(String runnableName, int
instanceId) {
resourceReport.removeRunnableResources(runnableName, containerId);
containerChange.signalAll();
+if (eventHandler != null) {
+ Integer exitStatus = containerExitStatus.get(containerId);
+ if (exitStatus == null) {
--- End diff --
So this is for handling the case when `handleCompleted` was not called
(e.g. the runnable container is not stopping after receiving the "stop"
command), right? If that's the case, isn't that the exit status is always not
there?
Also, do we need to do the similar logic to call event handler in the
`stopAll` method?
> Improve EventHandler to handle more application lifecycle events
>
>
> Key: TWILL-240
> URL: https://issues.apache.org/jira/browse/TWILL-240
> Project: Apache Twill
> Issue Type: New Feature
>Reporter: Chengfeng Mao
>Assignee: Chengfeng Mao
> Fix For: 0.12.0
>
>
> Application Master should be able to run application specific code when
> certain lifecycle events happen by calling methods from EventHandler. For
> instance, when the app first starts, completes, aborts and etc.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (TWILL-240) Improve EventHandler to handle more application lifecycle events
[
https://issues.apache.org/jira/browse/TWILL-240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16119075#comment-16119075
]
ASF GitHub Bot commented on TWILL-240:
--
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132038164
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
---
@@ -173,6 +183,9 @@ void start(String runnableName, ContainerInfo
containerInfo, TwillContainerLaunc
startSequence.addLast(runnableName);
}
containerChange.signalAll();
+ if (eventHandler != null) {
+eventHandler.containerLaunched(runnableName, instanceId,
containerInfo.getId());
--- End diff --
How do we handle exception raised from event handler methods?
> Improve EventHandler to handle more application lifecycle events
>
>
> Key: TWILL-240
> URL: https://issues.apache.org/jira/browse/TWILL-240
> Project: Apache Twill
> Issue Type: New Feature
>Reporter: Chengfeng Mao
>Assignee: Chengfeng Mao
> Fix For: 0.12.0
>
>
> Application Master should be able to run application specific code when
> certain lifecycle events happen by calling methods from EventHandler. For
> instance, when the app first starts, completes, aborts and etc.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] twill pull request #58: [TWILL-240] EventHandler Improvement
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132040768
--- Diff:
twill-yarn/src/test/java/org/apache/twill/yarn/EventHandlerTest.java ---
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests {@link EventHandler} methods
+ */
+public final class EventHandlerTest extends BaseYarnTest {
+ private static final Logger LOG =
LoggerFactory.getLogger(EventHandlerTest.class);
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+ public static final String STARTED_FILE = "started_file";
+ public static final String RUN_FILE = "run_file";
+ public static final String CONTAINER_LAUNCHED_FOLDER = "launched_folder";
+ public static final String CONTAINER_STOPPED_FOLDER = "stopped_folder";
+ public static final String COMPLETED_FILE = "completed_file";
+ public static final String KILLED_FILE = "killed_file";
+ public static final String ABORTED_FILE = "aborted_file";
+
+ @Test
+ public void testComplete() throws InterruptedException,
ExecutionException, TimeoutException, IOException {
+// Create a parent folder to be written by EventHandler
+File parentFolder = TMP_FOLDER.newFolder();
+parentFolder.setWritable(true, false);
+TwillController controller = getTwillRunner().prepare(new
CompleteApplication(parentFolder.getAbsolutePath()))
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out,
true)))
+ .start();
+
+// Wait for the app to complete within 120 seconds.
+try {
+ controller.awaitTerminated(120, TimeUnit.SECONDS);
+ Set handlerFiles = new
HashSet<>(Arrays.asList(parentFolder.list()));
+ Assert.assertEquals(5, handlerFiles.size());
+ // EventHandler#started() method should be called to create a file
+ Assert.assertTrue(handlerFiles.contains(STARTED_FILE));
+ // CompleteRunnable#run() method should be called to create a file
after EventHandler#started() method is called
+ Assert.assertTrue(handlerFiles.contains(RUN_FILE));
+ // EventHandler#containerLaunched(String, int, String) method should
be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_LAUNCHED_FOLDER));
+ // EventHandler#containerStopped(String, int, String, int) method
should be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_STOPPED_FOLDER));
+ // Assert that containerLaunched and containerStopped are called for
the same containers
+ // for the same number of times
+ String[] containerLaunchedFiles = new
File(parentFolder.getAbsolutePath(), CONTAINER_LAUNCHED_FOLDER).list();
+ String[] containerStoppedFiles = new
File(paren
[jira] [Commented] (TWILL-240) Improve EventHandler to handle more application lifecycle events
[
https://issues.apache.org/jira/browse/TWILL-240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16119072#comment-16119072
]
ASF GitHub Bot commented on TWILL-240:
--
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132037568
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
---
@@ -221,7 +229,8 @@ protected void doStart() throws Exception {
// initialize the event handler, if it fails, it will fail the
application.
if (eventHandler != null) {
- eventHandler.initialize(new
BasicEventHandlerContext(twillSpec.getEventHandler()));
+ eventHandler.initialize(new
BasicEventHandlerContext(twillRuntimeSpec, twillSpec.getEventHandler()));
--- End diff --
The `twillRuntimeSpec` already contains `twillSpec`. Seems unnecessary to
pass in `twillSpec.getEventHandler()` as a separate parameter.
> Improve EventHandler to handle more application lifecycle events
>
>
> Key: TWILL-240
> URL: https://issues.apache.org/jira/browse/TWILL-240
> Project: Apache Twill
> Issue Type: New Feature
>Reporter: Chengfeng Mao
>Assignee: Chengfeng Mao
> Fix For: 0.12.0
>
>
> Application Master should be able to run application specific code when
> certain lifecycle events happen by calling methods from EventHandler. For
> instance, when the app first starts, completes, aborts and etc.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[jira] [Commented] (TWILL-240) Improve EventHandler to handle more application lifecycle events
[
https://issues.apache.org/jira/browse/TWILL-240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16119073#comment-16119073
]
ASF GitHub Bot commented on TWILL-240:
--
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132038521
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
---
@@ -267,6 +280,14 @@ void stopByIdAndWait(String runnableName, int
instanceId) {
resourceReport.removeRunnableResources(runnableName, containerId);
containerChange.signalAll();
+if (eventHandler != null) {
+ Integer exitStatus = containerExitStatus.get(containerId);
--- End diff --
Should remove it from the exit status map, otherwise the map can keep
growing in size.
> Improve EventHandler to handle more application lifecycle events
>
>
> Key: TWILL-240
> URL: https://issues.apache.org/jira/browse/TWILL-240
> Project: Apache Twill
> Issue Type: New Feature
>Reporter: Chengfeng Mao
>Assignee: Chengfeng Mao
> Fix For: 0.12.0
>
>
> Application Master should be able to run application specific code when
> certain lifecycle events happen by calling methods from EventHandler. For
> instance, when the app first starts, completes, aborts and etc.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
[GitHub] twill pull request #58: [TWILL-240] EventHandler Improvement
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132037568
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
---
@@ -221,7 +229,8 @@ protected void doStart() throws Exception {
// initialize the event handler, if it fails, it will fail the
application.
if (eventHandler != null) {
- eventHandler.initialize(new
BasicEventHandlerContext(twillSpec.getEventHandler()));
+ eventHandler.initialize(new
BasicEventHandlerContext(twillRuntimeSpec, twillSpec.getEventHandler()));
--- End diff --
The `twillRuntimeSpec` already contains `twillSpec`. Seems unnecessary to
pass in `twillSpec.getEventHandler()` as a separate parameter.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] twill pull request #58: [TWILL-240] EventHandler Improvement
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132038521
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
---
@@ -267,6 +280,14 @@ void stopByIdAndWait(String runnableName, int
instanceId) {
resourceReport.removeRunnableResources(runnableName, containerId);
containerChange.signalAll();
+if (eventHandler != null) {
+ Integer exitStatus = containerExitStatus.get(containerId);
--- End diff --
Should remove it from the exit status map, otherwise the map can keep
growing in size.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] twill pull request #58: [TWILL-240] EventHandler Improvement
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132038164
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
---
@@ -173,6 +183,9 @@ void start(String runnableName, ContainerInfo
containerInfo, TwillContainerLaunc
startSequence.addLast(runnableName);
}
containerChange.signalAll();
+ if (eventHandler != null) {
+eventHandler.containerLaunched(runnableName, instanceId,
containerInfo.getId());
--- End diff --
How do we handle exception raised from event handler methods?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[jira] [Commented] (TWILL-240) Improve EventHandler to handle more application lifecycle events
[
https://issues.apache.org/jira/browse/TWILL-240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16119076#comment-16119076
]
ASF GitHub Bot commented on TWILL-240:
--
Github user chtyim commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132040768
--- Diff:
twill-yarn/src/test/java/org/apache/twill/yarn/EventHandlerTest.java ---
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests {@link EventHandler} methods
+ */
+public final class EventHandlerTest extends BaseYarnTest {
+ private static final Logger LOG =
LoggerFactory.getLogger(EventHandlerTest.class);
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+ public static final String STARTED_FILE = "started_file";
+ public static final String RUN_FILE = "run_file";
+ public static final String CONTAINER_LAUNCHED_FOLDER = "launched_folder";
+ public static final String CONTAINER_STOPPED_FOLDER = "stopped_folder";
+ public static final String COMPLETED_FILE = "completed_file";
+ public static final String KILLED_FILE = "killed_file";
+ public static final String ABORTED_FILE = "aborted_file";
+
+ @Test
+ public void testComplete() throws InterruptedException,
ExecutionException, TimeoutException, IOException {
+// Create a parent folder to be written by EventHandler
+File parentFolder = TMP_FOLDER.newFolder();
+parentFolder.setWritable(true, false);
+TwillController controller = getTwillRunner().prepare(new
CompleteApplication(parentFolder.getAbsolutePath()))
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out,
true)))
+ .start();
+
+// Wait for the app to complete within 120 seconds.
+try {
+ controller.awaitTerminated(120, TimeUnit.SECONDS);
+ Set handlerFiles = new
HashSet<>(Arrays.asList(parentFolder.list()));
+ Assert.assertEquals(5, handlerFiles.size());
+ // EventHandler#started() method should be called to create a file
+ Assert.assertTrue(handlerFiles.contains(STARTED_FILE));
+ // CompleteRunnable#run() method should be called to create a file
after EventHandler#started() method is called
+ Assert.assertTrue(handlerFiles.contains(RUN_FILE));
+ // EventHandler#containerLaunched(String, int, String) method should
be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_LAUNCHED_FOLDER));
+ // EventHandler#containerStopped(String, int, String, int) method
should be called to create a folder
+ Assert.assertTrue(handlerFiles.contains(CONTAINER_STOPPED_FOLDER));
+ // Assert that containerLaunched and containerStopped are cal
[GitHub] twill pull request #58: [TWILL-240] EventHandler Improvement
Github user maochf commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132048851
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
---
@@ -267,6 +280,14 @@ void stopByIdAndWait(String runnableName, int
instanceId) {
resourceReport.removeRunnableResources(runnableName, containerId);
containerChange.signalAll();
+if (eventHandler != null) {
+ Integer exitStatus = containerExitStatus.get(containerId);
+ if (exitStatus == null) {
--- End diff --
`controller.stopAndWait();` at line 269 blocks until `handleCompleted`
method runs
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[jira] [Commented] (TWILL-240) Improve EventHandler to handle more application lifecycle events
[
https://issues.apache.org/jira/browse/TWILL-240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16119110#comment-16119110
]
ASF GitHub Bot commented on TWILL-240:
--
Github user maochf commented on a diff in the pull request:
https://github.com/apache/twill/pull/58#discussion_r132048851
--- Diff:
twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
---
@@ -267,6 +280,14 @@ void stopByIdAndWait(String runnableName, int
instanceId) {
resourceReport.removeRunnableResources(runnableName, containerId);
containerChange.signalAll();
+if (eventHandler != null) {
+ Integer exitStatus = containerExitStatus.get(containerId);
+ if (exitStatus == null) {
--- End diff --
`controller.stopAndWait();` at line 269 blocks until `handleCompleted`
method runs
> Improve EventHandler to handle more application lifecycle events
>
>
> Key: TWILL-240
> URL: https://issues.apache.org/jira/browse/TWILL-240
> Project: Apache Twill
> Issue Type: New Feature
>Reporter: Chengfeng Mao
>Assignee: Chengfeng Mao
> Fix For: 0.12.0
>
>
> Application Master should be able to run application specific code when
> certain lifecycle events happen by calling methods from EventHandler. For
> instance, when the app first starts, completes, aborts and etc.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
