This is an automated email from the ASF dual-hosted git repository.
ayegorov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 80a1100 [CI] Dump stacktrace when a job is cancelled
80a1100 is described below
commit 80a110096368b54831eb0302bcdfb5b9555f86bc
Author: Nicolò Boschi <[email protected]>
AuthorDate: Tue Feb 15 02:43:28 2022 +0100
[CI] Dump stacktrace when a job is cancelled
### Motivation
Sometimes CI jobs fail due to timeout. It would be useful understand what
the latest test was doing before being interrupted.
### Changes
* Added a new script for dumping stacktrace.
* Added in all the jobs the step in case of `cancelled()` is true.
Reviewers: Andrey Yegorov <None>
This closes #3042 from nicoloboschi/ci-thread-dump
---
.github/workflows/bookie-tests.yml | 6 ++-
.github/workflows/client-tests.yml | 4 ++
.github/workflows/compatibility-check-java11.yml | 3 ++
.github/workflows/compatibility-check-java8.yml | 4 ++
.github/workflows/integration-tests.yml | 4 ++
.github/workflows/remaining-tests.yml | 6 ++-
.github/workflows/replication-tests.yml | 4 ++
.github/workflows/stream-tests.yml | 4 ++
.github/workflows/tls-tests.yml | 4 ++
build.gradle | 1 +
dev/ci-tool | 56 ++++++++++++++++++++++++
11 files changed, 94 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/bookie-tests.yml
b/.github/workflows/bookie-tests.yml
index 8eec299..8c0da44 100644
--- a/.github/workflows/bookie-tests.yml
+++ b/.github/workflows/bookie-tests.yml
@@ -50,4 +50,8 @@ jobs:
java-version: 1.8
- name: Run bookie test
- run: ./gradlew bookkeeper-server:test
--tests="org.apache.bookkeeper.bookie.*" ${GRADLE_ARGS}
\ No newline at end of file
+ run: ./gradlew bookkeeper-server:test
--tests="org.apache.bookkeeper.bookie.*" ${GRADLE_ARGS}
+
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
\ No newline at end of file
diff --git a/.github/workflows/client-tests.yml
b/.github/workflows/client-tests.yml
index 398278f..36bd403 100644
--- a/.github/workflows/client-tests.yml
+++ b/.github/workflows/client-tests.yml
@@ -50,3 +50,7 @@ jobs:
java-version: 1.8
- name: Run client tests
run: ./gradlew bookkeeper-server:test
--tests="org.apache.bookkeeper.client.*" ${GRADLE_ARGS}
+
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
diff --git a/.github/workflows/compatibility-check-java11.yml
b/.github/workflows/compatibility-check-java11.yml
index 7d93c50..9f2ef49 100644
--- a/.github/workflows/compatibility-check-java11.yml
+++ b/.github/workflows/compatibility-check-java11.yml
@@ -51,3 +51,6 @@ jobs:
- name: Build with gradle
run: |
./gradlew test -x bookkeeper-server:test -x
tests:integration:cluster:test -x tests:integration:smoke:test -x
tests:integration:standalone:test -PexcludeTests="**/distributedlog/**,
**/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**,
**/*backward*/**" ${GRADLE_ARGS}
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
diff --git a/.github/workflows/compatibility-check-java8.yml
b/.github/workflows/compatibility-check-java8.yml
index 629fed6..e5e9b94 100644
--- a/.github/workflows/compatibility-check-java8.yml
+++ b/.github/workflows/compatibility-check-java8.yml
@@ -51,3 +51,7 @@ jobs:
- name: Build with gradle
run: |
./gradlew test -x bookkeeper-server:test -x
tests:integration:cluster:test -x tests:integration:smoke:test -x
tests:integration:standalone:test -PexcludeTests="**/distributedlog/**,
**/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**,
**/*backward*/**" ${GRADLE_ARGS}
+
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
diff --git a/.github/workflows/integration-tests.yml
b/.github/workflows/integration-tests.yml
index 59c5128..0a4cf3a 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -57,3 +57,7 @@ jobs:
run: ./gradlew tests:integration:smoke:test ${GRADLE_ARGS}
- name: run standalone test
run: ./gradlew tests:integration:standalone:test ${GRADLE_ARGS}
+
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
diff --git a/.github/workflows/remaining-tests.yml
b/.github/workflows/remaining-tests.yml
index a3f53f4..2a8895d 100644
--- a/.github/workflows/remaining-tests.yml
+++ b/.github/workflows/remaining-tests.yml
@@ -50,4 +50,8 @@ jobs:
with:
java-version: 1.8
- name: Run remaining tests
- run: ./gradlew bookkeeper-server:test
-PexcludeTests="*org.apache.bookkeeper.bookie.*,
*org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*,
*org.apache.bookkeeper.tls.*" ${GRADLE_ARGS}
\ No newline at end of file
+ run: ./gradlew bookkeeper-server:test
-PexcludeTests="*org.apache.bookkeeper.bookie.*,
*org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*,
*org.apache.bookkeeper.tls.*" ${GRADLE_ARGS}
+
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
\ No newline at end of file
diff --git a/.github/workflows/replication-tests.yml
b/.github/workflows/replication-tests.yml
index 9664d8b..06fe4c3 100644
--- a/.github/workflows/replication-tests.yml
+++ b/.github/workflows/replication-tests.yml
@@ -50,3 +50,7 @@ jobs:
java-version: 1.8
- name: Run replication tests
run: ./gradlew bookkeeper-server:test
--tests="org.apache.bookkeeper.replication.*" ${GRADLE_ARGS}
+
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
diff --git a/.github/workflows/stream-tests.yml
b/.github/workflows/stream-tests.yml
index 612b47b..bbfda24 100644
--- a/.github/workflows/stream-tests.yml
+++ b/.github/workflows/stream-tests.yml
@@ -64,3 +64,7 @@ jobs:
- name: Run stream:storage:impl tests
run: ./gradlew stream:storage:impl:test ${GRADLE_ARGS}
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
+
diff --git a/.github/workflows/tls-tests.yml b/.github/workflows/tls-tests.yml
index 48f7f74..4281947 100644
--- a/.github/workflows/tls-tests.yml
+++ b/.github/workflows/tls-tests.yml
@@ -50,3 +50,7 @@ jobs:
java-version: 1.8
- name: Run tls tests
run: ./gradlew bookkeeper-server:test
--tests="org.apache.bookkeeper.tls.*" ${GRADLE_ARGS}
+
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: ./dev/ci-tool print_thread_dumps
diff --git a/build.gradle b/build.gradle
index ad5a9e1..fc7b665 100644
--- a/build.gradle
+++ b/build.gradle
@@ -315,6 +315,7 @@ allprojects {
}
dependencies {
implementation(enforcedPlatform(depLibs.nettyBom))
+ testImplementation depLibs.log4jSlf4jImpl
}
tasks.register('writeClasspath') {
doLast {
diff --git a/dev/ci-tool b/dev/ci-tool
new file mode 100755
index 0000000..604f4fb
--- /dev/null
+++ b/dev/ci-tool
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# shell function library for Bookkeeper CI builds
+
+# lists all available functions in this tool
+function ci_list_functions() {
+ declare -F | awk '{print $NF}' | sort | egrep '^ci_' | sed 's/^ci_//'
+}
+
+# prints thread dumps for all running JVMs
+# used in CI when a job gets cancelled because of a job timeout
+function ci_print_thread_dumps() {
+ for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do
+ echo "----------------------- pid $java_pid -----------------------"
+ cat /proc/$java_pid/cmdline | xargs -0 echo
+ jcmd $java_pid Thread.print -l
+ jcmd $java_pid GC.heap_info
+ done
+ return 0
+}
+
+if [ -z "$1" ]; then
+ echo "usage: $0 [ci_tool_function_name]"
+ echo "Available ci tool functions:"
+ ci_list_functions
+ exit 1
+fi
+ci_function_name="ci_$1"
+shift
+
+if [[ "$(LC_ALL=C type -t $ci_function_name)" == "function" ]]; then
+ eval "$ci_function_name" "$@"
+else
+ echo "Invalid ci tool function"
+ echo "Available ci tool functions:"
+ ci_list_functions
+ exit 1
+fi
\ No newline at end of file
