Repository: hive Updated Branches: refs/heads/master f6dbcac35 -> 7c1f01004
HIVE-19077: Handle duplicate ptests requests standing in queue at the same time (Adam Szita, via Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7c1f0100 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7c1f0100 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7c1f0100 Branch: refs/heads/master Commit: 7c1f010045db3af6fa5b805ae4f49fc9b07d5639 Parents: f6dbcac Author: Adam Szita <sz...@cloudera.com> Authored: Wed Apr 11 09:45:56 2018 +0200 Committer: Peter Vary <pv...@cloudera.com> Committed: Wed Apr 11 09:45:56 2018 +0200 ---------------------------------------------------------------------- dev-support/jenkins-common.sh | 2 + dev-support/jenkins-execute-build.sh | 3 +- .../hive/ptest/api/client/JenkinsQueueUtil.java | 133 +++++++++++++++++++ .../hive/ptest/api/client/PTestClient.java | 15 ++- 4 files changed, 150 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/dev-support/jenkins-common.sh ---------------------------------------------------------------------- diff --git a/dev-support/jenkins-common.sh b/dev-support/jenkins-common.sh index 0467d11..64f486f 100644 --- a/dev-support/jenkins-common.sh +++ b/dev-support/jenkins-common.sh @@ -15,6 +15,8 @@ # limitations under the License. JIRA_ROOT_URL="https://issues.apache.org" +JENKINS_URL="https://builds.apache.org" +JENKINS_QUEUE_QUERY="/queue/api/json?tree=items[task[name],inQueueSince,actions[parameters[name,value]],why]" fail() { echo "$@" 1>&2 http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/dev-support/jenkins-execute-build.sh ---------------------------------------------------------------------- diff --git a/dev-support/jenkins-execute-build.sh b/dev-support/jenkins-execute-build.sh index f660fcb..35392dd 100644 --- a/dev-support/jenkins-execute-build.sh +++ b/dev-support/jenkins-execute-build.sh @@ -51,7 +51,8 @@ call_ptest_server() { local PTEST_CLASSPATH="$PTEST_BUILD_DIR/hive/testutils/ptest2/target/hive-ptest-3.0-classes.jar:$PTEST_BUILD_DIR/hive/testutils/ptest2/target/lib/*" java -cp "$PTEST_CLASSPATH" org.apache.hive.ptest.api.client.PTestClient --command testStart \ - --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD" "$@" + --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD" \ + --jenkinsQueueUrl "$JENKINS_URL$JENKINS_QUEUE_QUERY" "$@" } # Unpack all test results http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java new file mode 100644 index 0000000..f973781 --- /dev/null +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hive.ptest.api.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.http.HttpResponse; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.util.EntityUtils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; + +/** + * Utility class for the Precommit test job queue on Jenkins + */ +public class JenkinsQueueUtil { + + private static final String JSON_ITEMS_FIELD = "items"; + private static final String JSON_TASK_FIELD = "task"; + private static final String JSON_TASK_NAME_FIELD = "name"; + private static final String JSON_PARAMETERS_FIELD = "parameters"; + private static final String JSON_PARAMETER_NAME_FIELD = "name"; + private static final String JSON_PARAMETER_VALUE_FIELD = "value"; + + private static final String JOB_NAME = "PreCommit-HIVE-Build"; + private static final String ISSUE_FIELD_KEY = "ISSUE_NUM"; + private static final String JIRA_KEY_PREFIX = "HIVE-"; + + + /** + * Looks up the current queue of the precommit job on a jenkins instance (specified by + * PTestClient.JENKINS_QUEUE_URL), and checks if current Jira is standing in queue already (i.e. + * will be executed in the future too) + * + * @param commandLine PTestClient's command line option values' list + * @return whether or not the Jira specified in the command line can be found in the job queue + */ + public static boolean isJiraAlreadyInQueue(CommandLine commandLine) { + if (!(commandLine.hasOption(PTestClient.JENKINS_QUEUE_URL) && + commandLine.hasOption(PTestClient.JIRA))) { + return false; + } + try { + System.out.println("Checking " + JOB_NAME + " queue..."); + String queueJson = httpGet(commandLine.getOptionValue(PTestClient.JENKINS_QUEUE_URL)); + List<String> jirasInQueue = parseJiras(queueJson); + if (jirasInQueue.size() > 0) { + System.out.println(JOB_NAME + " has the following jira(s) in queue: " + jirasInQueue); + } else { + return false; + } + + String jira = commandLine.getOptionValue(PTestClient.JIRA).replaceAll(JIRA_KEY_PREFIX,""); + if (jirasInQueue.contains(jira)) { + return true; + } + + } catch (IOException e) { + System.err.println("Error checking " + JOB_NAME + " build queue: " + e); + } + return false; + } + + /** + * Parses raw json to produce a list of Jira number strings. + * @param queueJson + * @return + * @throws IOException + */ + private static List<String> parseJiras(String queueJson) throws IOException { + List<String> jirasInQueue = new ArrayList<>(); + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode rootNode = objectMapper.readTree(queueJson); + List<JsonNode> items = Lists.newArrayList(rootNode.findValue(JSON_ITEMS_FIELD).iterator()); + for (JsonNode item : items) { + String taskName = item.path(JSON_TASK_FIELD).path(JSON_TASK_NAME_FIELD).asText(); + if (JOB_NAME.equals(taskName)) { + List<JsonNode> parameters = Lists.newArrayList(item.findValue(JSON_PARAMETERS_FIELD)); + for (JsonNode parameter : parameters) { + if (ISSUE_FIELD_KEY.equals(parameter.path(JSON_PARAMETER_NAME_FIELD).asText())) { + jirasInQueue.add(parameter.path(JSON_PARAMETER_VALUE_FIELD).asText()); + } + } + } + } + return jirasInQueue; + } + + private static String httpGet(String url) + throws IOException { + DefaultHttpClient httpClient = new DefaultHttpClient(); + HttpGet request = new HttpGet(url); + try { + request.addHeader("content-type", "application/json"); + httpClient.setHttpRequestRetryHandler(new PTestClient.PTestHttpRequestRetryHandler()); + HttpResponse httpResponse = httpClient.execute(request); + StatusLine statusLine = httpResponse.getStatusLine(); + if (statusLine.getStatusCode() != 200) { + throw new IllegalStateException(statusLine.getStatusCode() + " " + statusLine.getReasonPhrase()); + } + String response = EntityUtils.toString(httpResponse.getEntity(), "UTF-8"); + return response; + } finally { + request.abort(); + } + } + + +} http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java index b72a971..9970c36 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java @@ -81,10 +81,11 @@ public class PTestClient { private static final String PASSWORD = "password"; private static final String PROFILE = "profile"; private static final String PATCH = "patch"; - private static final String JIRA = "jira"; + public static final String JIRA = "jira"; private static final String OUTPUT_DIR = "outputDir"; private static final String TEST_HANDLE = "testHandle"; private static final String CLEAR_LIBRARY_CACHE = "clearLibraryCache"; + public static final String JENKINS_QUEUE_URL = "jenkinsQueueUrl"; private static final int MAX_RETRIES = 10; private final String mApiEndPoint; private final String mLogsEndpoint; @@ -242,7 +243,7 @@ public class PTestClient { request.abort(); } } - private static class PTestHttpRequestRetryHandler implements HttpRequestRetryHandler { + public static class PTestHttpRequestRetryHandler implements HttpRequestRetryHandler { @Override public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { @@ -298,6 +299,7 @@ public class PTestClient { options.addOption(null, OUTPUT_DIR, true, "Directory to download and save test-results.tar.gz to. (Optional for testStart)"); options.addOption(null, CLEAR_LIBRARY_CACHE, false, "Before starting the test, delete the ivy and maven directories (Optional for testStart)"); options.addOption(null, LOGS_ENDPOINT, true, "URL to get the logs"); + options.addOption(null, JENKINS_QUEUE_URL, true, "URL for quering Jenkins job queue"); CommandLine commandLine = parser.parse(options, args); @@ -319,6 +321,14 @@ public class PTestClient { PROFILE, TEST_HANDLE }); + + boolean jiraAlreadyInQueue = JenkinsQueueUtil.isJiraAlreadyInQueue(commandLine); + if (jiraAlreadyInQueue) { + System.out.println("Skipping ptest execution, as " + commandLine.getOptionValue(JIRA) + + " is scheduled in " + "queue in " + "the future too."); + System.exit(0); + } + result = client.testStart(commandLine.getOptionValue(PROFILE), commandLine.getOptionValue(TEST_HANDLE), commandLine.getOptionValue(JIRA), commandLine.getOptionValue(PATCH), commandLine.hasOption(CLEAR_LIBRARY_CACHE)); @@ -335,4 +345,5 @@ public class PTestClient { System.exit(1); } } + }