Till Westmann has submitted this change and it was merged. Change subject: [NO ISSUE][HYR][*DB][HTTP][TEST] Minor refactoring / test functionality ......................................................................
[NO ISSUE][HYR][*DB][HTTP][TEST] Minor refactoring / test functionality Adds ability to indicate which status code(s) to accept for an HTTP request, instead of just 200. Change-Id: I1f134d7874cc7de712e60911e89dac446c5c2cfd Reviewed-on: https://asterix-gerrit.ics.uci.edu/1948 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Till Westmann <ti...@apache.org> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java 3 files changed, 76 insertions(+), 49 deletions(-) Approvals: Till Westmann: Looks good to me, approved Jenkins: Verified; No violations found; ; Verified Objections: Anon. E. Moose #1000171: diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java index 82e8f7a..b24a9a1 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java @@ -26,7 +26,6 @@ import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.regex.Pattern; import org.apache.asterix.runtime.utils.CcApplicationContext; import org.apache.asterix.runtime.utils.ClusterStateManager; @@ -50,7 +49,6 @@ public class ClusterApiServlet extends AbstractServlet { private static final Logger LOGGER = Logger.getLogger(ClusterApiServlet.class.getName()); - private static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./"); protected static final String NODE_ID_KEY = "node_id"; protected static final String CONFIG_URI_KEY = "configUri"; protected static final String STATS_URI_KEY = "statsUri"; @@ -104,15 +102,8 @@ Section.COMMON, getConfigSelector())); ArrayNode ncs = (ArrayNode) json.get("ncs"); - final StringBuilder requestURL = new StringBuilder("http://"); - requestURL.append(request.getHeader(HttpHeaderNames.HOST)); - requestURL.append(request.getHttpRequest().uri()); - if (requestURL.charAt(requestURL.length() - 1) != '/') { - requestURL.append('/'); - } - requestURL.append(pathToNode); - String clusterURL = canonicalize(requestURL); - String adminURL = canonicalize(clusterURL + "../"); + String clusterURL = resolveClusterUrl(request, pathToNode); + String adminURL = HttpUtil.canonicalize(clusterURL + "../"); String nodeURL = clusterURL + "node/"; for (int i = 0; i < ncs.size(); i++) { ObjectNode nc = (ObjectNode) ncs.get(i); @@ -137,18 +128,20 @@ return json; } + protected String resolveClusterUrl(IServletRequest request, String pathToNode) { + final StringBuilder requestURL = new StringBuilder("http://"); + requestURL.append(request.getHeader(HttpHeaderNames.HOST)); + requestURL.append(request.getHttpRequest().uri()); + if (requestURL.charAt(requestURL.length() - 1) != '/') { + requestURL.append('/'); + } + requestURL.append(pathToNode); + return HttpUtil.canonicalize(requestURL); + } + protected Predicate<IOption> getConfigSelector() { return option -> !option.hidden() && option != ControllerConfig.Option.CONFIG_FILE && option != ControllerConfig.Option.CONFIG_FILE_URL; } - private String canonicalize(CharSequence requestURL) { - String clusterURL = ""; - String newClusterURL = requestURL.toString(); - while (!clusterURL.equals(newClusterURL)) { - clusterURL = newClusterURL; - newClusterURL = PARENT_DIR.matcher(clusterURL).replaceAll("/"); - } - return clusterURL; - } } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index a96adc0..e07ec72 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -103,17 +103,18 @@ // see // https://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers/417184 private static final long MAX_URL_LENGTH = 2000l; - private static final Pattern JAVA_BLOCK_COMMENT_PATTERN = - Pattern.compile("/\\*.*\\*/", Pattern.MULTILINE | Pattern.DOTALL); + private static final Pattern JAVA_BLOCK_COMMENT_PATTERN = Pattern.compile("/\\*.*\\*/", + Pattern.MULTILINE | Pattern.DOTALL); private static final Pattern JAVA_LINE_COMMENT_PATTERN = Pattern.compile("^//.*$", Pattern.MULTILINE); private static final Pattern SHELL_LINE_COMMENT_PATTERN = Pattern.compile("^#.*$", Pattern.MULTILINE); private static final Pattern REGEX_LINES_PATTERN = Pattern.compile("^(-)?/(.*)/([im]*)$"); - private static final Pattern POLL_TIMEOUT_PATTERN = - Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)", Pattern.MULTILINE); + private static final Pattern POLL_TIMEOUT_PATTERN = Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)", + Pattern.MULTILINE); private static final Pattern POLL_DELAY_PATTERN = Pattern.compile("polldelaysecs=(\\d+)(\\D|$)", Pattern.MULTILINE); private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)"); private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)"); private static final Pattern HTTP_PARAM_PATTERN = Pattern.compile("param (\\w+)=(.*)", Pattern.MULTILINE); + private static final Pattern HTTP_STATUSCODE_PATTERN = Pattern.compile("statuscode (.*)", Pattern.MULTILINE); public static final int TRUNCATE_THRESHOLD = 16384; @@ -168,10 +169,10 @@ public void runScriptAndCompareWithResult(File scriptFile, PrintWriter print, File expectedFile, File actualFile, ComparisonEnum compare) throws Exception { System.err.println("Expected results file: " + expectedFile.toString()); - BufferedReader readerExpected = - new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8")); - BufferedReader readerActual = - new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), "UTF-8")); + BufferedReader readerExpected = new BufferedReader( + new InputStreamReader(new FileInputStream(expectedFile), "UTF-8")); + BufferedReader readerActual = new BufferedReader( + new InputStreamReader(new FileInputStream(actualFile), "UTF-8")); boolean regex = false; try { if (ComparisonEnum.BINARY.equals(compare)) { @@ -354,10 +355,10 @@ public void runScriptAndCompareWithResultRegex(File scriptFile, File expectedFile, File actualFile) throws Exception { String lineExpected, lineActual; - try (BufferedReader readerExpected = - new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8")); - BufferedReader readerActual = - new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), "UTF-8"))) { + try (BufferedReader readerExpected = new BufferedReader( + new InputStreamReader(new FileInputStream(expectedFile), "UTF-8")); + BufferedReader readerActual = new BufferedReader( + new InputStreamReader(new FileInputStream(actualFile), "UTF-8"))) { StringBuilder actual = new StringBuilder(); while ((lineActual = readerActual.readLine()) != null) { actual.append(lineActual).append('\n'); @@ -697,8 +698,8 @@ // Insert and Delete statements are executed here public void executeUpdate(String str, URI uri) throws Exception { // Create a method instance. - HttpUriRequest request = - RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8)).build(); + HttpUriRequest request = RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8)) + .build(); // Execute the method. executeAndCheckHttpRequest(request); @@ -708,10 +709,10 @@ public InputStream executeAnyAQLAsync(String statement, boolean defer, OutputFormat fmt, URI uri, Map<String, Object> variableCtx) throws Exception { // Create a method instance. - HttpUriRequest request = - RequestBuilder.post(uri).addParameter("mode", defer ? "asynchronous-deferred" : "asynchronous") - .setEntity(new StringEntity(statement, StandardCharsets.UTF_8)) - .setHeader("Accept", fmt.mimeType()).build(); + HttpUriRequest request = RequestBuilder.post(uri) + .addParameter("mode", defer ? "asynchronous-deferred" : "asynchronous") + .setEntity(new StringEntity(statement, StandardCharsets.UTF_8)).setHeader("Accept", fmt.mimeType()) + .build(); String handleVar = getHandleVariable(statement); @@ -737,8 +738,8 @@ // create function statement public void executeDDL(String str, URI uri) throws Exception { // Create a method instance. - HttpUriRequest request = - RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8)).build(); + HttpUriRequest request = RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8)) + .build(); // Execute the method. executeAndCheckHttpRequest(request); @@ -748,8 +749,8 @@ // and returns the contents as a string // This string is later passed to REST API for execution. public String readTestFile(File testFile) throws Exception { - BufferedReader reader = - new BufferedReader(new InputStreamReader(new FileInputStream(testFile), StandardCharsets.UTF_8)); + BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream(testFile), StandardCharsets.UTF_8)); String line; StringBuilder stringBuilder = new StringBuilder(); String ls = System.getProperty("line.separator"); @@ -804,8 +805,8 @@ private static String getProcessOutput(Process p) throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Future<Integer> future = - Executors.newSingleThreadExecutor().submit(() -> IOUtils.copy(p.getInputStream(), new OutputStream() { + Future<Integer> future = Executors.newSingleThreadExecutor() + .submit(() -> IOUtils.copy(p.getInputStream(), new OutputStream() { @Override public void write(int b) throws IOException { baos.write(b); @@ -1091,11 +1092,12 @@ final String trimmedPathAndQuery = stripLineComments(stripJavaComments(statement)).trim(); final String variablesReplaced = replaceVarRef(trimmedPathAndQuery, variableCtx); final List<Parameter> params = extractParameters(statement); + final Predicate<Integer> statusCodePredicate = extractStatusCodePredicate(statement); InputStream resultStream; if ("http".equals(extension)) { - resultStream = executeHttp(reqType, variablesReplaced, fmt, params); + resultStream = executeHttp(reqType, variablesReplaced, fmt, params, statusCodePredicate); } else if ("uri".equals(extension)) { - resultStream = executeURI(reqType, URI.create(variablesReplaced), fmt, params); + resultStream = executeURI(reqType, URI.create(variablesReplaced), fmt, params, statusCodePredicate); } else { throw new IllegalArgumentException("Unexpected format for method " + reqType + ": " + extension); } @@ -1287,17 +1289,35 @@ return params; } - protected InputStream executeHttp(String ctxType, String endpoint, OutputFormat fmt, List<Parameter> params) - throws Exception { + protected static Predicate<Integer> extractStatusCodePredicate(String statement) { + List<Integer> codes = new ArrayList<>(); + final Matcher m = HTTP_STATUSCODE_PATTERN.matcher(statement); + while (m.find()) { + codes.add(Integer.parseInt(m.group(1))); + } + if (codes.isEmpty()) { + return code -> code == HttpStatus.SC_OK; + } else { + return codes::contains; + } + } + + protected InputStream executeHttp(String ctxType, String endpoint, OutputFormat fmt, List<Parameter> params, + Predicate<Integer> statusCodePredicate) throws Exception { String[] split = endpoint.split("\\?"); URI uri = createEndpointURI(split[0], split.length > 1 ? split[1] : null); - return executeURI(ctxType, uri, fmt, params); + return executeURI(ctxType, uri, fmt, params, statusCodePredicate); } private InputStream executeURI(String ctxType, URI uri, OutputFormat fmt, List<Parameter> params) throws Exception { return executeJSON(fmt, ctxType.toUpperCase(), uri, params); } + private InputStream executeURI(String ctxType, URI uri, OutputFormat fmt, List<Parameter> params, + Predicate<Integer> responseCodeValidator) throws Exception { + return executeJSON(fmt, ctxType.toUpperCase(), uri, params, responseCodeValidator); + } + private void killNC(String nodeId, CompilationUnit cUnit) throws Exception { //get node process id OutputFormat fmt = OutputFormat.forCompilationUnit(cUnit); diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java index 2babc73..fa3cc57 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java @@ -22,6 +22,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; @@ -33,6 +34,8 @@ import io.netty.handler.codec.http.HttpRequest; public class HttpUtil { + + private static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./"); private HttpUtil() { } @@ -131,4 +134,15 @@ return null; } } + + public static String canonicalize(CharSequence requestURL) { + String clusterURL = ""; + String newClusterURL = requestURL.toString(); + while (!clusterURL.equals(newClusterURL)) { + clusterURL = newClusterURL; + newClusterURL = PARENT_DIR.matcher(clusterURL).replaceAll("/"); + } + return clusterURL; + } + } -- To view, visit https://asterix-gerrit.ics.uci.edu/1948 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1f134d7874cc7de712e60911e89dac446c5c2cfd Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Michael Blow <mb...@apache.org> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Till Westmann <ti...@apache.org>