Chris Hillery has submitted this change and it was merged. Change subject: Add QueryServiceServlet ......................................................................
Add QueryServiceServlet Adds a new improved HTTP endpoint for queries. Also introduces initial stats gathering. Change-Id: Ia494c54f7252445ce38903c0b58fc4e23c324e6e Reviewed-on: https://asterix-gerrit.ics.uci.edu/597 Tested-by: Jenkins <[email protected]> Reviewed-by: Chris Hillery <[email protected]> --- M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java M asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java A asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java M asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java A asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java M asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java M asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java M asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java M asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java 9 files changed, 572 insertions(+), 36 deletions(-) Approvals: Chris Hillery: Looks good to me, approved Jenkins: Verified diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java index d815f30..e33aed2 100644 --- a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java +++ b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java @@ -190,6 +190,7 @@ } } catch (Exception e) { e.printStackTrace(); + System.exit(1); } } diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java b/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java index f0a0cc2..2ed67b2 100644 --- a/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java +++ b/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java @@ -25,15 +25,15 @@ /** * SessionConfig captures several different parameters for controlling * the execution of an APIFramework call. - * <li> It specifies how the execution will proceed (for instance, + * <li>It specifies how the execution will proceed (for instance, * whether to optimize, or whether to execute at all). - * <li> It allows you specify where the primary execution output will + * <li>It allows you specify where the primary execution output will * be sent. - * <li> It also allows you to request additional output for optional + * <li>It also allows you to request additional output for optional * out-of-band data about the execution (query plan, etc). - * <li> It allows you to specify the output format for the primary + * <li>It allows you to specify the output format for the primary * execution output - LOSSLESS_JSON, CSV, etc. - * <li> It allows you to specify output format-specific parameters. + * <li>It allows you to specify output format-specific parameters. */ public class SessionConfig { @@ -92,6 +92,10 @@ */ public static final String FORMAT_WRAPPER_ARRAY = "format-wrapper-array"; + public interface ResultDecorator { + PrintWriter print(PrintWriter pw); + } + // Standard execution flags. private final boolean executeQuery; private final boolean generateJobSpec; @@ -103,43 +107,66 @@ // Output format. private final OutputFormat fmt; + private final ResultDecorator preResultDecorator; + private final ResultDecorator postResultDecorator; + // Flags. - private final Map<String,Boolean> flags; + private final Map<String, Boolean> flags; /** * Create a SessionConfig object with all default values: - * * - All format flags set to "false". * - All out-of-band outputs set to "null". * - "Optimize" set to "true". * - "Execute Query" set to "true". * - "Generate Job Spec" set to "true". - * @param out PrintWriter for execution output. - * @param fmt Output format for execution output. + * + * @param out + * PrintWriter for execution output. + * @param fmt + * Output format for execution output. */ public SessionConfig(PrintWriter out, OutputFormat fmt) { - this(out, fmt, true, true, true); + this(out, fmt, null, null, true, true, true); + } + + public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator, + ResultDecorator postResultDecorator) { + this(out, fmt, preResultDecorator, postResultDecorator, true, true, true); + } + + public SessionConfig(PrintWriter out, OutputFormat fmt, boolean optimize, boolean executeQuery, + boolean generateJobSpec) { + this(out, fmt, null, null, optimize, executeQuery, generateJobSpec); } /** * Create a SessionConfig object with all optional values set to defaults: - * * - All format flags set to "false". * - All out-of-band outputs set to "false". - * @param out PrintWriter for execution output. - * @param fmt Output format for execution output. - * @param optimize Whether to optimize the execution. - * @param executeQuery Whether to execute the query or not. - * @param generateJobSpec Whether to generate the Hyracks job specification (if - * false, job cannot be executed). + * + * @param out + * PrintWriter for execution output. + * @param fmt + * Output format for execution output. + * @param optimize + * Whether to optimize the execution. + * @param executeQuery + * Whether to execute the query or not. + * @param generateJobSpec + * Whether to generate the Hyracks job specification (if + * false, job cannot be executed). */ - public SessionConfig(PrintWriter out, OutputFormat fmt, boolean optimize, boolean executeQuery, boolean generateJobSpec) { + public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator, + ResultDecorator postResultDecorator, boolean optimize, boolean executeQuery, boolean generateJobSpec) { this.out = out; this.fmt = fmt; + this.preResultDecorator = preResultDecorator; + this.postResultDecorator = postResultDecorator; this.optimize = optimize; this.executeQuery = executeQuery; this.generateJobSpec = generateJobSpec; - this.flags = new HashMap<String,Boolean>(); + this.flags = new HashMap<String, Boolean>(); } /** @@ -155,6 +182,14 @@ public OutputFormat fmt() { return this.fmt; } + + public PrintWriter resultPrefix(PrintWriter pw) { + return this.preResultDecorator != null ? this.preResultDecorator.print(pw) : pw; + }; + + public PrintWriter resultPostfix(PrintWriter pw) { + return this.postResultDecorator != null ? this.postResultDecorator.print(pw) : pw; + }; /** * Retrieve the value of the "execute query" flag. @@ -180,9 +215,8 @@ /** * Specify all out-of-band settings at once. For convenience of older code. */ - public void setOOBData(boolean expr_tree, boolean rewritten_expr_tree, - boolean logical_plan, boolean optimized_logical_plan, - boolean hyracks_job) { + public void setOOBData(boolean expr_tree, boolean rewritten_expr_tree, boolean logical_plan, + boolean optimized_logical_plan, boolean hyracks_job) { this.set(OOB_EXPR_TREE, expr_tree); this.set(OOB_REWRITTEN_EXPR_TREE, rewritten_expr_tree); this.set(OOB_LOGICAL_PLAN, logical_plan); @@ -192,8 +226,11 @@ /** * Specify a flag. - * @param flag One of the OOB_ or FORMAT_ constants from this class. - * @param value Value for the flag (all flags default to "false"). + * + * @param flag + * One of the OOB_ or FORMAT_ constants from this class. + * @param value + * Value for the flag (all flags default to "false"). */ public void set(String flag, boolean value) { flags.put(flag, Boolean.valueOf(value)); @@ -201,7 +238,9 @@ /** * Retrieve the setting of a format-specific flag. - * @param flag One of the FORMAT_ constants from this class. + * + * @param flag + * One of the FORMAT_ constants from this class. * @returns true or false (all flags default to "false"). */ public boolean is(String flag) { diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java new file mode 100644 index 0000000..56f349c --- /dev/null +++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.api.http.servlet; + +import java.util.Iterator; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +public class JSONUtil { + + static final String INDENT = " "; + + public static String indent(String str) { + try { + return append(new StringBuilder(), new JSONObject(str), 0).toString(); + } catch (JSONException e) { + throw new IllegalArgumentException(e); + } + } + + static StringBuilder append(StringBuilder sb, Object o, int indent) throws JSONException { + if (o instanceof JSONObject) { + return append(sb, (JSONObject) o, indent); + } else if (o instanceof JSONArray) { + return append(sb, (JSONArray) o, indent); + } else if (o instanceof String) { + return quote(sb, (String) o); + } else if (o instanceof Number || o instanceof Boolean) { + return sb.append(String.valueOf(o)); + } + throw new UnsupportedOperationException(o.getClass().getSimpleName()); + } + + static StringBuilder append(StringBuilder sb, JSONObject jobj, int indent) throws JSONException { + sb = sb.append("{\n"); + boolean first = true; + for (Iterator it = jobj.keys(); it.hasNext();) { + final String key = (String) it.next(); + if (first) { + first = false; + } else { + sb = sb.append(",\n"); + } + sb = indent(sb, indent + 1); + sb = quote(sb, key); + sb = sb.append(": "); + sb = append(sb, jobj.get(key), indent + 1); + } + sb = sb.append("\n"); + return indent(sb, indent).append("}"); + } + + static StringBuilder append(StringBuilder sb, JSONArray jarr, int indent) throws JSONException { + sb = sb.append("[\n"); + for (int i = 0; i < jarr.length(); ++i) { + if (i > 0) { + sb = sb.append(",\n"); + } + sb = indent(sb, indent + 1); + sb = append(sb, jarr.get(i), indent + 1); + } + sb = sb.append("\n"); + return indent(sb, indent).append("]"); + } + + static StringBuilder quote(StringBuilder sb, String str) { + return sb.append('"').append(str).append('"'); + } + + static StringBuilder indent(StringBuilder sb, int indent) { + while (indent > 0) { + sb.append(INDENT); + --indent; + } + return sb; + } + + public static String escape(String str) { + StringBuilder result = new StringBuilder(); + for (int i = 0; i < str.length(); ++i) { + appendEsc(result, str.charAt(i)); + } + return result.toString(); + } + + public static StringBuilder appendEsc(StringBuilder sb, char c) { + switch (c) { + case '"': + return sb.append("\\\""); + case '\\': + return sb.append("\\\\"); + case '/': + return sb.append("\\/"); + case '\b': + return sb.append("\\b"); + case '\n': + return sb.append("\\n"); + case '\f': + return sb.append("\\f"); + case '\r': + return sb.append("\\r"); + case '\t': + return sb.append("\\t"); + default: + return sb.append(c); + } + } + + public static void main(String[] args) { + String json = args.length > 0 ? args[0] : "{\"a\":[\"b\",\"c\\\nd\"],\"e\":42}"; + System.out.println(json); + System.out.println(indent(json)); + } +} diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java index 3f7e9b5..d150e5d 100644 --- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java +++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java @@ -26,9 +26,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.json.JSONArray; -import org.json.JSONObject; - import org.apache.asterix.api.common.SessionConfig; import org.apache.asterix.result.ResultReader; import org.apache.asterix.result.ResultUtils; @@ -38,6 +35,8 @@ import org.apache.hyracks.api.dataset.ResultSetId; import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.client.dataset.HyracksDataset; +import org.json.JSONArray; +import org.json.JSONObject; public class QueryResultAPIServlet extends HttpServlet { private static final long serialVersionUID = 1L; @@ -89,7 +88,7 @@ // originally determined there. Need to save this value on // some object that we can obtain here. SessionConfig sessionConfig = RESTAPIServlet.initResponse(request, response); - ResultUtils.displayResults(resultReader, sessionConfig); + ResultUtils.displayResults(resultReader, sessionConfig, new ResultUtils.Stats()); } catch (Exception e) { out.println(e.getMessage()); diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java new file mode 100644 index 0000000..c4d270b --- /dev/null +++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.api.http.servlet; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.asterix.api.common.SessionConfig; +import org.apache.asterix.aql.translator.QueryTranslator; +import org.apache.asterix.common.config.GlobalConfig; +import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.compiler.provider.ILangCompilationProvider; +import org.apache.asterix.compiler.provider.SqlppCompilationProvider; +import org.apache.asterix.lang.aql.parser.TokenMgrError; +import org.apache.asterix.lang.common.base.IParser; +import org.apache.asterix.lang.common.base.Statement; +import org.apache.asterix.metadata.MetadataManager; +import org.apache.asterix.result.ResultReader; +import org.apache.asterix.result.ResultUtils; +import org.apache.commons.io.IOUtils; +import org.apache.hyracks.api.client.IHyracksClientConnection; +import org.apache.hyracks.api.dataset.IHyracksDataset; +import org.apache.hyracks.client.dataset.HyracksDataset; + +public class QueryServiceServlet extends HttpServlet { + private static final long serialVersionUID = 1L; + + private static final Logger LOGGER = Logger.getLogger(QueryServiceServlet.class.getName()); + + public static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION"; + public static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET"; + + public enum Parameter { + // Standard + statement, + format, + // Asterix + header + } + + public enum Header { + Accept("Accept"), + ContentLength("Content-Length"); + + private final String str; + + Header(String str) { + this.str = str; + } + + public String str() { + return str; + } + } + + public enum MediaType { + CSV("text/csv"), + JSON("application/json"); + + private final String str; + + MediaType(String str) { + this.str = str; + } + + public String str() { + return str; + } + } + + public enum ResultFields { + requestID, + signature, + status, + results, + errors, + metrics + } + + public enum ResultStatus { + success, + timeout, + errors, + fatal + } + + public enum ErrorField { + code, + msg, + stack + } + + public enum Metrics { + elapsedTime, + executionTime, + resultCount, + resultSize + } + + private final ILangCompilationProvider compilationProvider = new SqlppCompilationProvider(); + + static SessionConfig.OutputFormat getFormat(HttpServletRequest request) { + // First check the "format" parameter. + String format = request.getParameter(Parameter.format.name()); + if (format != null && format.equals("CSV")) { + return SessionConfig.OutputFormat.CSV; + } + // Second check the Accept: HTTP header. + String accept = request.getHeader(Header.Accept.str()); + if (accept != null && accept.contains(MediaType.CSV.str())) { + return SessionConfig.OutputFormat.CSV; + } + return SessionConfig.OutputFormat.CLEAN_JSON; + } + + /** + * Construct a SessionConfig with the appropriate output writer and + * output-format based on the Accept: header and other servlet parameters. + */ + static SessionConfig createSessionConfig(HttpServletRequest request, PrintWriter resultWriter) { + SessionConfig.ResultDecorator resultPrefix = new SessionConfig.ResultDecorator() { + @Override + public PrintWriter print(PrintWriter pw) { + pw.print("\t\""); + pw.print(ResultFields.results.name()); + pw.print("\": "); + return pw; + } + }; + + SessionConfig.ResultDecorator resultPostfix = new SessionConfig.ResultDecorator() { + @Override + public PrintWriter print(PrintWriter pw) { + pw.print(",\n"); + return pw; + } + }; + + SessionConfig.OutputFormat format = getFormat(request); + SessionConfig sessionConfig = new SessionConfig(resultWriter, format, resultPrefix, resultPostfix); + sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, (format == SessionConfig.OutputFormat.CLEAN_JSON)); + + if (format == SessionConfig.OutputFormat.CSV && ("present".equals(request.getParameter(Parameter.header.name())) + || request.getHeader(Header.Accept.str()).contains("header=present"))) { + sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true); + } + return sessionConfig; + } + + /** + * Initialize the Content-Type of the response based on a SessionConfig. + */ + static void initResponse(HttpServletResponse response, SessionConfig sessionConfig) throws IOException { + response.setCharacterEncoding("utf-8"); + switch (sessionConfig.fmt()) { + case CLEAN_JSON: + response.setContentType(MediaType.JSON.str()); + break; + case CSV: + String contentType = MediaType.CSV.str() + "; header=" + + (sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER) ? "present" : "absent"); + response.setContentType(contentType); + break; + } + } + + static void printField(PrintWriter pw, String name, String value) { + printField(pw, name, value, true); + } + + static void printField(PrintWriter pw, String name, String value, boolean comma) { + pw.print("\t\""); + pw.print(name); + pw.print("\": \""); + pw.print(value); + pw.print('"'); + if (comma) { + pw.print(','); + } + pw.print('\n'); + } + + static UUID printRequestId(PrintWriter pw) { + UUID requestId = UUID.randomUUID(); + printField(pw, ResultFields.requestID.name(), requestId.toString()); + return requestId; + } + + static void printSignature(PrintWriter pw) { + printField(pw, ResultFields.signature.name(), "*"); + } + + static void printStatus(PrintWriter pw, ResultStatus rs) { + printField(pw, ResultFields.status.name(), rs.name()); + } + + static void printError(PrintWriter pw, Throwable e) { + final boolean addStack = false; + pw.print("\t\""); + pw.print(ResultFields.errors.name()); + pw.print("\": [{ \n"); + printField(pw, ErrorField.code.name(), "1"); + printField(pw, ErrorField.msg.name(), JSONUtil.escape(e.getMessage()), addStack); + if (addStack) { + StringWriter sw = new StringWriter(); + PrintWriter stackWriter = new PrintWriter(sw); + e.printStackTrace(stackWriter); + stackWriter.close(); + printField(pw, ErrorField.stack.name(), JSONUtil.escape(sw.toString()), false); + } + pw.print("\t}],\n"); + } + + static void printMetrics(PrintWriter pw, long elapsedTime, long executionTime, long resultCount, long resultSize) { + pw.print("\t\""); + pw.print(ResultFields.metrics.name()); + pw.print("\": {\n"); + pw.print("\t"); + printField(pw, Metrics.elapsedTime.name(), String.valueOf(elapsedTime)); + pw.print("\t"); + printField(pw, Metrics.executionTime.name(), String.valueOf(executionTime)); + pw.print("\t"); + printField(pw, Metrics.resultCount.name(), String.valueOf(resultCount)); + pw.print("\t"); + printField(pw, Metrics.resultSize.name(), String.valueOf(resultSize), false); + pw.print("\t}\n"); + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String query = request.getParameter(Parameter.statement.name()); + if (query == null) { + StringWriter sw = new StringWriter(); + IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name()); + query = sw.toString(); + } + handleRequest(request, response, query); + } + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + String query = request.getParameter(Parameter.statement.name()); + handleRequest(request, response, query); + } + + public void handleRequest(HttpServletRequest request, HttpServletResponse response, String query) + throws IOException { + long elapsedStart = System.nanoTime(); + + query = query + ";"; + + final StringWriter stringWriter = new StringWriter(); + final PrintWriter resultWriter = new PrintWriter(stringWriter); + + SessionConfig sessionConfig = createSessionConfig(request, resultWriter); + initResponse(response, sessionConfig); + + int respCode = HttpServletResponse.SC_OK; + ResultUtils.Stats stats = new ResultUtils.Stats(); + long execStart = 0, execEnd = 0; + + resultWriter.print("{\n"); + UUID requestId = printRequestId(resultWriter); + printSignature(resultWriter); + try { + IHyracksClientConnection hcc; + IHyracksDataset hds; + ServletContext context = getServletContext(); + synchronized (context) { + hcc = (IHyracksClientConnection) context.getAttribute(HYRACKS_CONNECTION_ATTR); + hds = (IHyracksDataset) context.getAttribute(HYRACKS_DATASET_ATTR); + if (hds == null) { + hds = new HyracksDataset(hcc, ResultReader.FRAME_SIZE, ResultReader.NUM_READERS); + context.setAttribute(HYRACKS_DATASET_ATTR, hds); + } + } + IParser parser = compilationProvider.getParserFactory().createParser(query); + List<Statement> aqlStatements = parser.parse(); + MetadataManager.INSTANCE.init(); + QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider); + execStart = System.nanoTime(); + translator.compileAndExecute(hcc, hds, QueryTranslator.ResultDelivery.SYNC, stats); + execEnd = System.nanoTime(); + printStatus(resultWriter, ResultStatus.success); + } catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) { + GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, pe.getMessage(), pe); + printError(resultWriter, pe); + printStatus(resultWriter, ResultStatus.fatal); + respCode = HttpServletResponse.SC_BAD_REQUEST; + } catch (Exception e) { + GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e); + printError(resultWriter, e); + printStatus(resultWriter, ResultStatus.fatal); + respCode = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + } + printMetrics(resultWriter, (System.nanoTime() - elapsedStart) / 1000, (execEnd - execStart) / 1000, stats.count, + stats.size); + resultWriter.print("}\n"); + resultWriter.flush(); + String result = stringWriter.toString(); + + GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, result); + //result = JSONUtil.indent(result); + + response.setIntHeader(Header.ContentLength.str(), result.length()); + response.getWriter().print(result); + if (response.getWriter().checkError()) { + LOGGER.warning("Error flushing output writer"); + } + response.setStatus(respCode); + } + +} diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java index 4adadf9..46ea72b 100644 --- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java +++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java @@ -203,7 +203,7 @@ ADDED_PENDINGOP_RECORD_TO_METADATA } - public static enum ResultDelivery { + public enum ResultDelivery { SYNC, ASYNC, ASYNC_DEFERRED @@ -238,6 +238,7 @@ /** * Compiles and submits for execution a list of AQL statements. + * * @param hcc * A Hyracks client connection that is used to submit a jobspec to Hyracks. * @param hdc @@ -249,6 +250,11 @@ */ public void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery) throws Exception { + compileAndExecute(hcc, hdc, resultDelivery, new ResultUtils.Stats()); + } + + public void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery, + ResultUtils.Stats stats) throws Exception { int resultSetIdCounter = 0; FileSplit outputFile = null; IAWriterFactory writerFactory = PrinterBasedWriterFactory.INSTANCE; @@ -381,7 +387,7 @@ metadataProvider.setResultSetId(new ResultSetId(resultSetIdCounter++)); metadataProvider.setResultAsyncMode( resultDelivery == ResultDelivery.ASYNC || resultDelivery == ResultDelivery.ASYNC_DEFERRED); - handleQuery(metadataProvider, (Query) stmt, hcc, hdc, resultDelivery); + handleQuery(metadataProvider, (Query) stmt, hcc, hdc, resultDelivery, stats); break; } @@ -2212,6 +2218,7 @@ /** * Generates a subscription request corresponding to a connect feed request. In addition, provides a boolean * flag indicating if feed intake job needs to be started (source primary feed not found to be active). + * * @param dataverse * @param feed * @param dataset @@ -2483,7 +2490,7 @@ } private void handleQuery(AqlMetadataProvider metadataProvider, Query query, IHyracksClientConnection hcc, - IHyracksDataset hdc, ResultDelivery resultDelivery) throws Exception { + IHyracksDataset hdc, ResultDelivery resultDelivery, ResultUtils.Stats stats) throws Exception { MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); boolean bActiveTxn = true; @@ -2523,7 +2530,7 @@ && sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER)) { ResultUtils.displayCSVHeader(metadataProvider.findOutputRecordType(), sessionConfig); } - ResultUtils.displayResults(resultReader, sessionConfig); + ResultUtils.displayResults(resultReader, sessionConfig, stats); break; case ASYNC_DEFERRED: handle = new JSONArray(); diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java index f2fc9bf..bee284d 100644 --- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java +++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java @@ -28,6 +28,7 @@ import org.apache.asterix.api.http.servlet.FeedServlet; import org.apache.asterix.api.http.servlet.QueryAPIServlet; import org.apache.asterix.api.http.servlet.QueryResultAPIServlet; +import org.apache.asterix.api.http.servlet.QueryServiceServlet; import org.apache.asterix.api.http.servlet.QueryStatusAPIServlet; import org.apache.asterix.api.http.servlet.ShutdownAPIServlet; import org.apache.asterix.api.http.servlet.UpdateAPIServlet; @@ -194,6 +195,8 @@ context.addServlet(new ServletHolder(new ConnectorAPIServlet()), "/connector"); context.addServlet(new ServletHolder(new ShutdownAPIServlet()), "/admin/shutdown"); context.addServlet(new ServletHolder(new VersionAPIServlet()), "/admin/version"); + + context.addServlet(new ServletHolder(new QueryServiceServlet()), "/query/service"); } private void setupFeedServer(AsterixExternalProperties externalProperties) throws Exception { @@ -220,4 +223,4 @@ ClusterState.ACTIVE); } } -} \ No newline at end of file +} diff --git a/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java b/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java index 2f82f80..f1d20d0 100644 --- a/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java +++ b/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java @@ -58,6 +58,11 @@ HTML_ENTITIES.put('>', ">"); } + public static class Stats { + public long count; + public long size; + } + public static String escapeHTML(String s) { for (Character c : HTML_ENTITIES.keySet()) { if (s.indexOf(c) >= 0) { @@ -91,7 +96,8 @@ public static FrameManager resultDisplayFrameMgr = new FrameManager(ResultReader.FRAME_SIZE); - public static void displayResults(ResultReader resultReader, SessionConfig conf) throws HyracksDataException { + public static void displayResults(ResultReader resultReader, SessionConfig conf, Stats stats) + throws HyracksDataException { IFrameTupleAccessor fta = resultReader.getFrameTupleAccessor(); IFrame frame = new VSizeFrame(resultDisplayFrameMgr); @@ -110,6 +116,8 @@ conf.out().println("<h4>Results:</h4>"); conf.out().println("<pre>"); } + + conf.resultPrefix(conf.out()); switch (conf.fmt()) { case LOSSLESS_JSON: @@ -153,6 +161,9 @@ if (conf.fmt() == OutputFormat.CSV) { conf.out().print("\r\n"); } + ++stats.count; + // TODO(tillw) fix this approximation + stats.size += result.length(); } frame.getBuffer().clear(); } finally { @@ -171,6 +182,8 @@ conf.out().println(" ]"); } + conf.resultPostfix(conf.out()); + if (conf.is(SessionConfig.FORMAT_HTML)) { conf.out().println("</pre>"); } diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java index 823e861..3e709bf 100644 --- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java +++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java @@ -505,4 +505,4 @@ MetadataManager.INSTANCE.releaseWriteLatch(); } } -} \ No newline at end of file +} -- To view, visit https://asterix-gerrit.ics.uci.edu/597 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ia494c54f7252445ce38903c0b58fc4e23c324e6e Gerrit-PatchSet: 5 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Till Westmann <[email protected]> Gerrit-Reviewer: Chris Hillery <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
