>From Ali Alsuliman <ali.al.solai...@gmail.com>: Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18006 )
Change subject: [NO ISSUE][API] Add query request param to indicate source ...................................................................... [NO ISSUE][API] Add query request param to indicate source - user model changes: no - storage format changes: no - interface changes: no Details: Add parameter 'source' to QueryServiceRequestParameters to indicate the source of the request. Change-Id: I04e2e2336d6bca82b4fa44a43cb188cfa9e807f7 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18006 Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Michael Blow <mb...@apache.org> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java 5 files changed, 58 insertions(+), 19 deletions(-) Approvals: Michael Blow: Looks good to me, approved Jenkins: Verified; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java index b6913e4..20d79d5 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java @@ -19,6 +19,8 @@ package org.apache.asterix.api.http.server; +import static org.apache.asterix.utils.RedactionUtil.REDACTED_SENSITIVE_ENTRY_VALUE; + import java.io.IOException; import java.util.HashMap; import java.util.Iterator; @@ -43,6 +45,7 @@ import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.server.utils.HttpUtil; import org.apache.hyracks.util.JSONUtil; +import org.apache.hyracks.util.LogRedactionUtil; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; @@ -80,7 +83,8 @@ SIGNATURE("signature"), MULTI_STATEMENT("multi-statement"), MAX_WARNINGS("max-warnings"), - SQL_COMPAT("sql-compat"); + SQL_COMPAT("sql-compat"), + SOURCE("source"); private final String str; @@ -124,6 +128,7 @@ private String statement; private String clientContextID; private String dataverse; + private String source; private ClientType clientType = ClientType.ASTERIX; private OutputFormat format = OutputFormat.CLEAN_JSON; private ResultDelivery mode = ResultDelivery.IMMEDIATE; @@ -172,6 +177,14 @@ this.statement = statement; } + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + public OutputFormat getFormat() { return format; } @@ -380,7 +393,8 @@ ObjectNode object = OBJECT_MAPPER.createObjectNode(); object.put("host", host); object.put("path", path); - object.put("statement", statement != null ? JSONUtil.escape(new StringBuilder(), statement).toString() : null); + object.put("statement", statement != null + ? LogRedactionUtil.statement(JSONUtil.escape(new StringBuilder(), statement).toString()) : null); object.put("pretty", pretty); object.put("mode", mode.getName()); object.put("clientContextID", clientContextID); @@ -402,9 +416,10 @@ object.put("readOnly", readOnly); object.put("maxWarnings", maxWarnings); object.put("sqlCompat", sqlCompatMode); + object.put("source", source); if (statementParams != null) { for (Map.Entry<String, JsonNode> statementParam : statementParams.entrySet()) { - object.set('$' + statementParam.getKey(), statementParam.getValue()); + object.set('$' + statementParam.getKey(), REDACTED_SENSITIVE_ENTRY_VALUE); } } return object; @@ -486,6 +501,7 @@ setSignature(parseBoolean(req, Parameter.SIGNATURE.str(), valGetter, isSignature())); setClientType(parseIfExists(req, Parameter.CLIENT_TYPE.str(), valGetter, getClientType(), clientTypes::get)); setSQLCompatMode(parseBoolean(req, Parameter.SQL_COMPAT.str(), valGetter, isSQLCompatMode())); + setSource(valGetter.apply(req, Parameter.SOURCE.str())); } protected void setExtraParams(JsonNode jsonRequest) throws HyracksDataException { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java index 1966a88..9a8c0d5 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java @@ -90,7 +90,6 @@ import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; import org.apache.hyracks.http.server.utils.HttpUtil; -import org.apache.hyracks.util.LogRedactionUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -278,7 +277,10 @@ if (forceReadOnly) { param.setReadOnly(true); } - LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.statement(param.toString())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("handleRequest: uuid={}, clientContextID={}, {}", requestRef.getUuid(), + param.getClientContextID(), param.toString()); + } delivery = param.getMode(); setSessionConfig(sessionOutput, param, delivery); final ResultProperties resultProperties = new ResultProperties(delivery, param.getMaxResultReads()); @@ -429,14 +431,12 @@ executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); return true; case REQUEST_TIMEOUT: - LOGGER.info(() -> "handleException: request execution timed out: " - + LogRedactionUtil.userData(param.toString())); + LOGGER.info(() -> "handleException: request execution timed out: " + param.toString()); executionState.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK); return true; case REJECT_NODE_UNREGISTERED: case REJECT_BAD_CLUSTER_STATE: - LOGGER.warn(() -> "handleException: " + ex.getMessage() + ": " - + LogRedactionUtil.userData(param.toString())); + LOGGER.warn(() -> "handleException: " + ex.getMessage() + ": " + param.toString()); executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE); return true; default: @@ -456,11 +456,9 @@ QueryServiceRequestParameters param, IServletResponse response) { if (t instanceof org.apache.asterix.lang.sqlpp.parser.TokenMgrError || t instanceof AlgebricksException) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.statement(param.toString()), - t); + LOGGER.debug("handleException: {}: {}", t.getMessage(), param.toString(), t); } else { - LOGGER.info(() -> "handleException: " + t.getMessage() + ": " - + LogRedactionUtil.statement(param.toString())); + LOGGER.info(() -> "handleException: " + t.getMessage() + ": " + param.toString()); } executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); return; @@ -472,7 +470,7 @@ return; } } - LOGGER.warn(() -> "handleException: unexpected exception: " + LogRedactionUtil.userData(param.toString()), t); + LOGGER.warn(() -> "handleException: unexpected exception: " + param.toString(), t); executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java index 05bc87b..e314177 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java @@ -243,7 +243,8 @@ @Override public String toString() { - return String.format("%s(id=%s, from=%s, uuid=%s): %s", getClass().getSimpleName(), requestMessageId, - requestNodeId, requestReference.getUuid(), LogRedactionUtil.statement(statementsText)); + return String.format("%s(id=%s, from=%s, uuid=%s, clientContextID=%s): %s", getClass().getSimpleName(), + requestMessageId, requestNodeId, requestReference.getUuid(), clientContextID, + LogRedactionUtil.statement(statementsText)); } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index afdd035..6b40cbf 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -4837,9 +4837,9 @@ // ensure request not cancelled before running job ensureNotCancelled(clientRequest); final JobId jobId = JobUtils.runJob(hcc, jobSpec, jobFlags, false); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("createAndRunJob jobId:{}, uuid:{}", jobId, - requestParameters.getRequestReference().getUuid()); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Created job {} for query uuid:{}, clientContextID:{}", jobId, + requestParameters.getRequestReference().getUuid(), requestParameters.getClientContextId()); } clientRequest.setJobId(jobId); if (jId != null) { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java index 156b78a..d835664 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java @@ -26,6 +26,8 @@ import org.apache.hyracks.util.ILogRedactor; +import com.fasterxml.jackson.databind.node.TextNode; + public class RedactionUtil { private RedactionUtil() { throw new AssertionError("do not instantiate"); @@ -34,6 +36,7 @@ private static final Pattern STATEMENT_PATTERN = Pattern.compile("(" + SECRET_ACCESS_KEY_FIELD_NAME + ").*", CASE_INSENSITIVE | DOTALL); private static final String STATEMENT_REPLACEMENT = "$1...<redacted sensitive data>"; + public static final TextNode REDACTED_SENSITIVE_ENTRY_VALUE = new TextNode("<redacted sensitive entry>"); public static final ILogRedactor LOG_REDACTOR = new ILogRedactor() { @Override -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18006 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: neo Gerrit-Change-Id: I04e2e2336d6bca82b4fa44a43cb188cfa9e807f7 Gerrit-Change-Number: 18006 Gerrit-PatchSet: 2 Gerrit-Owner: Ali Alsuliman <ali.al.solai...@gmail.com> Gerrit-Reviewer: Ali Alsuliman <ali.al.solai...@gmail.com> Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Michael Blow <mb...@apache.org> Gerrit-MessageType: merged
