This is an automated email from the ASF dual-hosted git repository.
siddteotia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 913f0d81a8 Add support for Base64 Encode/Decode Scalar Functions
(#9114)
913f0d81a8 is described below
commit 913f0d81a8998a2faedd4dc4a581bb53ea9e8a8e
Author: Yifan Zhao <[email protected]>
AuthorDate: Tue Aug 2 14:06:01 2022 -0700
Add support for Base64 Encode/Decode Scalar Functions (#9114)
* before testing on agg
* added null case tests for encode/decode, added invalid check for decode
* roll back style changes
* minor changes
* roll back style changes
* added base64 with user encoding scheme, added nested tranfunc for single
arg
* minor change
* tested binary to base64
* code reformat
* simplied logic, editted all tests, before reformat
* checked style
* fixed format issue
* fixed format issue
* added invalid arg test for toBase64
---
.../LiteralOnlyBrokerRequestTest.java | 92 ++++++++++
.../common/function/scalar/StringFunctions.java | 28 ++++
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 96 +++++++++++
.../ScalarTransformFunctionWrapperTest.java | 24 +++
.../tests/OfflineClusterIntegrationTest.java | 186 ++++++++++++++++++++-
5 files changed, 423 insertions(+), 3 deletions(-)
diff --git
a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java
b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java
index 0868213cb4..bd807818ce 100644
---
a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java
+++
b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java
@@ -20,12 +20,14 @@ package org.apache.pinot.broker.requesthandler;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.Collections;
+import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.pinot.broker.broker.AccessControlFactory;
import org.apache.pinot.broker.broker.AllowAllAccessControlFactory;
import org.apache.pinot.common.metrics.BrokerMetrics;
import org.apache.pinot.common.response.broker.BrokerResponseNative;
+import org.apache.pinot.common.response.broker.ResultTable;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.metrics.PinotMetricUtils;
@@ -104,6 +106,22 @@ public class LiteralOnlyBrokerRequestTest {
Assert.assertFalse(BaseBrokerRequestHandler.isLiteralOnlyQuery(CalciteSqlParser
.compileToPinotQuery("SELECT count(*) from foo "
+ "where bar =
decodeUrl('key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253')")));
+
Assert.assertTrue(BaseBrokerRequestHandler.isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery(
+ "SELECT toUtf8('hello!')," + " fromUtf8(toUtf8('hello!')) FROM
myTable")));
+
Assert.assertFalse(BaseBrokerRequestHandler.isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery(
+ "SELECT reverse(fromUtf8(foo))," + " toUtf8('hello!') FROM myTable")));
+
Assert.assertTrue(BaseBrokerRequestHandler.isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery(
+ "SELECT toBase64(toUtf8('hello!'))," + " fromBase64('aGVsbG8h') FROM
myTable")));
+
Assert.assertFalse(BaseBrokerRequestHandler.isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery(
+ "SELECT reverse(toBase64(foo))," + " toBase64(fromBase64('aGVsbG8h'))
FROM myTable")));
+ Assert.assertFalse(BaseBrokerRequestHandler.isLiteralOnlyQuery(
+ CalciteSqlParser.compileToPinotQuery("SELECT
fromBase64(toBase64(to_utf8(foo))) FROM myTable")));
+ Assert.assertFalse(BaseBrokerRequestHandler.isLiteralOnlyQuery(
+ CalciteSqlParser.compileToPinotQuery("SELECT count(*) from foo " +
"where bar = toBase64(toASCII('hello!'))")));
+ Assert.assertFalse(BaseBrokerRequestHandler.isLiteralOnlyQuery(
+ CalciteSqlParser.compileToPinotQuery("SELECT count(*) from foo " +
"where bar = fromBase64('aGVsbG8h')")));
+
Assert.assertFalse(BaseBrokerRequestHandler.isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery(
+ "SELECT count(*) from foo " + "where bar =
fromUtf8(fromBase64('aGVsbG8h'))")));
}
@Test
@@ -115,6 +133,10 @@ public class LiteralOnlyBrokerRequestTest {
Assert.assertTrue(BaseBrokerRequestHandler.isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery(
"SELECT encodeUrl('key1=value 1&key2=value@!$2&key3=value%3') AS
encoded, "
+
"decodeUrl('key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253') AS
decoded")));
+
Assert.assertTrue(BaseBrokerRequestHandler.isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery(
+ "SELECT toUtf8('hello!') AS encoded, " + "fromUtf8(toUtf8('hello!'))
AS decoded")));
+
Assert.assertTrue(BaseBrokerRequestHandler.isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery(
+ "SELECT toBase64(toUtf8('hello!')) AS encoded, " +
"fromBase64('aGVsbG8h') AS decoded")));
}
@Test
@@ -211,6 +233,76 @@ public class LiteralOnlyBrokerRequestTest {
Assert.assertEquals(brokerResponse.getResultTable().getRows().get(0)[1].toString(),
"key1=value 1&key2=value@!$2&key3=value%3");
Assert.assertEquals(brokerResponse.getTotalDocs(), 0);
+
+ request = JsonUtils.stringToJsonNode(
+ "{\"sql\":\"SELECT toBase64(toUtf8('hello!')) AS encoded, " +
"fromUtf8(fromBase64('aGVsbG8h')) AS decoded\"}");
+ requestStats = Tracing.getTracer().createRequestScope();
+ brokerResponse = requestHandler.handleRequest(request, null, requestStats);
+ ResultTable resultTable = brokerResponse.getResultTable();
+ DataSchema dataSchema = resultTable.getDataSchema();
+ List<Object[]> rows = resultTable.getRows();
+ Assert.assertEquals(dataSchema.getColumnName(0), "encoded");
+ Assert.assertEquals(dataSchema.getColumnDataType(0),
DataSchema.ColumnDataType.STRING);
+ Assert.assertEquals(dataSchema.getColumnName(1), "decoded");
+ Assert.assertEquals(dataSchema.getColumnDataType(1),
DataSchema.ColumnDataType.STRING);
+ Assert.assertEquals(rows.size(), 1);
+ Assert.assertEquals(rows.get(0).length, 2);
+ Assert.assertEquals(rows.get(0)[0].toString(), "aGVsbG8h");
+ Assert.assertEquals(rows.get(0)[1].toString(), "hello!");
+ Assert.assertEquals(brokerResponse.getTotalDocs(), 0);
+
+ request = JsonUtils.stringToJsonNode(
+ "{\"sql\":\"SELECT fromUtf8(fromBase64(toBase64(toUtf8('nested')))) AS
output\"}");
+ requestStats = Tracing.getTracer().createRequestScope();
+ brokerResponse = requestHandler.handleRequest(request, null, requestStats);
+ resultTable = brokerResponse.getResultTable();
+ dataSchema = resultTable.getDataSchema();
+ rows = resultTable.getRows();
+ Assert.assertEquals(dataSchema.getColumnName(0), "output");
+ Assert.assertEquals(dataSchema.getColumnDataType(0),
DataSchema.ColumnDataType.STRING);
+ Assert.assertEquals(rows.size(), 1);
+ Assert.assertEquals(rows.get(0).length, 1);
+ Assert.assertEquals(rows.get(0)[0].toString(), "nested");
+ Assert.assertEquals(brokerResponse.getTotalDocs(), 0);
+
+ request = JsonUtils.stringToJsonNode(
+ "{\"sql\":\"SELECT toBase64(toUtf8('this is a long string that will
encode to more than 76 characters using "
+ + "base64'))"
+ + " AS encoded\"}");
+ requestStats = Tracing.getTracer().createRequestScope();
+ brokerResponse = requestHandler.handleRequest(request, null, requestStats);
+ resultTable = brokerResponse.getResultTable();
+ dataSchema = resultTable.getDataSchema();
+ rows = resultTable.getRows();
+ Assert.assertEquals(dataSchema.getColumnName(0), "encoded");
+ Assert.assertEquals(dataSchema.getColumnDataType(0),
DataSchema.ColumnDataType.STRING);
+ Assert.assertEquals(rows.size(), 1);
+ Assert.assertEquals(rows.get(0).length, 1);
+ Assert.assertEquals(rows.get(0)[0].toString(),
+
"dGhpcyBpcyBhIGxvbmcgc3RyaW5nIHRoYXQgd2lsbCBlbmNvZGUgdG8gbW9yZSB0aGFuIDc2IGNoYXJhY3RlcnMgdXNpbmcgYmFzZTY0");
+ Assert.assertEquals(brokerResponse.getTotalDocs(), 0);
+
+ request = JsonUtils.stringToJsonNode("{\"sql\":\"SELECT
fromUtf8(fromBase64"
+ +
"('dGhpcyBpcyBhIGxvbmcgc3RyaW5nIHRoYXQgd2lsbCBlbmNvZGUgdG8gbW9yZSB0aGFuIDc2IGNoYXJhY3RlcnMgdXNpbmcgYmFzZTY0"
+ + "')) AS decoded\"}");
+ requestStats = Tracing.getTracer().createRequestScope();
+ brokerResponse = requestHandler.handleRequest(request, null, requestStats);
+ resultTable = brokerResponse.getResultTable();
+ dataSchema = resultTable.getDataSchema();
+ rows = resultTable.getRows();
+ Assert.assertEquals(dataSchema.getColumnName(0), "decoded");
+ Assert.assertEquals(dataSchema.getColumnDataType(0),
DataSchema.ColumnDataType.STRING);
+ Assert.assertEquals(rows.size(), 1);
+ Assert.assertEquals(rows.get(0).length, 1);
+ Assert.assertEquals(rows.get(0)[0].toString(),
+ "this is a long string that will encode to more than 76 characters
using base64");
+ Assert.assertEquals(brokerResponse.getTotalDocs(), 0);
+
+ request = JsonUtils.stringToJsonNode("{\"sql\":\"SELECT fromBase64" + "(0)
AS decoded\"}");
+ requestStats = Tracing.getTracer().createRequestScope();
+ brokerResponse = requestHandler.handleRequest(request, null, requestStats);
+ Assert.assertTrue(
+
brokerResponse.getProcessingExceptions().get(0).getMessage().contains("IllegalArgumentException"));
}
/** Tests for EXPLAIN PLAN for literal only queries. */
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
index 7d1d7a7389..08b9c5ac22 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java
@@ -23,6 +23,7 @@ import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.text.Normalizer;
+import java.util.Base64;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
@@ -393,6 +394,15 @@ public class StringFunctions {
return input.getBytes(StandardCharsets.UTF_8);
}
+ /**
+ * @param input bytes
+ * @return UTF8 encoded string
+ */
+ @ScalarFunction
+ public static String fromUtf8(byte[] input) {
+ return new String(input, StandardCharsets.UTF_8);
+ }
+
/**
* @see StandardCharsets#US_ASCII#encode(String)
* @param input
@@ -560,4 +570,22 @@ public class StringFunctions {
throws UnsupportedEncodingException {
return URLDecoder.decode(input, StandardCharsets.UTF_8.toString());
}
+
+ /**
+ * @param input binary data
+ * @return Base64 encoded String
+ */
+ @ScalarFunction
+ public static String toBase64(byte[] input) {
+ return Base64.getEncoder().encodeToString(input);
+ }
+
+ /**
+ * @param input Base64 encoded String
+ * @return decoded binary data
+ */
+ @ScalarFunction
+ public static byte[] fromBase64(String input) {
+ return Base64.getDecoder().decode(input);
+ }
}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
index cd88b02d8a..5801d14f0a 100644
---
a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
@@ -1890,6 +1890,68 @@ public class CalciteSqlCompilerTest {
decoded =
and.getOperands().get(1).getFunctionCall().getOperands().get(1).getLiteral().getStringValue();
Assert.assertEquals(encoded,
"key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253");
Assert.assertEquals(decoded, "key1=value 1&key2=value@!$2&key3=value%3");
+
+ query = "select toBase64(toUtf8('hello!')),
fromUtf8(fromBase64('aGVsbG8h')) from mytable";
+ pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ String encodedBase64 =
pinotQuery.getSelectList().get(0).getLiteral().getStringValue();
+ String decodedBase64 =
pinotQuery.getSelectList().get(1).getLiteral().getStringValue();
+ Assert.assertEquals(encodedBase64, "aGVsbG8h");
+ Assert.assertEquals(decodedBase64, "hello!");
+
+ query = "select toBase64(fromBase64('aGVsbG8h')),
fromUtf8(fromBase64(toBase64(toUtf8('hello!')))) from mytable";
+ pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ encodedBase64 =
pinotQuery.getSelectList().get(0).getLiteral().getStringValue();
+ decodedBase64 =
pinotQuery.getSelectList().get(1).getLiteral().getStringValue();
+ Assert.assertEquals(encodedBase64, "aGVsbG8h");
+ Assert.assertEquals(decodedBase64, "hello!");
+
+ query =
+ "select toBase64(toUtf8(upper('hello!'))),
fromUtf8(fromBase64(toBase64(toUtf8(upper('hello!'))))) from "
+ + "mytable";
+ pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ encodedBase64 =
pinotQuery.getSelectList().get(0).getLiteral().getStringValue();
+ decodedBase64 =
pinotQuery.getSelectList().get(1).getLiteral().getStringValue();
+ Assert.assertEquals(encodedBase64, "SEVMTE8h");
+ Assert.assertEquals(decodedBase64, "HELLO!");
+
+ query =
+ "select
reverse(fromUtf8(fromBase64(toBase64(toUtf8(upper('hello!')))))) from mytable
where fromUtf8"
+ + "(fromBase64(toBase64(toUtf8(upper('hello!')))))"
+ + " = bar";
+ pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ String arg1 =
pinotQuery.getSelectList().get(0).getLiteral().getStringValue();
+ String leftOp =
+
pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(1).getLiteral().getStringValue();
+ Assert.assertEquals(arg1, "!OLLEH");
+ Assert.assertEquals(leftOp, "HELLO!");
+
+ query = "select a from mytable where foo = toBase64(toUtf8('hello!')) and
bar = fromUtf8(fromBase64('aGVsbG8h'))";
+ pinotQuery = CalciteSqlParser.compileToPinotQuery(query);
+ and = pinotQuery.getFilterExpression().getFunctionCall();
+ encoded =
and.getOperands().get(0).getFunctionCall().getOperands().get(1).getLiteral().getStringValue();
+ decoded =
and.getOperands().get(1).getFunctionCall().getOperands().get(1).getLiteral().getStringValue();
+ Assert.assertEquals(encoded, "aGVsbG8h");
+ Assert.assertEquals(decoded, "hello!");
+
+ query = "select fromBase64('hello') from mytable";
+ Exception expectedError = null;
+ try {
+ CalciteSqlParser.compileToPinotQuery(query);
+ } catch (Exception e) {
+ expectedError = e;
+ }
+ Assert.assertNotNull(expectedError);
+ Assert.assertTrue(expectedError instanceof SqlCompilationException);
+
+ query = "select toBase64('hello!') from mytable";
+ expectedError = null;
+ try {
+ CalciteSqlParser.compileToPinotQuery(query);
+ } catch (Exception e) {
+ expectedError = e;
+ }
+ Assert.assertNotNull(expectedError);
+ Assert.assertTrue(expectedError instanceof SqlCompilationException);
}
@Test
@@ -2012,6 +2074,40 @@ public class CalciteSqlCompilerTest {
Assert.assertNotNull(expression.getFunctionCall());
Assert.assertEquals(expression.getFunctionCall().getOperator(), "count");
Assert.assertEquals(expression.getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"*");
+
+ expression =
CalciteSqlParser.compileToExpression("toBase64(toUtf8('hello!'))");
+ Assert.assertNotNull(expression.getFunctionCall());
+ pinotQuery.setFilterExpression(expression);
+ pinotQuery = compileTimeFunctionsInvoker.rewrite(pinotQuery);
+ expression = pinotQuery.getFilterExpression();
+ Assert.assertNotNull(expression.getLiteral());
+ Assert.assertEquals(expression.getLiteral().getFieldValue(), "aGVsbG8h");
+
+ expression =
CalciteSqlParser.compileToExpression("fromUtf8(fromBase64('aGVsbG8h'))");
+ Assert.assertNotNull(expression.getFunctionCall());
+ pinotQuery.setFilterExpression(expression);
+ pinotQuery = compileTimeFunctionsInvoker.rewrite(pinotQuery);
+ expression = pinotQuery.getFilterExpression();
+ Assert.assertNotNull(expression.getLiteral());
+ Assert.assertEquals(expression.getLiteral().getFieldValue(), "hello!");
+
+ expression = CalciteSqlParser.compileToExpression("fromBase64(foo)");
+ Assert.assertNotNull(expression.getFunctionCall());
+ pinotQuery.setFilterExpression(expression);
+ pinotQuery = compileTimeFunctionsInvoker.rewrite(pinotQuery);
+ expression = pinotQuery.getFilterExpression();
+ Assert.assertNotNull(expression.getFunctionCall());
+ Assert.assertEquals(expression.getFunctionCall().getOperator(),
"frombase64");
+
Assert.assertEquals(expression.getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"foo");
+
+ expression = CalciteSqlParser.compileToExpression("toBase64(foo)");
+ Assert.assertNotNull(expression.getFunctionCall());
+ pinotQuery.setFilterExpression(expression);
+ pinotQuery = compileTimeFunctionsInvoker.rewrite(pinotQuery);
+ expression = pinotQuery.getFilterExpression();
+ Assert.assertNotNull(expression.getFunctionCall());
+ Assert.assertEquals(expression.getFunctionCall().getOperator(),
"tobase64");
+
Assert.assertEquals(expression.getFunctionCall().getOperands().get(0).getIdentifier().getName(),
"foo");
}
@Test
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapperTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapperTest.java
index 69e4b045a7..5187a41ed0 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapperTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapperTest.java
@@ -23,6 +23,7 @@ import it.unimi.dsi.fastutil.objects.ObjectLinkedOpenHashSet;
import java.nio.charset.StandardCharsets;
import java.text.Normalizer;
import java.util.Arrays;
+import java.util.Base64;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
@@ -819,4 +820,27 @@ public class ScalarTransformFunctionWrapperTest extends
BaseTransformFunctionTes
}
testTransformFunctionMV(transformFunction, expectedValues);
}
+
+ @Test
+ public void testBase64TransformFunction() {
+ ExpressionContext expression =
RequestContextUtils.getExpression(String.format("toBase64(%s)",
BYTES_SV_COLUMN));
+ TransformFunction transformFunction =
TransformFunctionFactory.get(expression, _dataSourceMap);
+ assertTrue(transformFunction instanceof ScalarTransformFunctionWrapper);
+ assertEquals(transformFunction.getName(), "toBase64");
+ String[] expectedValues = new String[NUM_ROWS];
+ for (int i = 0; i < NUM_ROWS; i++) {
+ expectedValues[i] =
Base64.getEncoder().encodeToString(_bytesSVValues[i]);
+ }
+ testTransformFunction(transformFunction, expectedValues);
+
+ expression =
RequestContextUtils.getExpression(String.format("fromBase64(toBase64(%s))",
BYTES_SV_COLUMN));
+ transformFunction = TransformFunctionFactory.get(expression,
_dataSourceMap);
+ assertTrue(transformFunction instanceof ScalarTransformFunctionWrapper);
+ assertEquals(transformFunction.getName(), "fromBase64");
+ byte[][] expectedBinaryValues = new byte[NUM_ROWS][];
+ for (int i = 0; i < NUM_ROWS; i++) {
+ expectedBinaryValues[i] =
Base64.getDecoder().decode(Base64.getEncoder().encodeToString(_bytesSVValues[i]));
+ }
+ testTransformFunction(transformFunction, expectedBinaryValues);
+ }
}
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index 555cafe49f..c95e3d91e5 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -84,8 +84,7 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import static
org.apache.pinot.common.function.scalar.StringFunctions.decodeUrl;
-import static
org.apache.pinot.common.function.scalar.StringFunctions.encodeUrl;
+import static org.apache.pinot.common.function.scalar.StringFunctions.*;
import static
org.apache.pinot.controller.helix.core.PinotHelixResourceManager.EXTERNAL_VIEW_CHECK_INTERVAL_MS;
import static
org.apache.pinot.controller.helix.core.PinotHelixResourceManager.EXTERNAL_VIEW_ONLINE_SEGMENTS_MAX_WAIT_MS;
import static org.testng.Assert.*;
@@ -606,6 +605,178 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
assertEquals(decodedString, expectedUrlStr);
}
+ @Test
+ public void testBase64Func()
+ throws Exception {
+
+ // string literal
+ String sqlQuery = "SELECT toBase64(toUtf8('hello!')), " +
"fromUtf8(fromBase64('aGVsbG8h')) FROM myTable";
+ JsonNode response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ JsonNode resultTable = response.get("resultTable");
+ JsonNode dataSchema = resultTable.get("dataSchema");
+ assertEquals(dataSchema.get("columnDataTypes").toString(),
"[\"STRING\",\"STRING\"]");
+ JsonNode rows = response.get("resultTable").get("rows");
+
+ String encodedString = rows.get(0).get(0).asText();
+ String expectedEncodedStr = toBase64(toUtf8("hello!"));
+ assertEquals(encodedString, expectedEncodedStr);
+ String decodedString = rows.get(0).get(1).asText();
+ String expectedDecodedStr = fromUtf8(fromBase64("aGVsbG8h"));
+ assertEquals(decodedString, expectedDecodedStr);
+
+ // long string literal encode
+ sqlQuery =
+ "SELECT toBase64(toUtf8('this is a long string that will encode to
more than 76 characters using base64')) "
+ + "FROM "
+ + "myTable";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ rows = resultTable.get("rows");
+ encodedString = rows.get(0).get(0).asText();
+ assertEquals(encodedString,
+ toBase64(toUtf8("this is a long string that will encode to more than
76 characters using base64")));
+
+ // long string literal decode
+ sqlQuery = "SELECT fromUtf8(fromBase64"
+ +
"('dGhpcyBpcyBhIGxvbmcgc3RyaW5nIHRoYXQgd2lsbCBlbmNvZGUgdG8gbW9yZSB0aGFuIDc2IGNoYXJhY3RlcnMgdXNpbmcgYmFzZTY0"
+ + "')) FROM myTable";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ rows = resultTable.get("rows");
+ decodedString = rows.get(0).get(0).asText();
+ assertEquals(decodedString, fromUtf8(fromBase64(
+
"dGhpcyBpcyBhIGxvbmcgc3RyaW5nIHRoYXQgd2lsbCBlbmNvZGUgdG8gbW9yZSB0aGFuIDc2IGNoYXJhY3RlcnMgdXNpbmcgYmFzZTY0")));
+
+ // non-string literal
+ sqlQuery = "SELECT toBase64(toUtf8(123)),
fromUtf8(fromBase64(toBase64(toUtf8(123)))), 123 FROM myTable";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ rows = resultTable.get("rows");
+ encodedString = rows.get(0).get(0).asText();
+ decodedString = rows.get(0).get(1).asText();
+ String originalCol = rows.get(0).get(2).asText();
+ assertEquals(decodedString, originalCol);
+ assertEquals(encodedString, toBase64(toUtf8("123")));
+
+ // identifier
+ sqlQuery =
+ "SELECT Carrier, toBase64(toUtf8(Carrier)),
fromUtf8(fromBase64(toBase64(toUtf8(Carrier)))), fromBase64"
+ + "(toBase64(toUtf8(Carrier))) FROM myTable LIMIT 100";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ dataSchema = resultTable.get("dataSchema");
+ assertEquals(dataSchema.get("columnDataTypes").toString(),
"[\"STRING\",\"STRING\",\"STRING\",\"BYTES\"]");
+ rows = response.get("resultTable").get("rows");
+ assertEquals(rows.size(), 100);
+ for (int i = 0; i < 100; i++) {
+ String original = rows.get(0).asText();
+ String encoded = rows.get(1).asText();
+ String decoded = rows.get(2).asText();
+ assertEquals(original, decoded);
+ assertEquals(encoded, toBase64(toUtf8(original)));
+ assertEquals(decoded, fromUtf8(fromBase64(toBase64(toUtf8(original)))));
+ }
+
+ // invalid argument
+ sqlQuery = "SELECT toBase64() FROM myTable";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+
assertTrue(response.get("exceptions").get(0).get("message").toString().startsWith("\"QueryExecutionError"));
+
+ // invalid argument
+ sqlQuery = "SELECT fromBase64() FROM myTable";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+
assertTrue(response.get("exceptions").get(0).get("message").toString().startsWith("\"QueryExecutionError"));
+
+ // invalid argument
+ sqlQuery = "SELECT toBase64('hello!') FROM myTable";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+
assertTrue(response.get("exceptions").get(0).get("message").toString().contains("SqlCompilationException"));
+
+ // invalid argument
+ sqlQuery = "SELECT fromBase64('hello!') FROM myTable";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+
assertTrue(response.get("exceptions").get(0).get("message").toString().contains("IllegalArgumentException"));
+
+ // string literal used in a filter
+ sqlQuery =
+ "SELECT * FROM myTable WHERE fromUtf8(fromBase64('aGVsbG8h')) !=
Carrier AND toBase64(toUtf8('hello!')) != "
+ + "Carrier LIMIT 10";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ rows = resultTable.get("rows");
+ assertEquals(rows.size(), 10);
+
+ // non-string literal used in a filter
+ sqlQuery = "SELECT * FROM myTable WHERE
fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))) != Carrier LIMIT 10";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ rows = resultTable.get("rows");
+ assertEquals(rows.size(), 10);
+
+ // string identifier used in a filter
+ sqlQuery = "SELECT * FROM myTable WHERE
fromUtf8(fromBase64(toBase64(toUtf8(Carrier)))) = Carrier LIMIT 10";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ rows = resultTable.get("rows");
+ assertEquals(rows.size(), 10);
+
+ // non-string identifier used in a filter
+ sqlQuery =
+ "SELECT fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))), AirlineID
FROM myTable WHERE fromUtf8(fromBase64"
+ + "(toBase64(toUtf8(AirlineID)))) = "
+ + "AirlineID LIMIT 10";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ dataSchema = resultTable.get("dataSchema");
+ assertEquals(dataSchema.get("columnDataTypes").toString(),
"[\"STRING\",\"LONG\"]");
+ rows = resultTable.get("rows");
+ assertEquals(rows.size(), 10);
+
+ // string identifier used in group by order by
+ sqlQuery =
+ "SELECT Carrier as originalCol, toBase64(toUtf8(Carrier)) as encoded,
fromUtf8(fromBase64(toBase64(toUtf8"
+ + "(Carrier)))) as decoded "
+ + "FROM myTable GROUP BY Carrier, toBase64(toUtf8(Carrier)),
fromUtf8(fromBase64(toBase64(toUtf8(Carrier)"
+ + "))) ORDER BY toBase64(toUtf8(Carrier))"
+ + " LIMIT 10";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ dataSchema = resultTable.get("dataSchema");
+ assertEquals(dataSchema.get("columnDataTypes").toString(),
"[\"STRING\",\"STRING\",\"STRING\"]");
+ rows = resultTable.get("rows");
+ assertEquals(rows.size(), 10);
+ for (int i = 0; i < 10; i++) {
+ String original = rows.get(0).asText();
+ String encoded = rows.get(1).asText();
+ String decoded = rows.get(2).asText();
+ assertEquals(original, decoded);
+ assertEquals(encoded, toBase64(toUtf8(original)));
+ assertEquals(decoded, fromUtf8(fromBase64(toBase64(toUtf8(original)))));
+ }
+
+ // non-string identifier used in group by order by
+ sqlQuery =
+ "SELECT AirlineID as originalCol, toBase64(toUtf8(AirlineID)) as
encoded, fromUtf8(fromBase64(toBase64(toUtf8"
+ + "(AirlineID)))) as decoded "
+ + "FROM myTable GROUP BY AirlineID, toBase64(toUtf8(AirlineID)),
fromUtf8(fromBase64(toBase64(toUtf8"
+ + "(AirlineID)))) ORDER BY "
+ + "fromUtf8(fromBase64(toBase64(toUtf8(AirlineID)))) LIMIT 10";
+ response = postQuery(sqlQuery, _brokerBaseApiUrl);
+ resultTable = response.get("resultTable");
+ dataSchema = resultTable.get("dataSchema");
+ assertEquals(dataSchema.get("columnDataTypes").toString(),
"[\"LONG\",\"STRING\",\"STRING\"]");
+ rows = resultTable.get("rows");
+ assertEquals(rows.size(), 10);
+ for (int i = 0; i < 10; i++) {
+ String original = rows.get(0).asText();
+ String encoded = rows.get(1).asText();
+ String decoded = rows.get(2).asText();
+ assertEquals(original, decoded);
+ assertEquals(encoded, toBase64(toUtf8(original)));
+ assertEquals(decoded, fromUtf8(fromBase64(toBase64(toUtf8(original)))));
+ }
+ }
+
@Test
public void testLiteralOnlyFunc()
throws Exception {
@@ -614,7 +785,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
String sqlQuery =
"SELECT 1, now() as currentTs, ago('PT1H') as oneHourAgoTs, 'abc',
toDateTime(now(), 'yyyy-MM-dd z') as "
+ "today, now(), ago('PT1H'), encodeUrl('key1=value
1&key2=value@!$2&key3=value%3') as encodedUrl, "
- +
"decodeUrl('key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253') as
decodedUrl";
+ +
"decodeUrl('key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253') as
decodedUrl, toBase64"
+ + "(toUtf8('hello!')) as toBase64,
fromUtf8(fromBase64('aGVsbG8h')) as fromBase64";
JsonNode response = postQuery(sqlQuery, _brokerBaseApiUrl);
long currentTsMax = System.currentTimeMillis();
long oneHourAgoTsMax = currentTsMax - ONE_HOUR_IN_MS;
@@ -628,6 +800,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
String oneHourAgoColumnName =
response.get("resultTable").get("dataSchema").get("columnNames").get(6).asText();
assertEquals(response.get("resultTable").get("dataSchema").get("columnNames").get(7).asText(),
"encodedUrl");
assertEquals(response.get("resultTable").get("dataSchema").get("columnNames").get(8).asText(),
"decodedUrl");
+
assertEquals(response.get("resultTable").get("dataSchema").get("columnNames").get(9).asText(),
"toBase64");
+
assertEquals(response.get("resultTable").get("dataSchema").get("columnNames").get(10).asText(),
"fromBase64");
assertTrue(Long.parseLong(nowColumnName) > currentTsMin);
assertTrue(Long.parseLong(nowColumnName) < currentTsMax);
assertTrue(Long.parseLong(oneHourAgoColumnName) > oneHourAgoTsMin);
@@ -642,6 +816,8 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
assertEquals(response.get("resultTable").get("dataSchema").get("columnDataTypes").get(6).asText(),
"LONG");
assertEquals(response.get("resultTable").get("dataSchema").get("columnDataTypes").get(7).asText(),
"STRING");
assertEquals(response.get("resultTable").get("dataSchema").get("columnDataTypes").get(8).asText(),
"STRING");
+
assertEquals(response.get("resultTable").get("dataSchema").get("columnDataTypes").get(9).asText(),
"STRING");
+
assertEquals(response.get("resultTable").get("dataSchema").get("columnDataTypes").get(10).asText(),
"STRING");
int first = response.get("resultTable").get("rows").get(0).get(0).asInt();
long second =
response.get("resultTable").get("rows").get(0).get(1).asLong();
@@ -665,6 +841,10 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
"key1%3Dvalue+1%26key2%3Dvalue%40%21%242%26key3%3Dvalue%253");
assertEquals(response.get("resultTable").get("rows").get(0).get(8).asText(),
"key1=value 1&key2=value@!$2&key3=value%3");
+
assertEquals(response.get("resultTable").get("rows").get(0).get(9).asText(),
+ "aGVsbG8h");
+
assertEquals(response.get("resultTable").get("rows").get(0).get(10).asText(),
+ "hello!");
}
@Test(dependsOnMethods = "testBloomFilterTriggering")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
