Jackie-Jiang commented on code in PR #18979:
URL: https://github.com/apache/pinot/pull/18979#discussion_r4364679548
##########
pinot-common/src/main/java/org/apache/pinot/common/function/scalar/JsonFunctions.java:
##########
@@ -611,6 +634,409 @@ public static Object jsonExtractObject(@Nullable Object
object) {
return null;
}
+ /// Extract a scalar (or scalar-array) value from a JSON document and coerce
it to `resultsType`.
+ ///
+ /// Scalar-function counterpart of the `jsonExtractScalar` transform
(`JsonExtractScalarTransformFunction` in
+ /// pinot-core), so that `json_extract_scalar(...)` resolves in the
multi-stage engine and in ad-hoc scalar
+ /// contexts. `resultsType` is a Pinot [DataType] name, optionally suffixed
with `_ARRAY` for a multi-value
+ /// result. Supported types are
`INT/LONG/FLOAT/DOUBLE/BIG_DECIMAL/BOOLEAN/TIMESTAMP/STRING/JSON/BYTES` and
+ /// the `INT/LONG/FLOAT/DOUBLE/BIG_DECIMAL/BOOLEAN/TIMESTAMP/STRING` array
variants.
+ ///
+ /// The document may be a `String`, a UTF-8 encoded `byte[]` (BYTES columns)
or an already-parsed container.
+ /// Coercion mirrors the transform exactly: `BOOLEAN` is returned as its
stored `INT` (0/1), `TIMESTAMP` as
+ /// epoch millis (numeric values as-is, strings via ISO-8601), `BIG_DECIMAL`
/ `STRING` / `JSON` / `LONG`
+ /// use a BigDecimal-preserving parser. The 3-argument form throws on an
unresolved single-value path. The
+ /// 4-argument form returns `defaultValue` (including SQL `NULL`). A
multi-value path yields an empty
+ /// array when unresolved or when the resolved value is not a JSON array,
but a `null` element inside a
+ /// resolved array still throws unless a default is supplied. A malformed
JSON document is treated as
+ /// unresolved. An illegal JSONPath (for example `$[`) is rejected, matching
transform init.
+ @ScalarFunction
+ public static Object jsonExtractScalar(Object jsonInput, String jsonPath,
String resultsType) {
+ return jsonExtractScalarInternal(jsonInput, jsonPath, resultsType, null,
false);
+ }
+
+ /// See [#jsonExtractScalar(Object, String, String)]. `defaultValue` is
returned (coerced to `resultsType`)
+ /// when the path resolves to `null` or the document is malformed. An
explicit SQL `NULL` default returns
+ /// Java `null` rather than throwing.
+ @ScalarFunction(nullableParameters = true)
+ public static Object jsonExtractScalar(@Nullable Object jsonInput, String
jsonPath, String resultsType,
+ @Nullable Object defaultValue) {
+ return jsonExtractScalarInternal(jsonInput, jsonPath, resultsType,
defaultValue, true);
+ }
+
+ @Nullable
+ private static Object jsonExtractScalarInternal(@Nullable Object jsonInput,
String jsonPath, String resultsType,
+ @Nullable Object defaultValue, boolean hasDefault) {
+ String type = resultsType.toUpperCase();
+ boolean isSingleValue = !type.endsWith("_ARRAY");
+ DataType dataType;
+ try {
+ dataType = DataType.valueOf(isSingleValue ? type : type.substring(0,
type.length() - 6));
+ } catch (IllegalArgumentException e) {
+ throw new
IllegalArgumentException(unsupportedResultsTypeMessage(resultsType));
+ }
+ // BIG_DECIMAL / STRING / JSON / LONG must read floats as BigDecimal so
values above 2^53 stay exact.
+ boolean useBigDecimal = dataType == DataType.BIG_DECIMAL || dataType ==
DataType.STRING
+ || dataType == DataType.JSON || dataType == DataType.LONG;
+ // Compile the path before touching the document. `$[` must fail the
query, not become a default,
+ // matching JsonExtractScalarTransformFunction#init.
+ JsonPath compiledPath = JsonPathCache.INSTANCE.getOrCompute(jsonPath);
+ if (isSingleValue) {
+ Object value = readJsonPathValue(jsonInput, compiledPath, useBigDecimal);
+ if (value == null) {
+ if (!hasDefault) {
+ throw new IllegalArgumentException(
+ "Cannot resolve JSON path on some records. Consider setting a
default value.");
+ }
+ if (defaultValue == null) {
+ return null;
+ }
+ return coerceScalar(defaultValue, dataType, true);
+ }
+ return coerceScalar(value, dataType, false);
+ }
+ return coerceScalarArray(readJsonPathArray(jsonInput, compiledPath,
useBigDecimal), dataType, defaultValue,
+ hasDefault);
+ }
+
+ private static DocumentContext parseJsonDocument(Object jsonInput,
ParseContext parseContext) {
+ if (jsonInput instanceof String) {
+ return parseContext.parse((String) jsonInput);
+ }
+ if (jsonInput instanceof byte[]) {
+ // BYTES columns carry the raw UTF-8 document; parse(Object) would treat
the array as already parsed.
+ return parseContext.parseUtf8((byte[]) jsonInput);
+ }
+ return parseContext.parse(jsonInput);
+ }
+
+ @Nullable
+ private static Object readJsonPathValue(@Nullable Object jsonInput, JsonPath
jsonPath, boolean useBigDecimal) {
+ DocumentContext document = parseJsonDocumentOrNull(jsonInput,
useBigDecimal);
+ if (document == null) {
+ return null;
+ }
+ return document.read(jsonPath);
+ }
+
+ @Nullable
+ private static Object[] readJsonPathArray(@Nullable Object jsonInput,
JsonPath jsonPath, boolean useBigDecimal) {
+ DocumentContext document = parseJsonDocumentOrNull(jsonInput,
useBigDecimal);
+ if (document == null) {
+ return null;
+ }
+ return jsonArrayOrUnresolved(document.read(jsonPath));
+ }
+
+ /// Parses the document only. A bad document is unresolved (null). Path
compile already happened
+ /// in [jsonExtractScalarInternal], so a bad path is not swallowed here.
+ @Nullable
+ private static DocumentContext parseJsonDocumentOrNull(@Nullable Object
jsonInput, boolean useBigDecimal) {
+ if (jsonInput == null) {
+ return null;
+ }
+ try {
+ return parseJsonDocument(jsonInput, useBigDecimal ?
PARSE_CONTEXT_WITH_BIG_DECIMAL : PARSE_CONTEXT);
+ } catch (Exception e) {
+ // Malformed JSON (e.g. a plain-text row) is treated as unresolved,
mirroring the transform which swallows
+ // per-row parse errors; the caller then applies the default or throws.
+ return null;
+ }
+ }
+
+ /// Transform casts the JsonPath result to `List`. A scalar or object is
therefore unresolved and
+ /// becomes an empty array. Do not wrap a non-list in a one-element array.
+ @Nullable
+ private static Object[] jsonArrayOrUnresolved(@Nullable Object value) {
+ if (value instanceof List) {
+ return ((List<?>) value).toArray();
+ }
+ return null;
+ }
+
+ private static Object coerceScalar(Object value, DataType dataType, boolean
isDefault) {
+ switch (dataType) {
+ case INT:
+ return coerceToInt(value, false);
+ case BOOLEAN:
+ return coerceToInt(value, true);
+ case LONG:
+ return coerceToLong(value, false);
+ case TIMESTAMP:
+ return coerceToLong(value, true);
+ case FLOAT:
+ return coerceToFloat(value);
+ case DOUBLE:
+ return coerceToDouble(value);
+ case BIG_DECIMAL:
+ return coerceToBigDecimal(value);
+ case STRING:
+ case JSON:
+ return coerceToString(value);
+ case BYTES:
+ return isDefault ? coerceDefaultToBytes(value) :
coerceExtractedToBytes(value);
+ default:
+ throw new
IllegalArgumentException(unsupportedResultsTypeMessage(dataType.name()));
+ }
+ }
+
+ private static Object coerceScalarArray(@Nullable Object[] array, DataType
dataType, @Nullable Object defaultValue,
+ boolean hasDefault) {
+ switch (dataType) {
+ case INT:
+ return toIntArray(array, defaultValue, hasDefault, false);
+ case BOOLEAN:
+ return toIntArray(array, defaultValue, hasDefault, true);
+ case LONG:
+ return toLongArray(array, defaultValue, hasDefault, false);
+ case TIMESTAMP:
+ return toLongArray(array, defaultValue, hasDefault, true);
+ case FLOAT:
+ return toFloatArray(array, defaultValue, hasDefault);
+ case DOUBLE:
+ return toDoubleArray(array, defaultValue, hasDefault);
+ case BIG_DECIMAL:
+ return toBigDecimalArray(array, defaultValue, hasDefault);
+ case STRING:
+ return toStringArray(array, defaultValue, hasDefault);
+ default:
+ throw new
IllegalArgumentException(unsupportedResultsTypeMessage(dataType.name() +
"_ARRAY"));
+ }
+ }
+
+ /// Resolve a single array element: pass through a non-null element,
substitute the default when the
+ /// element is `null` and a default was supplied, or throw when a null
element has no default.
+ /// An explicit SQL `NULL` default returns `null` so the caller can write a
type placeholder.
+ @Nullable
+ private static Object resolveArrayElement(@Nullable Object element,
@Nullable Object defaultValue,
+ boolean hasDefault) {
+ if (element != null) {
+ return element;
+ }
+ if (!hasDefault) {
+ throw new IllegalArgumentException(
+ "At least one of the resolved JSON arrays include nulls, which is
not supported in Pinot. "
+ + "Consider setting a default value as the fourth argument of
json_extract_scalar.");
+ }
+ return defaultValue;
+ }
+
+ private static int[] toIntArray(@Nullable Object[] array, @Nullable Object
defaultValue, boolean hasDefault,
+ boolean isBoolean) {
+ if (array == null) {
+ return new int[0];
+ }
+ int[] values = new int[array.length];
+ for (int i = 0; i < array.length; i++) {
+ Object resolved = resolveArrayElement(array[i], defaultValue,
hasDefault);
+ values[i] = resolved == null ? 0 : coerceToInt(resolved, isBoolean);
+ }
+ return values;
+ }
+
+ private static long[] toLongArray(@Nullable Object[] array, @Nullable Object
defaultValue, boolean hasDefault,
+ boolean isTimestamp) {
+ if (array == null) {
+ return new long[0];
+ }
+ long[] values = new long[array.length];
+ for (int i = 0; i < array.length; i++) {
+ Object resolved = resolveArrayElement(array[i], defaultValue,
hasDefault);
+ values[i] = resolved == null ? 0L : coerceToLong(resolved, isTimestamp);
+ }
+ return values;
+ }
+
+ private static float[] toFloatArray(@Nullable Object[] array, @Nullable
Object defaultValue, boolean hasDefault) {
+ if (array == null) {
+ return new float[0];
+ }
+ float[] values = new float[array.length];
+ for (int i = 0; i < array.length; i++) {
+ Object resolved = resolveArrayElement(array[i], defaultValue,
hasDefault);
+ values[i] = resolved == null ? 0f : coerceToFloat(resolved);
+ }
+ return values;
+ }
+
+ private static double[] toDoubleArray(@Nullable Object[] array, @Nullable
Object defaultValue, boolean hasDefault) {
+ if (array == null) {
+ return new double[0];
+ }
+ double[] values = new double[array.length];
+ for (int i = 0; i < array.length; i++) {
+ Object resolved = resolveArrayElement(array[i], defaultValue,
hasDefault);
+ values[i] = resolved == null ? 0d : coerceToDouble(resolved);
+ }
+ return values;
+ }
+
+ private static BigDecimal[] toBigDecimalArray(@Nullable Object[] array,
@Nullable Object defaultValue,
+ boolean hasDefault) {
+ if (array == null) {
+ return new BigDecimal[0];
+ }
+ BigDecimal[] values = new BigDecimal[array.length];
+ for (int i = 0; i < array.length; i++) {
+ Object resolved = resolveArrayElement(array[i], defaultValue,
hasDefault);
+ values[i] = resolved == null ? BigDecimal.ZERO :
coerceToBigDecimal(resolved);
+ }
+ return values;
+ }
+
+ private static String[] toStringArray(@Nullable Object[] array, @Nullable
Object defaultValue, boolean hasDefault) {
+ if (array == null) {
+ return new String[0];
+ }
+ String[] values = new String[array.length];
+ for (int i = 0; i < array.length; i++) {
+ Object resolved = resolveArrayElement(array[i], defaultValue,
hasDefault);
+ values[i] = resolved == null ? "" : coerceToString(resolved);
+ }
+ return values;
+ }
+
+ /// SQL BYTES defaults are hex literals or raw `byte[]`. Do not
Base64-decode them.
+ private static byte[] coerceDefaultToBytes(Object value) {
+ if (value instanceof byte[]) {
+ return (byte[]) value;
+ }
+ return BytesUtils.toBytes(value.toString());
+ }
+
+ /// Coerces a JsonPath result to stored `INT`. When `isBoolean` is true,
follows Pinot's numeric
+ /// BOOLEAN convention (any non-zero `Number` is true; `"true"` / `"TRUE"` /
`"1"` via
+ /// [BooleanUtils#toInt(String)]).
+ public static int coerceToInt(Object value, boolean isBoolean) {
+ if (isBoolean) {
+ if (value instanceof Boolean) {
+ return (Boolean) value ? 1 : 0;
+ }
+ // For BOOLEAN result, follow Pinot's numeric convention: any non-zero
number is true.
+ if (value instanceof Number) {
+ return ((Number) value).doubleValue() != 0 ? 1 : 0;
+ }
+ // String fallback: BooleanUtils.toInt accepts "true" / "TRUE" / "1".
+ return BooleanUtils.toInt(value.toString());
+ }
+ if (value instanceof Number) {
+ return ((Number) value).intValue();
+ }
+ if (value instanceof Boolean) {
+ return (Boolean) value ? 1 : 0;
+ }
+ return Integer.parseInt(value.toString());
+ }
+
+ /// Coerces a JsonPath result to stored `LONG`. When `isTimestamp` is true,
numeric values are
+ /// epoch millis and strings go through [TimestampUtils#toMillisSinceEpoch].
Otherwise string
+ /// numbers use [JsonNumberUtils#parseJsonLong] (exact decimal, truncate
toward zero, reject
+ /// overflow). Unquoted JSON numbers arrive as [Number]; floats should
already be [BigDecimal]
+ /// from the LONG parse context so values above `2^53` stay exact.
+ public static long coerceToLong(Object value, boolean isTimestamp) {
+ if (value instanceof Number) {
+ return longFromJsonNumber((Number) value);
+ }
+ if (isTimestamp) {
+ return TimestampUtils.toMillisSinceEpoch(value.toString());
+ }
+ if (value instanceof Boolean) {
+ return (Boolean) value ? 1L : 0L;
+ }
+ try {
+ return JsonNumberUtils.parseJsonLong(value.toString());
+ } catch (NumberFormatException e) {
+ throw new NumberFormatException("For input string: \"" + value + "\"");
+ }
+ }
+
+ /// Converts a JsonPath [Number] to long without saturating.
`Double.longValue()` and
+ /// `BigInteger.longValue()` wrap or clamp values outside the long range.
+ private static long longFromJsonNumber(Number number) {
+ if (number instanceof BigInteger) {
+ try {
+ return ((BigInteger) number).longValueExact();
+ } catch (ArithmeticException e) {
+ throw new NumberFormatException("For input string: \"" + number +
"\"");
+ }
+ }
+ if (number instanceof BigDecimal) {
+ try {
+ return ((BigDecimal) number).setScale(0,
RoundingMode.DOWN).longValueExact();
+ } catch (ArithmeticException e) {
+ throw new NumberFormatException("For input string: \"" + number +
"\"");
+ }
+ }
+ if (number instanceof Double || number instanceof Float) {
+ try {
+ return BigDecimal.valueOf(number.doubleValue()).setScale(0,
RoundingMode.DOWN).longValueExact();
Review Comment:
Verified in ae353ebf6b. Materialized Double/Float inputs use a finite,
range-checked cast, preserving 100000000000000032d and (double) Long.MIN_VALUE.
NaN, infinities, and 2^63 are rejected; the corresponding JsonFunctionsTest
regressions pass.
##########
pinot-common/src/main/java/org/apache/pinot/common/function/scalar/JsonFunctions.java:
##########
@@ -611,6 +634,409 @@ public static Object jsonExtractObject(@Nullable Object
object) {
return null;
}
+ /// Extract a scalar (or scalar-array) value from a JSON document and coerce
it to `resultsType`.
+ ///
+ /// Scalar-function counterpart of the `jsonExtractScalar` transform
(`JsonExtractScalarTransformFunction` in
+ /// pinot-core), so that `json_extract_scalar(...)` resolves in the
multi-stage engine and in ad-hoc scalar
+ /// contexts. `resultsType` is a Pinot [DataType] name, optionally suffixed
with `_ARRAY` for a multi-value
+ /// result. Supported types are
`INT/LONG/FLOAT/DOUBLE/BIG_DECIMAL/BOOLEAN/TIMESTAMP/STRING/JSON/BYTES` and
+ /// the `INT/LONG/FLOAT/DOUBLE/BIG_DECIMAL/BOOLEAN/TIMESTAMP/STRING` array
variants.
+ ///
+ /// The document may be a `String`, a UTF-8 encoded `byte[]` (BYTES columns)
or an already-parsed container.
+ /// Coercion mirrors the transform exactly: `BOOLEAN` is returned as its
stored `INT` (0/1), `TIMESTAMP` as
+ /// epoch millis (numeric values as-is, strings via ISO-8601), `BIG_DECIMAL`
/ `STRING` / `JSON` / `LONG`
+ /// use a BigDecimal-preserving parser. The 3-argument form throws on an
unresolved single-value path. The
+ /// 4-argument form returns `defaultValue` (including SQL `NULL`). A
multi-value path yields an empty
+ /// array when unresolved or when the resolved value is not a JSON array,
but a `null` element inside a
+ /// resolved array still throws unless a default is supplied. A malformed
JSON document is treated as
+ /// unresolved. An illegal JSONPath (for example `$[`) is rejected, matching
transform init.
+ @ScalarFunction
+ public static Object jsonExtractScalar(Object jsonInput, String jsonPath,
String resultsType) {
+ return jsonExtractScalarInternal(jsonInput, jsonPath, resultsType, null,
false);
+ }
+
+ /// See [#jsonExtractScalar(Object, String, String)]. `defaultValue` is
returned (coerced to `resultsType`)
+ /// when the path resolves to `null` or the document is malformed. An
explicit SQL `NULL` default returns
+ /// Java `null` rather than throwing.
+ @ScalarFunction(nullableParameters = true)
+ public static Object jsonExtractScalar(@Nullable Object jsonInput, String
jsonPath, String resultsType,
+ @Nullable Object defaultValue) {
+ return jsonExtractScalarInternal(jsonInput, jsonPath, resultsType,
defaultValue, true);
+ }
+
+ @Nullable
+ private static Object jsonExtractScalarInternal(@Nullable Object jsonInput,
String jsonPath, String resultsType,
+ @Nullable Object defaultValue, boolean hasDefault) {
+ String type = resultsType.toUpperCase();
+ boolean isSingleValue = !type.endsWith("_ARRAY");
+ DataType dataType;
+ try {
+ dataType = DataType.valueOf(isSingleValue ? type : type.substring(0,
type.length() - 6));
+ } catch (IllegalArgumentException e) {
+ throw new
IllegalArgumentException(unsupportedResultsTypeMessage(resultsType));
+ }
+ // BIG_DECIMAL / STRING / JSON / LONG must read floats as BigDecimal so
values above 2^53 stay exact.
+ boolean useBigDecimal = dataType == DataType.BIG_DECIMAL || dataType ==
DataType.STRING
+ || dataType == DataType.JSON || dataType == DataType.LONG;
+ // Compile the path before touching the document. `$[` must fail the
query, not become a default,
+ // matching JsonExtractScalarTransformFunction#init.
+ JsonPath compiledPath = JsonPathCache.INSTANCE.getOrCompute(jsonPath);
+ if (isSingleValue) {
+ Object value = readJsonPathValue(jsonInput, compiledPath, useBigDecimal);
+ if (value == null) {
+ if (!hasDefault) {
+ throw new IllegalArgumentException(
+ "Cannot resolve JSON path on some records. Consider setting a
default value.");
+ }
+ if (defaultValue == null) {
+ return null;
+ }
+ return coerceScalar(defaultValue, dataType, true);
+ }
+ return coerceScalar(value, dataType, false);
+ }
+ return coerceScalarArray(readJsonPathArray(jsonInput, compiledPath,
useBigDecimal), dataType, defaultValue,
+ hasDefault);
+ }
+
+ private static DocumentContext parseJsonDocument(Object jsonInput,
ParseContext parseContext) {
+ if (jsonInput instanceof String) {
+ return parseContext.parse((String) jsonInput);
+ }
+ if (jsonInput instanceof byte[]) {
+ // BYTES columns carry the raw UTF-8 document; parse(Object) would treat
the array as already parsed.
+ return parseContext.parseUtf8((byte[]) jsonInput);
+ }
+ return parseContext.parse(jsonInput);
+ }
+
+ @Nullable
+ private static Object readJsonPathValue(@Nullable Object jsonInput, JsonPath
jsonPath, boolean useBigDecimal) {
+ DocumentContext document = parseJsonDocumentOrNull(jsonInput,
useBigDecimal);
+ if (document == null) {
+ return null;
+ }
+ return document.read(jsonPath);
+ }
+
+ @Nullable
+ private static Object[] readJsonPathArray(@Nullable Object jsonInput,
JsonPath jsonPath, boolean useBigDecimal) {
+ DocumentContext document = parseJsonDocumentOrNull(jsonInput,
useBigDecimal);
+ if (document == null) {
+ return null;
+ }
+ return jsonArrayOrUnresolved(document.read(jsonPath));
+ }
+
+ /// Parses the document only. A bad document is unresolved (null). Path
compile already happened
+ /// in [jsonExtractScalarInternal], so a bad path is not swallowed here.
+ @Nullable
+ private static DocumentContext parseJsonDocumentOrNull(@Nullable Object
jsonInput, boolean useBigDecimal) {
+ if (jsonInput == null) {
+ return null;
+ }
+ try {
+ return parseJsonDocument(jsonInput, useBigDecimal ?
PARSE_CONTEXT_WITH_BIG_DECIMAL : PARSE_CONTEXT);
+ } catch (Exception e) {
+ // Malformed JSON (e.g. a plain-text row) is treated as unresolved,
mirroring the transform which swallows
+ // per-row parse errors; the caller then applies the default or throws.
+ return null;
+ }
+ }
+
+ /// Transform casts the JsonPath result to `List`. A scalar or object is
therefore unresolved and
+ /// becomes an empty array. Do not wrap a non-list in a one-element array.
+ @Nullable
+ private static Object[] jsonArrayOrUnresolved(@Nullable Object value) {
+ if (value instanceof List) {
Review Comment:
Verified in ae353ebf6b. Both parser contexts recognize Object[] containers,
and array-result handling accepts Object[] and List without wrapping
scalar/object results. The root-array, nested LONG traversal, and jsonPathArray
ingestion-chain regressions pass.
##########
pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunction.java:
##########
@@ -144,6 +144,17 @@ public
ArrayLiteralTransformFunction(List<ExpressionContext> literalContexts) {
}
_dataType = literalContexts.get(0).getLiteral().getType();
switch (_dataType) {
+ case BOOLEAN:
Review Comment:
Applied in 996a97fb30. BOOLEAN array literals now support long, float,
double, and string conversion. Added focused conversion coverage and a
QueryRunnerTest regression for arraySum(jsonExtractScalar('{"v":[true,false]}',
'$.v', 'BOOLEAN_ARRAY')) under both null-handling modes; it returns 1 and
passes.
##########
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java:
##########
@@ -88,13 +91,15 @@ private static LogicalProject
constructNewProject(LogicalProject oldProject, Log
List<RexNode> castedNewProjects = new ArrayList<>(numProjects);
boolean needCast = false;
for (int i = 0; i < numProjects; i++) {
- RexNode oldNode = oldProjects.get(i);
RexNode newNode = newProjects.get(i);
+ RelDataType expectedType =
oldProject.getRowType().getFieldList().get(i).getType();
// Need to cast the result to the original type if the literal type is
changed, e.g. VARCHAR literal is typed as
- // CHAR(STRING_LENGTH) in Calcite, but we need to cast it back to
VARCHAR.
- if (!oldNode.getType().equals(newNode.getType())) {
+ // CHAR(STRING_LENGTH) in Calcite, but we need to cast it back to
VARCHAR. Use the project's declared row type
+ // instead of the original expression type because a nullable
user-defined function can retain a non-null operand
+ // type while the validated projection is nullable.
+ if (!expectedType.equals(newNode.getType())) {
needCast = true;
- newNode = rexBuilder.makeCast(oldNode.getType(), newNode, true);
+ newNode = rexBuilder.makeAbstractCast(expectedType, newNode, false);
Review Comment:
Applied in 996a97fb30. Projection restoration first uses makeCast to
preserve ordinary literal normalization, and only falls back to
makeAbstractCast when the required type was not restored. The full
ResourceBasedQueryPlansTest suite (including the previously failing
literal-evaluation cases), QueryCompilationTest, and targeted runtime tests
pass.
##########
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/operands/FunctionOperand.java:
##########
@@ -121,6 +130,19 @@ public Object apply(List<Object> row) {
Object value = operand.apply(row);
_reusableOperandHolder[i] = value != null ?
operand.getResultType().toExternal(value) : null;
}
+ if (_replaceNullJsonOperands) {
+ if (_reusableOperandHolder[0] == null) {
+ ColumnDataType inputType = _operands.get(0).getResultType();
+ Object nullPlaceholder = inputType.getNullPlaceholder();
+ // An untyped SQL NULL has UNKNOWN type and therefore no generic
placeholder. The leaf transform reads every
+ // non-BYTES JSON input through transformToStringValuesSV(), whose
NULL literal value is the empty string.
+ _reusableOperandHolder[0] = nullPlaceholder != null ?
inputType.toExternal(nullPlaceholder) : "";
+ }
+ if (_reusableOperandHolder.length == 4 && _reusableOperandHolder[3] ==
null
+ && _reusableOperandHolder[2] != null) {
+ _reusableOperandHolder[3] =
getJsonNullDefault(_reusableOperandHolder[2].toString());
Review Comment:
Applied in 996a97fb30. FunctionOperand now prepares the NULL-default
replacement once in its constructor and reuses it during evaluation, removing
repeated uppercasing, suffix handling, dispatch, and empty-byte-array
allocation from the row path.
##########
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotEvaluateLiteralRule.java:
##########
@@ -322,4 +354,38 @@ private static Object convertResultValue(@Nullable Object
resultValue, RelDataTy
// TODO: Add more type handling
return resultValue;
}
+
+ /// Boxes a Java array so the folder can build a Calcite array literal.
`jsonExtractScalar`
Review Comment:
Addressed in 996a97fb30 while merging master. Literal-array conversion now
uses one Array.getLength/Array.get loop and calls convertResultValue directly
into the final list, preserving primitive/object-array support without the
intermediate boxing helper.
##########
pinot-common/src/main/java/org/apache/pinot/common/utils/JsonNumberUtils.java:
##########
@@ -0,0 +1,237 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.utils;
+
+import java.math.BigDecimal;
+
+
+/// Shared JSON-number parser used by `jsonExtractScalar` (scalar and
transform).
+///
+/// Accepts regular long syntax plus JSON numeric forms: `1E1` → `10`, `1.9` →
`1` (truncate toward
+/// zero), `1.123E1` → `11`. Dotted-exponent values are parsed as an exact
decimal ([BigDecimal]),
+/// truncated toward zero, then range-checked, so `9.223372036854775807E18` is
[Long#MAX_VALUE] and
+/// `9007199254740993.0E0` keeps the bit that a `double` would drop. Throws
[NumberFormatException]
+/// with `For input string: "<value>"` on overflow (`9223372036854775808`,
`2.0E19`), illegal
+/// exponent (`2E20`, `2E-1`), and other malformed input.
+///
+/// Thread-safe: no mutable state.
+public final class JsonNumberUtils {
+ private static final BigDecimal LONG_MIN_MINUS_ONE = new
BigDecimal("-9223372036854775809");
+ private static final BigDecimal LONG_MAX_PLUS_ONE = new
BigDecimal("9223372036854775808");
+
+ private static final long[] POWERS_OF_10 = new long[]{
+ 1L,
+ 10L,
+ 100L,
+ 1000L,
+ 10000L,
+ 100000L,
+ 1000000L,
+ 10000000L,
+ 100000000L,
+ 1000000000L,
+ 10000000000L,
+ 100000000000L,
+ 1000000000000L,
+ 10000000000000L,
+ 100000000000000L,
+ 1000000000000000L,
+ 10000000000000000L,
+ 100000000000000000L,
+ 1000000000000000000L,
+ };
+
+ private JsonNumberUtils() {
+ }
+
+ /// Parses a JSON numeric string to a long.
+ ///
+ /// @param cs char sequence to parse
+ /// @return parsed long value
+ /// @throws NumberFormatException if `cs` is null, empty, out of long range,
or not a JSON number
+ public static long parseJsonLong(CharSequence cs) {
+ if (cs == null) {
+ throw new NumberFormatException("Can't parse null string");
+ }
+
+ boolean negative = false;
+ int i = 0;
+ int len = cs.length();
+ long limit = -Long.MAX_VALUE;
+
+ if (len <= 0) {
+ throw formatException(cs);
+ }
+
+ boolean dotFound = false;
+ boolean exponentFound = false;
+
+ char firstChar = cs.charAt(0);
+ if (firstChar < '0') { // Possible leading "+" or "-"
+ if (firstChar == '-') {
+ negative = true;
+ limit = Long.MIN_VALUE;
+ } else if (firstChar != '+') {
+ throw formatException(cs);
+ }
+
+ if (len == 1) { // Cannot have lone "+" or "-"
+ throw formatException(cs);
+ }
+ i++;
+ }
+ long multmin = limit / 10;
+ long result = 0;
+ while (i < len) {
+ // Accumulating negatively avoids surprises near MAX_VALUE
+ char c = cs.charAt(i++);
+ if (c < '0' || c > '9' || result < multmin) {
+ if (c == '.') {
+ // ignore the rest of the integer digits
+ dotFound = true;
+ break;
+ } else if (c == 'e' || c == 'E') {
+ exponentFound = true;
+ break;
+ }
+ throw formatException(cs);
+ }
+
+ int digit = c - '0';
+ result *= 10;
+ if (result < limit + digit) {
+ throw formatException(cs);
+ }
+ result -= digit;
+ }
+
+ if (dotFound) {
+ // scan rest of the string to make sure it's only digits (or an exponent)
+ while (i < len) {
+ char c = cs.charAt(i++);
+ if (c < '0' || c > '9') {
+ if ((c | 32) == 'e') {
+ exponentFound = true;
+ break;
+ } else {
+ throw formatException(cs);
+ }
+ }
+ }
+ }
+
+ if (exponentFound) {
+ if (dotFound) {
+ // Do not go through double: 9.223372036854775807E18 is
Long.MAX_VALUE, but
+ // Double.parseDouble rounds it to 2^63, which is out of range.
+ return parseExactLong(cs);
+ }
+
+ long exp;
+ try {
+ exp = parseWholeLong(cs, i, len);
+ } catch (NumberFormatException nfe) {
+ throw new NumberFormatException("Wrong exponent");
+ }
+
+ if (exp < 0 || exp >= POWERS_OF_10.length) {
+ throw new NumberFormatException("Wrong exponent");
+ }
+
+ try {
+ return Math.multiplyExact(negative ? result : -result,
POWERS_OF_10[(int) exp]);
+ } catch (ArithmeticException e) {
+ throw formatException(cs);
+ }
+ }
+
+ return negative ? result : -result;
+ }
+
+ /// Parses a JSON numeric string as an exact decimal, truncates toward zero,
then requires a long.
+ private static long parseExactLong(CharSequence cs) {
+ try {
+ return truncateToLong(new BigDecimal(cs.toString()));
+ } catch (NumberFormatException e) {
+ throw formatException(cs);
+ }
+ }
+
+ /// Truncates an exact decimal toward zero and requires the result to fit in
a long.
+ ///
+ /// The comparisons happen before conversion so a value such as
`1E999999999` is rejected
+ /// without expanding its exponent into an enormous integer. Values strictly
+ /// between `Long.MIN_VALUE - 1` and `Long.MAX_VALUE + 1` truncate into the
long range.
+ public static long truncateToLong(BigDecimal value) {
+ if (value.compareTo(LONG_MIN_MINUS_ONE) <= 0 ||
value.compareTo(LONG_MAX_PLUS_ONE) >= 0) {
+ throw formatException(value.toString());
+ }
+ return value.longValue();
+ }
+
+ /// Parses `cs[start, end)` as a whole long (sign allowed). Used for the
exponent field.
+ private static long parseWholeLong(CharSequence cs, int start, int end) {
Review Comment:
Applied in 996a97fb30. parseWholeLong now validates ASCII digits and
delegates to Long.parseLong(cs, start, end, 10). Leading-sign behavior and the
caller's Wrong exponent translation are preserved, with exponent edge-case
regression coverage.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
