This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 14c377d Fix JSONPath cache inefficient issue (#7409)
14c377d is described below
commit 14c377d3edb0e993925d0e271f17e9dc81b2e4ca
Author: wentao jin <[email protected]>
AuthorDate: Wed Nov 3 15:27:20 2021 +0800
Fix JSONPath cache inefficient issue (#7409)
Also fix the issue of setting different default configurations
---
.../pinot/common/function/JsonPathCache.java | 47 ++++++++++++++++
.../common/function/scalar/JsonFunctions.java | 42 ++++----------
.../function/JsonExtractKeyTransformFunction.java | 64 +++++++---------------
.../JsonExtractScalarTransformFunction.java | 49 +++++------------
.../tests/JsonPathClusterIntegrationTest.java | 10 ++++
5 files changed, 103 insertions(+), 109 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/JsonPathCache.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/JsonPathCache.java
new file mode 100644
index 0000000..3f012f4
--- /dev/null
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/JsonPathCache.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.function;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.CacheBuilder;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.spi.cache.Cache;
+
+
+public class JsonPathCache implements Cache {
+ private static final long DEFAULT_CACHE_MAXIMUM_SIZE = 10000;
+
+ private final com.google.common.cache.Cache<String, JsonPath> _jsonPathCache
=
+
CacheBuilder.newBuilder().maximumSize(DEFAULT_CACHE_MAXIMUM_SIZE).build();
+
+ @Override
+ public JsonPath get(String key) {
+ return _jsonPathCache.getIfPresent(key);
+ }
+
+ @Override
+ public void put(String key, JsonPath value) {
+ _jsonPathCache.put(key, value);
+ }
+
+ @VisibleForTesting
+ public long size() {
+ return _jsonPathCache.size();
+ }
+}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/JsonFunctions.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/JsonFunctions.java
index acb3a37..0285f7b 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/JsonFunctions.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/JsonFunctions.java
@@ -21,19 +21,16 @@ package org.apache.pinot.common.function.scalar;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.annotations.VisibleForTesting;
import com.jayway.jsonpath.Configuration;
-import com.jayway.jsonpath.Option;
+import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.ParseContext;
import com.jayway.jsonpath.Predicate;
-import com.jayway.jsonpath.internal.ParseContextImpl;
+import com.jayway.jsonpath.spi.cache.CacheProvider;
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
-import com.jayway.jsonpath.spi.json.JsonProvider;
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
-import com.jayway.jsonpath.spi.mapper.MappingProvider;
import java.util.Arrays;
-import java.util.EnumSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+import org.apache.pinot.common.function.JsonPathCache;
import org.apache.pinot.spi.annotations.ScalarFunction;
import org.apache.pinot.spi.utils.JsonUtils;
@@ -50,32 +47,17 @@ import org.apache.pinot.spi.utils.JsonUtils;
* </code>
*/
public class JsonFunctions {
- private static final ParseContext PARSE_CONTEXT;
- private static final Predicate[] NO_PREDICATES = new Predicate[0];
- static {
- Configuration.setDefaults(new Configuration.Defaults() {
- private final JsonProvider _jsonProvider = new
ArrayAwareJacksonJsonProvider();
- private final MappingProvider _mappingProvider = new
JacksonMappingProvider();
-
- @Override
- public JsonProvider jsonProvider() {
- return _jsonProvider;
- }
-
- @Override
- public MappingProvider mappingProvider() {
- return _mappingProvider;
- }
-
- @Override
- public Set<Option> options() {
- return EnumSet.noneOf(Option.class);
- }
- });
- PARSE_CONTEXT = new ParseContextImpl(Configuration.defaultConfiguration());
+ private JsonFunctions() {
}
- private JsonFunctions() {
+ private static final Predicate[] NO_PREDICATES = new Predicate[0];
+ private static final ParseContext PARSE_CONTEXT = JsonPath.using(
+ new Configuration.ConfigurationBuilder().jsonProvider(new
ArrayAwareJacksonJsonProvider())
+ .mappingProvider(new JacksonMappingProvider()).build());
+
+ static {
+ // Set the JsonPath cache before the cache is accessed
+ CacheProvider.setCache(new JsonPathCache());
}
/**
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java
index 7f95e1a..5798910 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java
@@ -18,20 +18,17 @@
*/
package org.apache.pinot.core.operator.transform.function;
-import com.google.common.collect.ImmutableSet;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
+import com.jayway.jsonpath.ParseContext;
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
-import com.jayway.jsonpath.spi.json.JsonProvider;
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
-import com.jayway.jsonpath.spi.mapper.MappingProvider;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import javax.annotation.Nonnull;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
+import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -48,12 +45,15 @@ import org.apache.pinot.segment.spi.datasource.DataSource;
*
*/
public class JsonExtractKeyTransformFunction extends BaseTransformFunction {
-
public static final String FUNCTION_NAME = "jsonExtractKey";
- private static final Configuration JSON_PATH_KEY_CONFIG =
- Configuration.builder().options(Option.AS_PATH_LIST).build();
+
+ private static final ParseContext JSON_PARSER_CONTEXT = JsonPath.using(
+ new Configuration.ConfigurationBuilder().jsonProvider(new
JacksonJsonProvider())
+ .mappingProvider(new
JacksonMappingProvider()).options(Option.AS_PATH_LIST,
Option.SUPPRESS_EXCEPTIONS)
+ .build());
+
private TransformFunction _jsonFieldTransformFunction;
- private String _jsonPath;
+ private JsonPath _jsonPath;
@Override
public String getName() {
@@ -61,7 +61,7 @@ public class JsonExtractKeyTransformFunction extends
BaseTransformFunction {
}
@Override
- public void init(@Nonnull List<TransformFunction> arguments, @Nonnull
Map<String, DataSource> dataSourceMap) {
+ public void init(List<TransformFunction> arguments, Map<String, DataSource>
dataSourceMap) {
// Check that there are exactly 2 arguments
if (arguments.size() != 2) {
throw new IllegalArgumentException(
@@ -75,7 +75,7 @@ public class JsonExtractKeyTransformFunction extends
BaseTransformFunction {
+ "function");
}
_jsonFieldTransformFunction = firstArgument;
- _jsonPath = ((LiteralTransformFunction) arguments.get(1)).getLiteral();
+ _jsonPath = JsonPath.compile(((LiteralTransformFunction)
arguments.get(1)).getLiteral());
}
@Override
@@ -84,39 +84,17 @@ public class JsonExtractKeyTransformFunction extends
BaseTransformFunction {
}
@Override
- public String[][] transformToStringValuesMV(@Nonnull ProjectionBlock
projectionBlock) {
- final String[] stringValuesMV =
_jsonFieldTransformFunction.transformToStringValuesSV(projectionBlock);
- final String[][] results = new String[projectionBlock.getNumDocs()][];
- for (int i = 0; i < projectionBlock.getNumDocs(); i++) {
- final List<String> stringVals =
JsonPath.using(JSON_PATH_KEY_CONFIG).parse(stringValuesMV[i]).read(_jsonPath);
- results[i] = new String[stringVals.size()];
- for (int j = 0; j < stringVals.size(); j++) {
- results[i][j] = stringVals.get(j);
- }
+ public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock)
{
+ if (_stringValuesMV == null) {
+ _stringValuesMV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
}
- return results;
- }
-
- static {
- Configuration.setDefaults(new Configuration.Defaults() {
- private final JsonProvider _jsonProvider = new JacksonJsonProvider();
- private final MappingProvider _mappingProvider = new
JacksonMappingProvider();
-
- @Override
- public JsonProvider jsonProvider() {
- return _jsonProvider;
- }
-
- @Override
- public MappingProvider mappingProvider() {
- return _mappingProvider;
- }
-
- @Override
- public Set<Option> options() {
- return ImmutableSet.of(Option.SUPPRESS_EXCEPTIONS);
- }
- });
+ String[] jsonStrings =
_jsonFieldTransformFunction.transformToStringValuesSV(projectionBlock);
+ int numDocs = projectionBlock.getNumDocs();
+ for (int i = 0; i < numDocs; i++) {
+ List<String> values =
JSON_PARSER_CONTEXT.parse(jsonStrings[i]).read(_jsonPath);
+ _stringValuesMV[i] = values.toArray(new String[0]);
+ }
+ return _stringValuesMV;
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
index 83cab12..38e6ac2 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
@@ -18,18 +18,14 @@
*/
package org.apache.pinot.core.operator.transform.function;
-import com.google.common.collect.ImmutableSet;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
import com.jayway.jsonpath.ParseContext;
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
-import com.jayway.jsonpath.spi.json.JsonProvider;
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
-import com.jayway.jsonpath.spi.mapper.MappingProvider;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.core.plan.DocIdSetPlanNode;
@@ -54,11 +50,14 @@ import org.apache.pinot.spi.utils.JsonUtils;
*/
public class JsonExtractScalarTransformFunction extends BaseTransformFunction {
public static final String FUNCTION_NAME = "jsonExtractScalar";
- private static final ParseContext JSON_PARSER_CONTEXT =
-
JsonPath.using(Configuration.defaultConfiguration().addOptions(Option.SUPPRESS_EXCEPTIONS));
+
+ private static final ParseContext JSON_PARSER_CONTEXT = JsonPath.using(
+ new Configuration.ConfigurationBuilder().jsonProvider(new
JacksonJsonProvider())
+ .mappingProvider(new
JacksonMappingProvider()).options(Option.SUPPRESS_EXCEPTIONS).build());
private TransformFunction _jsonFieldTransformFunction;
- private String _jsonPath;
+ private String _jsonPathString;
+ private JsonPath _jsonPath;
private Object _defaultValue = null;
private TransformResultMetadata _resultMetadata;
@@ -83,7 +82,8 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
+ "function");
}
_jsonFieldTransformFunction = firstArgument;
- _jsonPath = ((LiteralTransformFunction) arguments.get(1)).getLiteral();
+ _jsonPathString = ((LiteralTransformFunction)
arguments.get(1)).getLiteral();
+ _jsonPath = JsonPath.compile(_jsonPathString);
String resultsType = ((LiteralTransformFunction)
arguments.get(2)).getLiteral().toUpperCase();
boolean isSingleValue = !resultsType.endsWith("_ARRAY");
try {
@@ -126,7 +126,7 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
continue;
}
throw new RuntimeException(
- String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPath, jsonStrings[i]));
+ String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPathString, jsonStrings[i]));
}
if (result instanceof Number) {
_intValuesSV[i] = ((Number) result).intValue();
@@ -157,7 +157,7 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
continue;
}
throw new RuntimeException(
- String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPath, jsonStrings[i]));
+ String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPathString, jsonStrings[i]));
}
if (result instanceof Number) {
_longValuesSV[i] = ((Number) result).longValue();
@@ -189,7 +189,7 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
continue;
}
throw new RuntimeException(
- String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPath, jsonStrings[i]));
+ String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPathString, jsonStrings[i]));
}
if (result instanceof Number) {
_floatValuesSV[i] = ((Number) result).floatValue();
@@ -220,7 +220,7 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
continue;
}
throw new RuntimeException(
- String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPath, jsonStrings[i]));
+ String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPathString, jsonStrings[i]));
}
if (result instanceof Number) {
_doubleValuesSV[i] = ((Number) result).doubleValue();
@@ -251,7 +251,7 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
continue;
}
throw new RuntimeException(
- String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPath, jsonStrings[i]));
+ String.format("Illegal Json Path: [%s], when reading [%s]",
_jsonPathString, jsonStrings[i]));
}
if (result instanceof String) {
_stringValuesSV[i] = (String) result;
@@ -401,27 +401,4 @@ public class JsonExtractScalarTransformFunction extends
BaseTransformFunction {
}
return _stringValuesMV;
}
-
- static {
- Configuration.setDefaults(new Configuration.Defaults() {
-
- private final JsonProvider _jsonProvider = new JacksonJsonProvider();
- private final MappingProvider _mappingProvider = new
JacksonMappingProvider();
-
- @Override
- public JsonProvider jsonProvider() {
- return _jsonProvider;
- }
-
- @Override
- public MappingProvider mappingProvider() {
- return _mappingProvider;
- }
-
- @Override
- public Set<Option> options() {
- return ImmutableSet.of(Option.SUPPRESS_EXCEPTIONS);
- }
- });
- }
}
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/JsonPathClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/JsonPathClusterIntegrationTest.java
index 9121bb2..a4cf201 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/JsonPathClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/JsonPathClusterIntegrationTest.java
@@ -22,6 +22,8 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
+import com.jayway.jsonpath.spi.cache.Cache;
+import com.jayway.jsonpath.spi.cache.CacheProvider;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
@@ -35,6 +37,7 @@ import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.function.JsonPathCache;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
@@ -471,6 +474,13 @@ public class JsonPathClusterIntegrationTest extends
BaseClusterIntegrationTest {
Assert.assertEquals(pinotResponse.get("totalDocs").asInt(), 0);
}
+ @Test
+ public void testJsonPathCache() {
+ Cache cache = CacheProvider.getCache();
+ Assert.assertTrue(cache instanceof JsonPathCache);
+ Assert.assertTrue(((JsonPathCache) cache).size() > 0);
+ }
+
@AfterClass
public void tearDown()
throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]