This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new e50a46b723 Change the string field in Elasticsearch storage from
**keyword** type to *text* if they appoint size more than `32766` (#11731)
e50a46b723 is described below
commit e50a46b7235758f82c7241c74c2395807543a0c7
Author: mrproliu <[email protected]>
AuthorDate: Sun Jan 7 18:13:54 2024 +0800
Change the string field in Elasticsearch storage from **keyword** type to
*text* if they appoint size more than `32766` (#11731)
---
docs/en/changes/changes.md | 2 ++
.../elasticsearch/base/ColumnTypeEsMapping.java | 6 ++++--
.../elasticsearch/base/StorageEsInstaller.java | 6 ++++--
.../ElasticSearchColumnTypeMappingTestCase.java | 24 +++++++++++++---------
4 files changed, 24 insertions(+), 14 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index a34901c978..5e37c09535 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -13,6 +13,8 @@
* Set up the length of source and dest IDs in relation entities of service,
instance, endpoint, and process to 250(was
200).
* Support build Service/Instance Hierarchy and query.
+* Change the string field in Elasticsearch storage from **keyword** type to
**text** type if it set more than `32766` length.
+* [Break Change] Change the configuration field of `ui_template` and `ui_menu`
in Elasticsearch storage from **keyword** type to **text**.
#### UI
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ColumnTypeEsMapping.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ColumnTypeEsMapping.java
index e0d2af9540..914154ea40 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ColumnTypeEsMapping.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ColumnTypeEsMapping.java
@@ -28,13 +28,15 @@ import
org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObje
public class ColumnTypeEsMapping {
- public String transform(Class<?> type, Type genericType, final
ElasticSearchExtension elasticSearchExtension) {
+ public String transform(Class<?> type, Type genericType, int length,
boolean storageOnly, final ElasticSearchExtension elasticSearchExtension) {
if (Integer.class.equals(type) || int.class.equals(type) ||
Layer.class.equals(type)) {
return "integer";
} else if (Long.class.equals(type) || long.class.equals(type)) {
return "long";
} else if (Double.class.equals(type) || double.class.equals(type)) {
return "double";
+ } else if (String.class.equals(type) && storageOnly && length > 32766)
{ // 32766 is the max length of a keywords field in ES
+ return "text";
} else if (String.class.equals(type) ||
elasticSearchExtension.isKeyword()) {
return "keyword";
} else if (StorageDataComplexObject.class.isAssignableFrom(type)) {
@@ -45,7 +47,7 @@ public class ColumnTypeEsMapping {
return "text";
} else if (List.class.isAssignableFrom(type)) {
final Type elementType = ((ParameterizedType)
genericType).getActualTypeArguments()[0];
- return transform((Class<?>) elementType, elementType,
elasticSearchExtension);
+ return transform((Class<?>) elementType, elementType, length,
storageOnly, elasticSearchExtension);
} else {
throw new IllegalArgumentException("Unsupported data type: " +
type.getName());
}
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
index 39cc09e82e..050941886a 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
@@ -255,7 +255,7 @@ public class StorageEsInstaller extends ModelInstaller {
if (!CollectionUtils.isEmpty(specificSettings)) {
indexSettings.putAll(specificSettings);
}
-
+
return setting;
}
@@ -308,7 +308,9 @@ public class StorageEsInstaller extends ModelInstaller {
Map<String, Object> properties = new HashMap<>();
Mappings.Source source = new Mappings.Source();
for (ModelColumn columnDefine : model.getColumns()) {
- final String type =
columnTypeEsMapping.transform(columnDefine.getType(),
columnDefine.getGenericType(), columnDefine.getElasticSearchExtension());
+ final String type =
columnTypeEsMapping.transform(columnDefine.getType(),
columnDefine.getGenericType(),
+ columnDefine.getLength(), columnDefine.isStorageOnly(),
+ columnDefine.getElasticSearchExtension());
String columnName = columnDefine.getColumnName().getName();
String legacyName =
columnDefine.getElasticSearchExtension().getLegacyColumnName();
if (config.isLogicSharding() &&
!Strings.isNullOrEmpty(legacyName)) {
diff --git
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ElasticSearchColumnTypeMappingTestCase.java
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ElasticSearchColumnTypeMappingTestCase.java
index 550683b231..26ba769b20 100644
---
a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ElasticSearchColumnTypeMappingTestCase.java
+++
b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ElasticSearchColumnTypeMappingTestCase.java
@@ -32,26 +32,30 @@ public class ElasticSearchColumnTypeMappingTestCase {
@Test
public void test() throws NoSuchFieldException {
ColumnTypeEsMapping mapping = new ColumnTypeEsMapping();
+ int defaultLength = 200;
+ boolean defaultStorageOnly = false;
- Assertions.assertEquals("integer", mapping.transform(int.class,
int.class, null));
- Assertions.assertEquals("integer", mapping.transform(Integer.class,
Integer.class, null));
+ Assertions.assertEquals("integer", mapping.transform(int.class,
int.class, defaultLength, defaultStorageOnly, null));
+ Assertions.assertEquals("integer", mapping.transform(Integer.class,
Integer.class, defaultLength, defaultStorageOnly, null));
- Assertions.assertEquals("long", mapping.transform(long.class,
long.class, null));
- Assertions.assertEquals("long", mapping.transform(Long.class,
Long.class, null));
+ Assertions.assertEquals("long", mapping.transform(long.class,
long.class, defaultLength, defaultStorageOnly, null));
+ Assertions.assertEquals("long", mapping.transform(Long.class,
Long.class, defaultLength, defaultStorageOnly, null));
- Assertions.assertEquals("double", mapping.transform(double.class,
double.class, null));
- Assertions.assertEquals("double", mapping.transform(Double.class,
Double.class, null));
+ Assertions.assertEquals("double", mapping.transform(double.class,
double.class, defaultLength, defaultStorageOnly, null));
+ Assertions.assertEquals("double", mapping.transform(Double.class,
Double.class, defaultLength, defaultStorageOnly, null));
- Assertions.assertEquals("keyword", mapping.transform(String.class,
String.class, null));
+ Assertions.assertEquals("keyword", mapping.transform(String.class,
String.class, defaultLength, defaultStorageOnly, null));
+ Assertions.assertEquals("keyword", mapping.transform(String.class,
String.class, 100_000, defaultStorageOnly, null));
+ Assertions.assertEquals("text", mapping.transform(String.class,
String.class, 100_000, true, null));
final Type listFieldType =
this.getClass().getField("a").getGenericType();
- Assertions.assertEquals("keyword", mapping.transform(List.class,
listFieldType,
+ Assertions.assertEquals("keyword", mapping.transform(List.class,
listFieldType, defaultLength, defaultStorageOnly,
new
ElasticSearchExtension(null, null, false, false)
));
- Assertions.assertEquals("keyword", mapping.transform(IntList.class,
int.class,
+ Assertions.assertEquals("keyword", mapping.transform(IntList.class,
int.class, defaultLength, defaultStorageOnly,
new
ElasticSearchExtension(null, null, true, false)));
- Assertions.assertEquals("text", mapping.transform(IntList.class,
int.class,
+ Assertions.assertEquals("text", mapping.transform(IntList.class,
int.class, defaultLength, defaultStorageOnly,
new
ElasticSearchExtension(null, null, false, false)));
}
}
