This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch specific-linear in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit cbf400a0eb93bb8cc8506a025c2a822b8b7f5871 Author: Wu Sheng <wu.sh...@foxmail.com> AuthorDate: Wed Jan 29 15:35:50 2020 +0800 Refactor the storage to support `getSubsetOfMultipleLinearIntValues` --- .../oap/server/core/query/MetricQueryService.java | 17 +++++++++++-- .../core/storage/query/IMetricsQueryDAO.java | 18 +++++++++----- .../oap/query/graphql/resolver/MetricQuery.java | 28 +++++++++++++++++----- .../src/main/resources/query-protocol | 2 +- .../elasticsearch/query/MetricsQueryEsDAO.java | 9 +++---- .../plugin/jdbc/h2/dao/H2MetricsQueryDAO.java | 9 +++---- 6 files changed, 60 insertions(+), 23 deletions(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricQueryService.java index 8636edc..d2cb9af 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricQueryService.java @@ -100,6 +100,19 @@ public class MetricQueryService implements Service { final Downsampling downsampling, final long startTB, final long endTB) throws IOException { + List<Integer> linearIndex = new ArrayList<>(numOfLinear); + for (int i = 0; i < numOfLinear; i++) { + linearIndex.add(i); + } + + return getSubsetOfMultipleLinearIntValues(indName, id, linearIndex, downsampling, startTB, endTB); + } + + public List<IntValues> getSubsetOfMultipleLinearIntValues(final String indName, final String id, + final List<Integer> linearIndex, + final Downsampling downsampling, + final long startTB, + final long endTB) throws IOException { List<DurationPoint> durationPoints = DurationUtils.INSTANCE.getDurationPoints(downsampling, startTB, endTB); List<String> ids = new ArrayList<>(); if (StringUtil.isEmpty(id)) { @@ -108,9 +121,9 @@ public class MetricQueryService implements Service { durationPoints.forEach(durationPoint -> ids.add(durationPoint.getPoint() + Const.ID_SPLIT + id)); } - IntValues[] multipleLinearIntValues = getMetricQueryDAO().getMultipleLinearIntValues(indName, downsampling, ids, numOfLinear, ValueColumnIds.INSTANCE.getValueCName(indName)); + IntValues[] multipleLinearIntValues = getMetricQueryDAO().getMultipleLinearIntValues(indName, downsampling, ids, linearIndex, ValueColumnIds.INSTANCE.getValueCName(indName)); - List<IntValues> response = new ArrayList<>(numOfLinear); + List<IntValues> response = new ArrayList<>(linearIndex.size()); Collections.addAll(response, multipleLinearIntValues); return response; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java index c10aafa..512ac58 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java @@ -21,8 +21,10 @@ package org.apache.skywalking.oap.server.core.storage.query; import java.io.IOException; import java.util.List; import org.apache.skywalking.oap.server.core.analysis.Downsampling; -import org.apache.skywalking.oap.server.core.query.entity.*; -import org.apache.skywalking.oap.server.core.query.sql.*; +import org.apache.skywalking.oap.server.core.query.entity.IntValues; +import org.apache.skywalking.oap.server.core.query.entity.Thermodynamic; +import org.apache.skywalking.oap.server.core.query.sql.Function; +import org.apache.skywalking.oap.server.core.query.sql.Where; import org.apache.skywalking.oap.server.core.storage.DAO; /** @@ -30,11 +32,15 @@ import org.apache.skywalking.oap.server.core.storage.DAO; */ public interface IMetricsQueryDAO extends DAO { - IntValues getValues(String indName, Downsampling downsampling, long startTB, long endTB, Where where, String valueCName, Function function) throws IOException; + IntValues getValues(String indName, Downsampling downsampling, long startTB, long endTB, Where where, + String valueCName, Function function) throws IOException; - IntValues getLinearIntValues(String indName, Downsampling downsampling, List<String> ids, String valueCName) throws IOException; + IntValues getLinearIntValues(String indName, Downsampling downsampling, List<String> ids, + String valueCName) throws IOException; - IntValues[] getMultipleLinearIntValues(String indName, Downsampling downsampling, List<String> ids, int numOfLinear, String valueCName) throws IOException; + IntValues[] getMultipleLinearIntValues(String indName, Downsampling downsampling, List<String> ids, + List<Integer> linearIndex, String valueCName) throws IOException; - Thermodynamic getThermodynamic(String indName, Downsampling downsampling, List<String> ids, String valueCName) throws IOException; + Thermodynamic getThermodynamic(String indName, Downsampling downsampling, List<String> ids, + String valueCName) throws IOException; } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricQuery.java index b4f948f..5a3a9f1 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricQuery.java @@ -22,10 +22,15 @@ import com.coxautodev.graphql.tools.GraphQLQueryResolver; import java.io.IOException; import java.text.ParseException; import java.util.List; -import org.apache.skywalking.oap.query.graphql.type.*; +import org.apache.skywalking.oap.query.graphql.type.BatchMetricConditions; +import org.apache.skywalking.oap.query.graphql.type.Duration; +import org.apache.skywalking.oap.query.graphql.type.MetricCondition; import org.apache.skywalking.oap.server.core.CoreModule; -import org.apache.skywalking.oap.server.core.query.*; -import org.apache.skywalking.oap.server.core.query.entity.*; +import org.apache.skywalking.oap.server.core.query.DurationUtils; +import org.apache.skywalking.oap.server.core.query.MetricQueryService; +import org.apache.skywalking.oap.server.core.query.StepToDownsampling; +import org.apache.skywalking.oap.server.core.query.entity.IntValues; +import org.apache.skywalking.oap.server.core.query.entity.Thermodynamic; import org.apache.skywalking.oap.server.library.module.ModuleManager; /** @@ -54,21 +59,32 @@ public class MetricQuery implements GraphQLQueryResolver { return getMetricQueryService().getValues(metrics.getName(), metrics.getIds(), StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket); } - public IntValues getLinearIntValues(final MetricCondition metrics, final Duration duration) throws IOException, ParseException { + public IntValues getLinearIntValues(final MetricCondition metrics, + final Duration duration) throws IOException, ParseException { long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); return getMetricQueryService().getLinearIntValues(metrics.getName(), metrics.getId(), StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket); } - public List<IntValues> getMultipleLinearIntValues(final MetricCondition metrics, final int numOfLinear, final Duration duration) throws IOException, ParseException { + public List<IntValues> getMultipleLinearIntValues(final MetricCondition metrics, final int numOfLinear, + final Duration duration) throws IOException, ParseException { long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); return getMetricQueryService().getMultipleLinearIntValues(metrics.getName(), metrics.getId(), numOfLinear, StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket); } - public Thermodynamic getThermodynamic(final MetricCondition metrics, final Duration duration) throws IOException, ParseException { + public List<IntValues> getSubsetOfMultipleLinearIntValues(final MetricCondition metrics, + final List<Integer> linearIndex, final Duration duration) throws IOException, ParseException { + long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); + long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); + + return getMetricQueryService().getSubsetOfMultipleLinearIntValues(metrics.getName(), metrics.getId(), linearIndex, StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket); + } + + public Thermodynamic getThermodynamic(final MetricCondition metrics, + final Duration duration) throws IOException, ParseException { long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 249adde..03ed785 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 249addeaaf524c0dd990444e5f4bcaf355ce8e01 +Subproject commit 03ed7858ea05ade81b9ceaa3abe468b422ce8110 diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java index 7325fd4..5ce7df7 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java @@ -135,13 +135,13 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO { } @Override public IntValues[] getMultipleLinearIntValues(String indName, Downsampling downsampling, - List<String> ids, int numOfLinear, String valueCName) throws IOException { + List<String> ids, List<Integer> linearIndex, String valueCName) throws IOException { String indexName = ModelName.build(downsampling, indName); SearchResponse response = getClient().ids(indexName, ids.toArray(new String[0])); Map<String, Map<String, Object>> idMap = toMap(response); - IntValues[] intValuesArray = new IntValues[numOfLinear]; + IntValues[] intValuesArray = new IntValues[linearIndex.size()]; for (int i = 0; i < intValuesArray.length; i++) { intValuesArray[i] = new IntValues(); } @@ -159,8 +159,9 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO { IntKeyLongValueHashMap multipleValues = new IntKeyLongValueHashMap(5); multipleValues.toObject((String)source.getOrDefault(valueCName, "")); - for (int i = 0; i < intValuesArray.length; i++) { - intValuesArray[i].getLast().setValue(multipleValues.get(i).getValue()); + for (int i = 0; i < linearIndex.size(); i++) { + Integer index = linearIndex.get(i); + intValuesArray[i].getLast().setValue(multipleValues.get(index).getValue()); } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java index 0e24925..8b37015 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java @@ -145,7 +145,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO @Override public IntValues[] getMultipleLinearIntValues(String indName, Downsampling downsampling, List<String> ids, - int numOfLinear, + final List<Integer> linearIndex, String valueCName) throws IOException { String tableName = ModelName.build(downsampling, indName); @@ -157,7 +157,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO idValues.append("'").append(ids.get(valueIdx)).append("'"); } - IntValues[] intValuesArray = new IntValues[numOfLinear]; + IntValues[] intValuesArray = new IntValues[linearIndex.size()]; for (int i = 0; i < intValuesArray.length; i++) { intValuesArray[i] = new IntValues(); } @@ -170,10 +170,11 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO IntKeyLongValueHashMap multipleValues = new IntKeyLongValueHashMap(5); multipleValues.toObject(resultSet.getString(valueCName)); - for (int i = 0; i < intValuesArray.length; i++) { + for (int i = 0; i < linearIndex.size(); i++) { + Integer index = linearIndex.get(i); KVInt kv = new KVInt(); kv.setId(id); - kv.setValue(multipleValues.get(i).getValue()); + kv.setValue(multipleValues.get(index).getValue()); intValuesArray[i].addKVInt(kv); } }