This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch scroller-size in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 82c0dd4fb4fa96673593b3be56e827b84f790f2a Author: kezhenxu94 <kezhenx...@apache.org> AuthorDate: Thu Nov 9 06:50:34 2023 +0000 Fix `limit` doesn't work for `findEndpoint` API in ES storage --- docs/en/changes/changes.md | 1 + .../elasticsearch/ElasticSearchScroller.java | 8 +++---- .../elasticsearch/bulk/ElasticSearchIT.java | 27 ++++++++++++++++++++++ .../elasticsearch/query/MetadataQueryEsDAO.java | 2 +- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index e2e7f614c1..34019bd98c 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -33,6 +33,7 @@ * Add a component ID for Fiber (ID=5021). * BanyanDBStorageClient: Add `define(Property property, PropertyStore.Strategy strategy)` API. * Support GraalVM native-image (Experimental). +* Fix `limit` doesn't work for `findEndpoint` API in ES storage. #### UI diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchScroller.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchScroller.java index f414f4c502..62504e83fc 100644 --- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchScroller.java +++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchScroller.java @@ -61,12 +61,12 @@ public class ElasticSearchScroller<T> { } for (final var searchHit : response.getHits()) { results.add(resultConverter.apply(searchHit)); + if (queryMaxSize > 0 && results.size() >= queryMaxSize) { + return results; + } } if (search.getSize() != null && response.getHits().getHits().size() < search.getSize()) { - break; - } - if (queryMaxSize > 0 && results.size() >= queryMaxSize) { - break; + return results; } response = client.scroll(SCROLL_CONTEXT_RETENTION, scrollId); } diff --git a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ElasticSearchIT.java b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ElasticSearchIT.java index cdaca0a2b4..fa6fcb43e9 100644 --- a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ElasticSearchIT.java +++ b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/ElasticSearchIT.java @@ -26,8 +26,10 @@ import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder import org.apache.skywalking.library.elasticsearch.response.Document; import org.apache.skywalking.library.elasticsearch.response.Index; import org.apache.skywalking.library.elasticsearch.response.Mappings; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchScroller; import org.apache.skywalking.oap.server.library.client.elasticsearch.IndexRequestWrapper; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.junit.jupiter.api.Assertions; @@ -41,6 +43,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.function.Function; @Slf4j @@ -171,6 +174,30 @@ public class ElasticSearchIT { .get("message")); client.deleteById(indexName, id); Assertions.assertFalse(client.existDoc(indexName, id)); + + for (int i = 0; i < 100; i++) { + builder = ImmutableMap.<String, Object>builder() + .put("user", "sw") + .put("post_date", "2009-11-15T14:12:12") + .put("message", "trying out Elasticsearch") + .build(); + + indexName = "test_scroller"; + client.forceInsert(indexName, UUID.randomUUID().toString(), builder); + } + + SearchBuilder search = Search.builder().size(50); + search.query(Query.term("user", "sw")); + final var scroller = ElasticSearchScroller + .<SearchHit>builder() + .client(client) + .search(search.build()) + .index(indexName) + .queryMaxSize(40) + .resultConverter(Function.identity()) + .build(); + Assertions.assertEquals(40, scroller.scroll().size()); + client.shutdown(); server.stop(); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java index 9a103b536c..5bae5a6431 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java @@ -238,7 +238,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { .client(getClient()) .search(search.build()) .index(index) - .queryMaxSize(queryMaxSize) + .queryMaxSize(limit) .resultConverter(searchHit -> { final var sourceAsMap = searchHit.getSource();