This is an automated email from the ASF dual-hosted git repository.

jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 29cf33b  Enhance StarTreeIndexViewer to support multiple trees (#6569)
29cf33b is described below

commit 29cf33b280116c4cf7d3305204c3531f2261b1f9
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Sun Feb 14 15:13:41 2021 -0800

    Enhance StarTreeIndexViewer to support multiple trees (#6569)
    
    Enhance the StarTreeIndexViewers to support:
    - Visualize multiple star-trees
    - Include dimension name in the node
    - Fix a bug of miscounting the top nodes
---
 .../pinot/core/startree/OffHeapStarTreeNode.java   |   4 +-
 .../tests/StarTreeClusterIntegrationTest.java      |   2 +-
 .../apache/pinot/tools/StarTreeIndexViewer.java    | 217 +++++++++++----------
 pinot-tools/src/main/resources/star-tree.html      |   2 +-
 4 files changed, 115 insertions(+), 110 deletions(-)

diff --git 
a/pinot-core/src/main/java/org/apache/pinot/core/startree/OffHeapStarTreeNode.java
 
b/pinot-core/src/main/java/org/apache/pinot/core/startree/OffHeapStarTreeNode.java
index e918680..0104ac6 100644
--- 
a/pinot-core/src/main/java/org/apache/pinot/core/startree/OffHeapStarTreeNode.java
+++ 
b/pinot-core/src/main/java/org/apache/pinot/core/startree/OffHeapStarTreeNode.java
@@ -39,7 +39,7 @@ public class OffHeapStarTreeNode implements StarTreeNode {
 
   public OffHeapStarTreeNode(PinotDataBuffer dataBuffer, int nodeId) {
     _dataBuffer = dataBuffer;
-    long offset = nodeId * SERIALIZABLE_SIZE_IN_BYTES;
+    long offset = (long) nodeId * SERIALIZABLE_SIZE_IN_BYTES;
 
     _dimensionId = dataBuffer.getInt(offset);
     offset += Integer.BYTES;
@@ -77,7 +77,7 @@ public class OffHeapStarTreeNode implements StarTreeNode {
     if (_firstChildId == INVALID_ID) {
       return INVALID_ID;
     } else {
-      return _dataBuffer.getInt(_firstChildId * SERIALIZABLE_SIZE_IN_BYTES);
+      return _dataBuffer.getInt((long) _firstChildId * 
SERIALIZABLE_SIZE_IN_BYTES);
     }
   }
 
diff --git 
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java
 
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java
index 2d2585b..0d29bee 100644
--- 
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java
+++ 
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeClusterIntegrationTest.java
@@ -172,7 +172,7 @@ public class StarTreeClusterIntegrationTest extends 
BaseClusterIntegrationTest {
         functionColumnPairs.add(new 
AggregationFunctionColumnPair(functionType, metric).toColumnName());
       }
     }
-    return new StarTreeIndexConfig(dimensions, null, functionColumnPairs, 0);
+    return new StarTreeIndexConfig(dimensions, null, functionColumnPairs, 100);
   }
 
   @Test
diff --git 
a/pinot-tools/src/main/java/org/apache/pinot/tools/StarTreeIndexViewer.java 
b/pinot-tools/src/main/java/org/apache/pinot/tools/StarTreeIndexViewer.java
index 2afd8db..f237a2d 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/StarTreeIndexViewer.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/StarTreeIndexViewer.java
@@ -26,11 +26,9 @@ import java.io.InputStream;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
@@ -39,15 +37,13 @@ import javax.ws.rs.core.MediaType;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pinot.common.segment.ReadMode;
-import org.apache.pinot.core.common.DataSource;
 import org.apache.pinot.core.indexsegment.IndexSegment;
 import org.apache.pinot.core.indexsegment.immutable.ImmutableSegmentLoader;
-import org.apache.pinot.core.segment.index.metadata.SegmentMetadataImpl;
 import org.apache.pinot.core.segment.index.readers.Dictionary;
 import org.apache.pinot.core.startree.StarTree;
 import org.apache.pinot.core.startree.StarTreeNode;
+import org.apache.pinot.core.startree.v2.StarTreeV2;
 import org.apache.pinot.spi.utils.JsonUtils;
-import org.glassfish.grizzly.http.server.HttpServer;
 import org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpServerFactory;
 import org.glassfish.jersey.server.ResourceConfig;
 import org.slf4j.Logger;
@@ -55,6 +51,57 @@ import org.slf4j.LoggerFactory;
 
 
 public class StarTreeIndexViewer {
+  private static final Logger LOGGER = 
LoggerFactory.getLogger(StarTreeIndexViewer.class);
+
+  /*
+   * MAX num children to show in the UI
+   */
+  private static final int MAX_CHILDREN = 100;
+
+  public StarTreeIndexViewer(File segmentDir)
+      throws Exception {
+    IndexSegment indexSegment = ImmutableSegmentLoader.load(segmentDir, 
ReadMode.heap);
+    List<StarTreeV2> starTrees = indexSegment.getStarTrees();
+    if (starTrees == null) {
+      throw new IllegalStateException("Cannot find star-tree in segment 
directory: " + segmentDir);
+    }
+
+    StarTreeJsonNode rootJsonNode = new StarTreeJsonNode("ROOT");
+    int numStarTrees = starTrees.size();
+    if (numStarTrees == 1) {
+      new SingleTreeViewer(starTrees.get(0)).build(rootJsonNode);
+    } else {
+      for (int i = 0; i < numStarTrees; i++) {
+        StarTreeJsonNode singleTreeRootJsonNode = new 
StarTreeJsonNode("Star-Tree-" + i);
+        new SingleTreeViewer(starTrees.get(i)).build(singleTreeRootJsonNode);
+        rootJsonNode.addChild(singleTreeRootJsonNode);
+      }
+    }
+
+    indexSegment.destroy();
+    String rootJsonString = JsonUtils.objectToPrettyString(rootJsonNode);
+    LOGGER.info(rootJsonString);
+    startServer(rootJsonString);
+  }
+
+  private void startServer(String jsonString) {
+    
GrizzlyHttpServerFactory.createHttpServer(URI.create("http://0.0.0.0:8090/";), 
new StarTreeResource(jsonString));
+    LOGGER.info("Go to http://localhost:8090/ to view the star-trees");
+  }
+
+  public static void main(String[] args)
+      throws Exception {
+    if (args.length != 1) {
+      LOGGER.error("USAGE: StarIndexViewer <segmentDirectory>");
+      System.exit(1);
+    }
+    String segmentDir = args[0];
+    if (!new File(segmentDir).isDirectory()) {
+      throw new FileNotFoundException("Missing directory: " + segmentDir);
+    }
+    new StarTreeIndexViewer(new File(segmentDir));
+  }
+
   private static class StarTreeJsonNode {
     String _name;
     List<StarTreeJsonNode> _children;
@@ -80,107 +127,74 @@ public class StarTreeIndexViewer {
     }
   }
 
-  private static final Logger LOGGER = 
LoggerFactory.getLogger(StarTreeIndexViewer.class);
-
-  /*
-   * MAX num children to show in the UI
-   */
-  static int MAX_CHILDREN = 100;
-  private List<String> _dimensionNames;
-  private Map<String, Dictionary> dictionaries;
-
-  public StarTreeIndexViewer(File segmentDir)
-      throws Exception {
-    IndexSegment indexSegment = ImmutableSegmentLoader.load(segmentDir, 
ReadMode.heap);
-
-    dictionaries = new HashMap<>();
-    SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
-
-    for (String columnName : metadata.getAllColumns()) {
-      DataSource dataSource = indexSegment.getDataSource(columnName);
-      dictionaries.put(columnName, dataSource.getDictionary());
+  private static class SingleTreeViewer {
+    private final StarTreeNode _rootNode;
+    private final List<String> _dimensionNames;
+    private final List<Dictionary> _dimensionDictionaries;
+
+    public SingleTreeViewer(StarTreeV2 starTreeV2) {
+      StarTree starTree = starTreeV2.getStarTree();
+      _rootNode = starTree.getRoot();
+      _dimensionNames = starTree.getDimensionNames();
+      int numDimensions = _dimensionNames.size();
+      _dimensionDictionaries = new ArrayList<>(numDimensions);
+      for (String dimension : _dimensionNames) {
+        
_dimensionDictionaries.add(starTreeV2.getDataSource(dimension).getDictionary());
+      }
     }
-    StarTree tree = indexSegment.getStarTrees().get(0).getStarTree();
-    _dimensionNames = tree.getDimensionNames();
-    StarTreeJsonNode jsonRoot = new StarTreeJsonNode("ROOT");
-    build(tree.getRoot(), jsonRoot);
-    indexSegment.destroy();
 
-    String writeValueAsString = JsonUtils.objectToPrettyString(jsonRoot);
-    LOGGER.info(writeValueAsString);
-    startServer(segmentDir, writeValueAsString);
-  }
-
-  private int build(StarTreeNode indexNode, StarTreeJsonNode json) {
-    Iterator<? extends StarTreeNode> childrenIterator = 
indexNode.getChildrenIterator();
-    if (!childrenIterator.hasNext()) {
-      return 0;
+    public void build(StarTreeJsonNode rootJsonNode) {
+      build(_rootNode, rootJsonNode);
     }
-    int childDimensionId = indexNode.getChildDimensionId();
-    String childDimensionName = _dimensionNames.get(childDimensionId);
-    Dictionary dictionary = dictionaries.get(childDimensionName);
-    int totalChildNodes = indexNode.getNumChildren();
 
-    Comparator<Pair<String, Integer>> comparator = new Comparator<Pair<String, 
Integer>>() {
-
-      @Override
-      public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
-        return -1 * Integer.compare(o1.getRight(), o2.getRight());
-      }
-    };
-    MinMaxPriorityQueue<Pair<String, Integer>> queue =
-        
MinMaxPriorityQueue.orderedBy(comparator).maximumSize(MAX_CHILDREN).create();
-    StarTreeJsonNode allNode = null;
-
-    while (childrenIterator.hasNext()) {
-      StarTreeNode childIndexNode = childrenIterator.next();
-      int childDimensionValueId = childIndexNode.getDimensionValue();
-      String childDimensionValue = "ALL";
-      if (childDimensionValueId != StarTreeNode.ALL) {
-        childDimensionValue = dictionary.get(childDimensionValueId).toString();
+    @SuppressWarnings("UnstableApiUsage")
+    private long build(StarTreeNode treeNode, StarTreeJsonNode jsonNode) {
+      if (treeNode.isLeaf()) {
+        return 0;
       }
-      StarTreeJsonNode childJson = new StarTreeJsonNode(childDimensionValue);
-      totalChildNodes += build(childIndexNode, childJson);
-      if (childDimensionValueId != StarTreeNode.ALL) {
-        json.addChild(childJson);
-        queue.add(ImmutablePair.of(childDimensionValue, totalChildNodes));
-      } else {
-        allNode = childJson;
-      }
-    }
-    //put ALL node at the end
-    if (allNode != null) {
-      json.addChild(allNode);
-    }
-    if (totalChildNodes > MAX_CHILDREN) {
-      Iterator<Pair<String, Integer>> qIterator = queue.iterator();
-      Set<String> topKDimensions = new HashSet<>();
-      topKDimensions.add("ALL");
-      while (qIterator.hasNext()) {
-        topKDimensions.add(qIterator.next().getKey());
+
+      int childDimensionId = treeNode.getChildDimensionId();
+      String childDimension = _dimensionNames.get(childDimensionId);
+      Dictionary dictionary = _dimensionDictionaries.get(childDimensionId);
+      int numChildNodes = treeNode.getNumChildren();
+      long totalChildNodes = numChildNodes;
+
+      Comparator<Pair<String, Long>> comparator = (o1, o2) -> 
Long.compare(o2.getRight(), o1.getRight());
+      MinMaxPriorityQueue<Pair<String, Long>> queue =
+          
MinMaxPriorityQueue.orderedBy(comparator).maximumSize(MAX_CHILDREN).create();
+      StarTreeJsonNode starJsonNode = null;
+
+      Iterator<? extends StarTreeNode> childrenIterator = 
treeNode.getChildrenIterator();
+      while (childrenIterator.hasNext()) {
+        StarTreeNode childTreeNode = childrenIterator.next();
+        int childDimensionValueId = childTreeNode.getDimensionValue();
+        String childDimensionValue =
+            childDimensionValueId != StarTreeNode.ALL ? 
dictionary.getStringValue(childDimensionValueId) : "ALL";
+        String childJsonNodeName = childDimension + ": " + childDimensionValue;
+        StarTreeJsonNode childJsonNode = new 
StarTreeJsonNode(childJsonNodeName);
+        long result = build(childTreeNode, childJsonNode);
+        totalChildNodes += result;
+        if (childDimensionValueId != StarTreeNode.ALL) {
+          jsonNode.addChild(childJsonNode);
+          queue.add(ImmutablePair.of(childJsonNodeName, result));
+        } else {
+          starJsonNode = childJsonNode;
+        }
       }
-      Iterator<StarTreeJsonNode> iterator = json.getChildren().iterator();
-      while (iterator.hasNext()) {
-        StarTreeJsonNode next = iterator.next();
-        if (!topKDimensions.contains(next.getName())) {
-          iterator.remove();
+      if (numChildNodes > MAX_CHILDREN) {
+        // Keep the nodes with most child nodes
+        Set<String> nodeNamesToKeep = new HashSet<>();
+        for (Pair<String, Long> pair : queue) {
+          nodeNamesToKeep.add(pair.getKey());
         }
+        jsonNode.getChildren().removeIf(next -> 
!nodeNamesToKeep.contains(next.getName()));
       }
+      // Add star-node at last
+      if (starJsonNode != null) {
+        jsonNode.addChild(starJsonNode);
+      }
+      return totalChildNodes;
     }
-    return totalChildNodes;
-  }
-
-  public static void main(String[] args)
-      throws Exception {
-    if (args.length != 1) {
-      LOGGER.error("USAGE: StarIndexViewer <segmentDirectory>");
-      System.exit(1);
-    }
-    String segmentDir = args[0];
-    if (!new File(segmentDir).exists()) {
-      throw new FileNotFoundException("Missing directory:" + segmentDir);
-    }
-    new StarTreeIndexViewer(new File(segmentDir));
   }
 
   private static class StarTreeResource extends ResourceConfig {
@@ -191,15 +205,6 @@ public class StarTreeIndexViewer {
     }
   }
 
-  private void startServer(final File segmentDirectory, final String json)
-      throws Exception {
-    int httpPort = 8090;
-    URI baseUri = URI.create("http://0.0.0.0:"; + Integer.toString(httpPort) + 
"/");
-    HttpServer httpServer = GrizzlyHttpServerFactory.createHttpServer(baseUri, 
new StarTreeResource(json));
-
-    LOGGER.info("Go to http://{}:{}/  to view the star tree", "localhost", 
httpPort);
-  }
-
   @Path("/")
   public static final class StarTreeViewRestResource {
     public static String json;
diff --git a/pinot-tools/src/main/resources/star-tree.html 
b/pinot-tools/src/main/resources/star-tree.html
index c39d7b7..7385606 100644
--- a/pinot-tools/src/main/resources/star-tree.html
+++ b/pinot-tools/src/main/resources/star-tree.html
@@ -49,7 +49,7 @@
 <script>
 
 var margin = {top: 20, right: 120, bottom: 20, left: 120},
-    width = 960 - margin.right - margin.left,
+    width = 5000 - margin.right - margin.left,
     height = 15000 - margin.top - margin.bottom;
 
 var i = 0,


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to