This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c662e43b5b UI Load time Improvement API Fixes #13278 (#13296)
c662e43b5b is described below
commit c662e43b5bbc2428a372d778048db6c4758e1fff
Author: Chaitanya Deepthi <[email protected]>
AuthorDate: Wed Aug 28 16:41:43 2024 -0700
UI Load time Improvement API Fixes #13278 (#13296)
---
.../api/resources/PinotSchemaRestletResource.java | 32 ++++
.../api/resources/PinotSegmentRestletResource.java | 26 ++-
.../api/resources/PinotTaskRestletResource.java | 9 +
.../api/resources/SegmentStatusInfo.java | 49 ++++++
.../pinot/controller/api/resources/TableSize.java | 15 +-
.../pinot/controller/api/resources/TableViews.java | 84 ++++++++++
.../helix/core/PinotHelixResourceManager.java | 20 +++
.../src/main/resources/app/interfaces/types.d.ts | 17 ++
.../src/main/resources/app/pages/TenantDetails.tsx | 21 +--
.../src/main/resources/app/requests/index.ts | 21 ++-
.../main/resources/app/utils/PinotMethodUtils.ts | 82 ++++++----
.../controller/helix/SegmentStatusCheckerTest.java | 181 +++++++++++++++++++++
.../java/org/apache/pinot/core/auth/Actions.java | 3 +
.../java/org/apache/pinot/spi/data/SchemaInfo.java | 70 ++++++++
.../apache/pinot/spi/utils/CommonConstants.java | 6 +
.../org/apache/pinot/spi/data/SchemaInfoTest.java | 78 +++++++++
16 files changed, 656 insertions(+), 58 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
index 4b2c1e33c9..eecfe63b6c 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
@@ -31,6 +31,8 @@ import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
@@ -50,6 +52,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pinot.common.config.provider.TableCache;
import org.apache.pinot.common.exception.SchemaAlreadyExistsException;
import org.apache.pinot.common.exception.SchemaBackwardIncompatibleException;
import org.apache.pinot.common.exception.SchemaNotFoundException;
@@ -72,6 +75,7 @@ import org.apache.pinot.segment.local.utils.SchemaUtils;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.SchemaInfo;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.glassfish.grizzly.http.server.Request;
@@ -118,6 +122,34 @@ public class PinotSchemaRestletResource {
return
_pinotHelixResourceManager.getSchemaNames(headers.getHeaderString(DATABASE));
}
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("/schemas/info")
+ @Authorize(targetType = TargetType.CLUSTER, action =
Actions.Cluster.GET_SCHEMAS_INFO)
+ @ApiOperation(value = "List all schemas info with count of field specs",
notes = "Lists all schemas with field "
+ + "count details")
+ public String getSchemasInfo(@Context HttpHeaders headers)
+ throws JsonProcessingException {
+ Map<String, Object> schemaInfoObjects = new LinkedHashMap<>();
+ List<SchemaInfo> schemasInfo = new ArrayList<>();
+ List<String> uncachedSchemas = new ArrayList<>();
+ TableCache cache = _pinotHelixResourceManager.getTableCache();
+ List<String> schemas =
_pinotHelixResourceManager.getSchemaNames(headers.getHeaderString(DATABASE));
+ for (String schemaName : schemas) {
+ Schema schema = cache.getSchema(schemaName);
+ if (schema != null) {
+ schemasInfo.add(new SchemaInfo(schema));
+ } else {
+ uncachedSchemas.add(schemaName);
+ }
+ }
+ schemaInfoObjects.put("schemasInfo", schemasInfo);
+ if (!uncachedSchemas.isEmpty()) {
+ schemaInfoObjects.put("uncachedSchemas", uncachedSchemas);
+ }
+ return JsonUtils.objectToPrettyString(schemaInfoObjects);
+ }
+
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/schemas/{schemaName}")
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
index 74c09d8da7..2f0d565590 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
@@ -237,21 +237,37 @@ public class PinotSegmentRestletResource {
notes = "Get a map from server to segments hosted by the server")
public List<Map<String, Object>> getServerToSegmentsMap(
@ApiParam(value = "Name of the table", required = true)
@PathParam("tableName") String tableName,
- @ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String
tableTypeStr, @Context HttpHeaders headers) {
+ @ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String
tableTypeStr,
+ @QueryParam("verbose") @DefaultValue("true") boolean verbose, @Context
HttpHeaders headers) {
tableName = DatabaseUtils.translateTableName(tableName, headers);
- List<String> tableNamesWithType = ResourceUtils
- .getExistingTableNamesWithType(_pinotHelixResourceManager, tableName,
Constants.validateTableType(tableTypeStr),
- LOGGER);
+ List<String> tableNamesWithType =
ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager,
tableName,
+ Constants.validateTableType(tableTypeStr), LOGGER);
List<Map<String, Object>> resultList = new
ArrayList<>(tableNamesWithType.size());
for (String tableNameWithType : tableNamesWithType) {
Map<String, Object> resultForTable = new LinkedHashMap<>();
resultForTable.put("tableName", tableNameWithType);
- resultForTable.put("serverToSegmentsMap",
_pinotHelixResourceManager.getServerToSegmentsMap(tableNameWithType));
+ if (!verbose) {
+ resultForTable.put("serverToSegmentsCountMap",
+
_pinotHelixResourceManager.getServerToSegmentsCountMap(tableNameWithType));
+ } else {
+ Map<String, List<String>> serverToSegmentsMap =
+
_pinotHelixResourceManager.getServerToSegmentsMap(tableNameWithType);
+ resultForTable.put("serverToSegmentsMap", serverToSegmentsMap);
+ resultForTable.put("serverToSegmentsCountMap",
getServerToSegmentCountMap(serverToSegmentsMap));
+ }
resultList.add(resultForTable);
}
return resultList;
}
+ private Map<String, Integer> getServerToSegmentCountMap(Map<String,
List<String>> serverToSegmentsMap) {
+ Map<String, Integer> serverToSegmentCount = new TreeMap<>();
+ for (Map.Entry<String, List<String>> entry :
serverToSegmentsMap.entrySet()) {
+ serverToSegmentCount.put(entry.getKey(), entry.getValue().size());
+ }
+ return serverToSegmentCount;
+ }
+
@GET
@Path("segments/{tableName}/lineage")
@Authorize(targetType = TargetType.TABLE, paramName = "tableName", action =
Actions.Table.GET_SEGMENT_LINEAGE)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
index bc02a82ff5..9cfcbc3a06 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
@@ -200,6 +200,15 @@ public class PinotTaskRestletResource {
return tasks;
}
+ @GET
+ @Path("/tasks/{taskType}/tasks/count")
+ @Authorize(targetType = TargetType.CLUSTER, action =
Actions.Cluster.GET_TASK_COUNT)
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation("Count of all tasks for the given task type")
+ public int getTasksCount(@ApiParam(value = "Task type", required = true)
@PathParam("taskType") String taskType) {
+ return _pinotHelixTaskResourceManager.getTasks(taskType).size();
+ }
+
@GET
@Path("/tasks/{taskType}/{tableNameWithType}/state")
@Authorize(targetType = TargetType.CLUSTER, action =
Actions.Cluster.GET_TASK)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/SegmentStatusInfo.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/SegmentStatusInfo.java
new file mode 100644
index 0000000000..613a744570
--- /dev/null
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/SegmentStatusInfo.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.api.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * This class gives the details of a particular segment and it's status
+ *
+ */
+public class SegmentStatusInfo {
+ @JsonProperty("segmentName")
+ String _segmentName;
+
+ @JsonProperty("segmentStatus")
+ String _segmentStatus;
+
+ public String getSegmentName() {
+ return _segmentName;
+ }
+
+ public String getSegmentStatus() {
+ return _segmentStatus;
+ }
+
+ @JsonCreator
+ public SegmentStatusInfo(@JsonProperty("segmentName") String segmentName,
+ @JsonProperty("segmentStatus") String segmentStatus) {
+ _segmentName = segmentName;
+ _segmentStatus = segmentStatus;
+ }
+}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
index ea3671b083..ede9d3eb73 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
@@ -27,11 +27,14 @@ import io.swagger.annotations.ApiResponses;
import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
+import java.util.HashMap;
import javax.inject.Inject;
+import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
@@ -80,12 +83,22 @@ public class TableSize {
})
public TableSizeReader.TableSizeDetails getTableSize(
@ApiParam(value = "Table name without type", required = true, example =
"myTable | myTable_OFFLINE")
- @PathParam("tableName") String tableName, @Context HttpHeaders headers) {
+ @PathParam("tableName") String tableName,
+ @ApiParam(value = "Provide detailed information") @DefaultValue("true")
@QueryParam("verbose") boolean verbose,
+ @Context HttpHeaders headers) {
tableName = DatabaseUtils.translateTableName(tableName, headers);
TableSizeReader.TableSizeDetails tableSizeDetails = null;
try {
tableSizeDetails =
_tableSizeReader.getTableSizeDetails(tableName,
_controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
+ if (!verbose) {
+ if (tableSizeDetails._offlineSegments != null) {
+ tableSizeDetails._offlineSegments._segments = new HashMap<>();
+ }
+ if (tableSizeDetails._realtimeSegments != null) {
+ tableSizeDetails._realtimeSegments._segments = new HashMap<>();
+ }
+ }
} catch (Throwable t) {
throw new ControllerApplicationException(LOGGER, String.format("Failed
to read table size for %s", tableName),
Response.Status.INTERNAL_SERVER_ERROR, t);
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableViews.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableViews.java
index a7b33ef133..c4ade0c947 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableViews.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableViews.java
@@ -19,6 +19,7 @@
package org.apache.pinot.controller.api.resources;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiKeyAuthDefinition;
import io.swagger.annotations.ApiOperation;
@@ -26,6 +27,9 @@ import io.swagger.annotations.ApiParam;
import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -48,6 +52,8 @@ import org.apache.pinot.core.auth.Actions;
import org.apache.pinot.core.auth.Authorize;
import org.apache.pinot.core.auth.TargetType;
import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -110,6 +116,84 @@ public class TableViews {
return getTableState(tableName, EXTERNALVIEW, tableType);
}
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("/tables/{tableName}/segmentsStatus")
+ @Authorize(targetType = TargetType.TABLE, paramName = "tableName", action =
Actions.Table.GET_SEGMENT_STATUS)
+ @ApiOperation(value = "Get segment names to segment status map", notes =
"Get segment statuses of each segment")
+ public String getSegmentsStatusDetails(
+ @ApiParam(value = "Name of the table", required = true)
@PathParam("tableName") String tableName,
+ @ApiParam(value = "realtime|offline", required = false)
@QueryParam("tableType") String tableTypeStr,
+ @Context HttpHeaders headers)
+ throws JsonProcessingException {
+ tableName = DatabaseUtils.translateTableName(tableName, headers);
+ TableType tableType = validateTableType(tableTypeStr);
+ TableViews.TableView externalView = getTableState(tableName,
TableViews.EXTERNALVIEW, tableType);
+ TableViews.TableView idealStateView = getTableState(tableName,
TableViews.IDEALSTATE, tableType);
+ List<SegmentStatusInfo> segmentStatusInfoListMap = new ArrayList<>();
+ segmentStatusInfoListMap = getSegmentStatuses(externalView,
idealStateView);
+ return JsonUtils.objectToPrettyString(segmentStatusInfoListMap);
+ }
+
+ public List<SegmentStatusInfo> getSegmentStatuses(TableViews.TableView
externalView,
+ TableViews.TableView idealStateView) {
+ Map<String, Map<String, String>> idealStateMap =
getStateMap(idealStateView);
+ Map<String, Map<String, String>> externalViewMap =
getStateMap(externalView);
+ List<SegmentStatusInfo> segmentStatusInfoList = new ArrayList<>();
+
+ for (Map.Entry<String, Map<String, String>> entry :
externalViewMap.entrySet()) {
+ String segment = entry.getKey();
+ Map<String, String> externalViewEntryValue = entry.getValue();
+ Map<String, String> idealViewEntryValue = idealStateMap.get(segment);
+ if (isErrorSegment(externalViewEntryValue)) {
+ segmentStatusInfoList.add(
+ new SegmentStatusInfo(segment,
CommonConstants.Helix.StateModel.DisplaySegmentStatus.BAD));
+ } else {
+ boolean isViewsEqual =
externalViewEntryValue.equals(idealViewEntryValue);
+ if (isViewsEqual) {
+ if (isOnlineOrConsumingSegment(externalViewEntryValue)) {
+ segmentStatusInfoList.add(
+ new SegmentStatusInfo(segment,
CommonConstants.Helix.StateModel.DisplaySegmentStatus.GOOD));
+ } else if (isOfflineSegment(externalViewEntryValue)) {
+ segmentStatusInfoList.add(
+ new SegmentStatusInfo(segment,
CommonConstants.Helix.StateModel.DisplaySegmentStatus.GOOD));
+ } else {
+ segmentStatusInfoList.add(
+ new SegmentStatusInfo(segment,
CommonConstants.Helix.StateModel.DisplaySegmentStatus.UPDATING));
+ }
+ } else {
+ segmentStatusInfoList.add(
+ new SegmentStatusInfo(segment,
CommonConstants.Helix.StateModel.DisplaySegmentStatus.UPDATING));
+ }
+ }
+ }
+ return segmentStatusInfoList;
+ }
+
+ private Map<String, Map<String, String>> getStateMap(TableViews.TableView
view) {
+ if (view != null && view._offline != null && !view._offline.isEmpty()) {
+ return view._offline;
+ } else if (view != null && view._realtime != null &&
!view._realtime.isEmpty()) {
+ return view._realtime;
+ } else {
+ return new HashMap<>();
+ }
+ }
+
+ private boolean isErrorSegment(Map<String, String> stateMap) {
+ return
stateMap.values().contains(CommonConstants.Helix.StateModel.SegmentStateModel.ERROR);
+ }
+
+ private boolean isOnlineOrConsumingSegment(Map<String, String> stateMap) {
+ return stateMap.values().stream().allMatch(
+ state ->
state.equals(CommonConstants.Helix.StateModel.SegmentStateModel.CONSUMING) ||
state.equals(
+ CommonConstants.Helix.StateModel.SegmentStateModel.ONLINE));
+ }
+
+ private boolean isOfflineSegment(Map<String, String> stateMap) {
+ return
stateMap.values().contains(CommonConstants.Helix.StateModel.SegmentStateModel.OFFLINE);
+ }
+
// we use name "view" to closely match underlying names and to not
// confuse with table state of enable/disable
private TableView getTableState(String tableName, String view, TableType
tableType) {
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 2171d6a70e..e8122b5485 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -2865,6 +2865,26 @@ public class PinotHelixResourceManager {
return serverToSegmentsMap;
}
+ /**
+ * Returns a map from server instance to count of segments it serves for the
given table. Ignore OFFLINE segments from
+ * the ideal state because they are not supposed to be served.
+ */
+ public Map<String, Integer> getServerToSegmentsCountMap(String
tableNameWithType) {
+ Map<String, Integer> serverToSegmentCountMap = new TreeMap<>();
+ IdealState idealState =
_helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType);
+ if (idealState == null) {
+ throw new IllegalStateException("Ideal state does not exist for table: "
+ tableNameWithType);
+ }
+ for (Map.Entry<String, Map<String, String>> entry :
idealState.getRecord().getMapFields().entrySet()) {
+ for (Map.Entry<String, String> instanceStateEntry :
entry.getValue().entrySet()) {
+ if (!instanceStateEntry.getValue().equals(SegmentStateModel.OFFLINE)) {
+ serverToSegmentCountMap.merge(instanceStateEntry.getKey(), 1,
Integer::sum);
+ }
+ }
+ }
+ return serverToSegmentCountMap;
+ }
+
/**
* Returns a set of server instances for a given table and segment. Ignore
OFFLINE segments from the ideal state
* because they are not supposed to be served.
diff --git a/pinot-controller/src/main/resources/app/interfaces/types.d.ts
b/pinot-controller/src/main/resources/app/interfaces/types.d.ts
index 52f937bf0a..084f139518 100644
--- a/pinot-controller/src/main/resources/app/interfaces/types.d.ts
+++ b/pinot-controller/src/main/resources/app/interfaces/types.d.ts
@@ -110,6 +110,16 @@ declare module 'Models' {
error?: string;
};
+ export type ServerToSegmentsCount = {
+ tableName: string;
+ serverToSegmentsCountMap: number;
+ };
+
+ export type SegmentStatusInfo = {
+ segmentName: string;
+ segmentStatus: DISPLAY_SEGMENT_STATUS;
+ };
+
export type QueryTables = {
tables: Array<string>;
};
@@ -130,6 +140,13 @@ declare module 'Models' {
fieldType?: string
};
+ export type SchemaInfo = {
+ schemaName: string
+ numDimensionFields: number
+ numDateTimeFields: number
+ numMetricFields: number
+ };
+
export type SQLResult = {
resultTable: {
dataSchema: {
diff --git a/pinot-controller/src/main/resources/app/pages/TenantDetails.tsx
b/pinot-controller/src/main/resources/app/pages/TenantDetails.tsx
index 0fbd9c6af4..a761f15fba 100644
--- a/pinot-controller/src/main/resources/app/pages/TenantDetails.tsx
+++ b/pinot-controller/src/main/resources/app/pages/TenantDetails.tsx
@@ -186,24 +186,11 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
const fetchSegmentData = async () => {
const result = await PinotMethodUtils.getSegmentList(tableName);
- const {columns, records, externalViewObj} = result;
- const instanceObj = {};
- externalViewObj && Object.keys(externalViewObj).map((segmentName)=>{
- const instanceKeys = Object.keys(externalViewObj[segmentName]);
- instanceKeys.map((instanceName)=>{
- if(!instanceObj[instanceName]){
- instanceObj[instanceName] = 0;
- }
- instanceObj[instanceName] += 1;
- })
- });
- const instanceRecords = [];
- Object.keys(instanceObj).map((instanceName)=>{
- instanceRecords.push([instanceName, instanceObj[instanceName]]);
- })
+ const data = await
PinotMethodUtils.fetchServerToSegmentsCountData(tableName, tableType);
+ const {columns, records} = result;
setInstanceCountData({
columns: instanceColumns,
- records: instanceRecords
+ records: data.records
});
const segmentTableRows = [];
@@ -221,10 +208,10 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
},
])
);
-
setSegmentList({columns, records: segmentTableRows});
};
+
const fetchTableSchema = async () => {
const result = await PinotMethodUtils.getTableSchemaData(tableName);
if(result.error){
diff --git a/pinot-controller/src/main/resources/app/requests/index.ts
b/pinot-controller/src/main/resources/app/requests/index.ts
index e2d4d54910..cb31fac76b 100644
--- a/pinot-controller/src/main/resources/app/requests/index.ts
+++ b/pinot-controller/src/main/resources/app/requests/index.ts
@@ -46,6 +46,9 @@ import {
QuerySchemas,
TableType,
InstanceState, SegmentMetadata,
+ SchemaInfo,
+ SegmentStatusInfo,
+ ServerToSegmentsCount
} from 'Models';
const headers = {
@@ -73,6 +76,9 @@ export const putTable = (name: string, params: string):
Promise<AxiosResponse<Op
export const getSchemaList = (): Promise<AxiosResponse<QuerySchemas>> =>
baseApi.get(`/schemas`);
+export const getSchemaInfo = (): Promise<AxiosResponse<OperationResponse>> =>
+ baseApi.get(`/schemas/info`);
+
export const getSchema = (name: string):
Promise<AxiosResponse<OperationResponse>> =>
baseApi.get(`/schemas/${name}`);
@@ -89,8 +95,8 @@ export const putSchema = (name: string, params: string,
reload?: boolean): Promi
export const getSegmentMetadata = (tableName: string, segmentName: string):
Promise<AxiosResponse<SegmentMetadata>> =>
baseApi.get(`/segments/${tableName}/${segmentName}/metadata?columns=*`);
-export const getTableSize = (name: string): Promise<AxiosResponse<TableSize>>
=>
- baseApi.get(`/tables/${name}/size`);
+export const getTableSize = (name: string, verbose: boolean = false):
Promise<AxiosResponse<TableSize>> =>
+ baseApi.get(`/tables/${name}/size?verbose=${verbose}`);
export const getIdealState = (name: string):
Promise<AxiosResponse<IdealState>> =>
baseApi.get(`/tables/${name}/idealstate`);
@@ -98,6 +104,12 @@ export const getIdealState = (name: string):
Promise<AxiosResponse<IdealState>>
export const getExternalView = (name: string):
Promise<AxiosResponse<IdealState>> =>
baseApi.get(`/tables/${name}/externalview`);
+export const getServerToSegmentsCount = (name: string, tableType: TableType,
verbose: boolean = false): Promise<AxiosResponse<ServerToSegmentsCount[]>> =>
+
baseApi.get(`/segments/${name}/servers?type=${tableType}&verbose=${verbose}`);
+
+export const getSegmentsStatus = (name: string):
Promise<AxiosResponse<SegmentStatusInfo[]>> =>
+ baseApi.get(`/tables/${name}/segmentsStatus`);
+
export const getInstances = (): Promise<AxiosResponse<Instances>> =>
baseApi.get('/instances');
@@ -128,6 +140,9 @@ export const getTaskTypes = ():
Promise<AxiosResponse<OperationResponse>> =>
export const getTaskTypeTasks = (taskType: string):
Promise<AxiosResponse<OperationResponse>> =>
baseApi.get(`/tasks/${taskType}/tasks`, { headers: { ...headers, Accept:
'application/json' } });
+export const getTaskTypeTasksCount = (taskType: string):
Promise<AxiosResponse<number>> =>
+ baseApi.get(`/tasks/${taskType}/tasks/count`, { headers: { ...headers,
Accept: 'application/json' } });
+
export const getTaskTypeState = (taskType: string):
Promise<AxiosResponse<OperationResponse>> =>
baseApi.get(`/tasks/${taskType}/state`, { headers: { ...headers, Accept:
'application/json' } });
@@ -294,4 +309,4 @@ export const requestDeleteUser = (userObject: UserObject):
Promise<AxiosResponse
baseApi.delete(`/users/${userObject.username}?component=${userObject.component}`);
export const requestUpdateUser = (userObject: UserObject, passwordChanged:
boolean): Promise<AxiosResponse<any>> =>
-
baseApi.put(`/users/${userObject.username}?component=${userObject.component}&passwordChanged=${passwordChanged}`,
JSON.stringify(userObject), {headers});
\ No newline at end of file
+
baseApi.put(`/users/${userObject.username}?component=${userObject.component}&passwordChanged=${passwordChanged}`,
JSON.stringify(userObject), {headers});
diff --git a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
index 7d971036dd..d3af96ff69 100644
--- a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
+++ b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
@@ -34,6 +34,7 @@ import {
getTaskTypeDebug,
getTables,
getTaskTypeTasks,
+ getTaskTypeTasksCount,
getTaskTypeState,
stopTasks,
resumeTasks,
@@ -94,7 +95,10 @@ import {
requestUpdateUser,
getTaskProgress,
getSegmentReloadStatus,
- getTaskRuntimeConfig
+ getTaskRuntimeConfig,
+ getSchemaInfo,
+ getSegmentsStatus,
+ getServerToSegmentsCount
} from '../requests';
import { baseApi } from './axios-config';
import Utils, { getDisplaySegmentStatus } from './Utils';
@@ -380,17 +384,13 @@ const allSchemaDetailsColumnHeader = ["Schema Name",
"Dimension Columns", "Date-
const getAllSchemaDetails = async (schemaList) => {
let schemaDetails:Array<any> = [];
- let promiseArr = [];
- promiseArr = schemaList.map(async (o)=>{
- return await getSchema(o);
- });
- const results = await Promise.all(promiseArr);
+ const results = await getSchemaDataInfo();
schemaDetails = results.map((obj)=>{
let schemaObj = [];
- schemaObj.push(obj.data.schemaName);
- schemaObj.push(obj.data.dimensionFieldSpecs ?
obj.data.dimensionFieldSpecs.length : 0);
- schemaObj.push(obj.data.dateTimeFieldSpecs ?
obj.data.dateTimeFieldSpecs.length : 0);
- schemaObj.push(obj.data.metricFieldSpecs ?
obj.data.metricFieldSpecs.length : 0);
+ schemaObj.push(obj.schemaName);
+ schemaObj.push(obj.numDimensionFields);
+ schemaObj.push(obj.numDateTimeFields);
+ schemaObj.push(obj.numMetricFields);
schemaObj.push(schemaObj[1] + schemaObj[2] + schemaObj[3]);
return schemaObj;
})
@@ -506,28 +506,38 @@ const getTableSummaryData = (tableName) => {
});
};
-// This method is used to display segment list of a particular tenant table
-// API: /tables/:tableName/idealstate
-// /tables/:tableName/externalview
-// Expected Output: {columns: [], records: [], externalViewObject: {}}
-const getSegmentList = (tableName) => {
- const promiseArr = [];
- promiseArr.push(getIdealState(tableName));
- promiseArr.push(getExternalView(tableName));
+// This method is used to display segment list of a particular table with
segment name and it's status
+// API: /tables/${name}/segmentsStatus
+// Expected Output: {columns: [], records: []}
+ const getSegmentList = (tableName) => {
+ return getSegmentsStatus(tableName).then((results) => {
+ const segmentsArray = results.data; // assuming the array is inside
`segments` property
+ return {
+ columns: ['Segment Name', 'Status'],
+ records: segmentsArray.map((segment) => [
+ segment.segmentName,
+ segment.segmentStatus
+ ])
+ };
+ });
+ };
- return Promise.all(promiseArr).then((results) => {
- const idealStateObj = results[0].data.OFFLINE || results[0].data.REALTIME;
- const externalViewObj = results[1].data.OFFLINE ||
results[1].data.REALTIME;
+const getExternalViewObj = (tableName) => {
+ return getExternalView(tableName).then((result) => {
+ return result.data.OFFLINE || result.data.REALTIME;
+ });
+};
+const fetchServerToSegmentsCountData = (tableName, tableType) => {
+ return getServerToSegmentsCount(tableName, tableType).then((results) => {
+ const segmentsArray = results.data;
return {
- columns: ['Segment Name', 'Status'],
- records: Object.keys(idealStateObj).map((key) => {
- return [
- key,
- getDisplaySegmentStatus(idealStateObj[key], externalViewObj[key])
- ];
- }),
- externalViewObj
+ records: segmentsArray.flatMap((server) =>
+ Object.entries(server.serverToSegmentsCountMap).map(([serverName,
segmentsCount]) => [
+ serverName,
+ segmentsCount
+ ])
+ )
};
});
};
@@ -806,7 +816,7 @@ const getAllTaskTypes = async () => {
};
const getTaskInfo = async (taskType) => {
- const tasksRes = await getTaskTypeTasks(taskType);
+ const tasksResLength = await getTaskTypeTasksCount(taskType);
const stateRes = await getTaskTypeState(taskType);
let state = get(stateRes, 'data', '');
@@ -814,7 +824,7 @@ const getTaskInfo = async (taskType) => {
if(typeof state !== "string") {
state = "";
}
- return [tasksRes?.data?.length || 0, state];
+ return [tasksResLength.data || 0, state];
};
const stopAllTasks = (taskType) => {
@@ -1007,6 +1017,12 @@ const getSchemaData = (schemaName) => {
});
};
+const getSchemaDataInfo = () => {
+ return getSchemaInfo().then((response)=>{
+ return response.data.schemasInfo;
+ });
+};
+
const getTableState = (tableName, tableType) => {
return getState(tableName, tableType).then((response)=>{
return response.data;
@@ -1246,6 +1262,7 @@ export default {
getAllTableDetails,
getTableSummaryData,
getSegmentList,
+ getExternalViewObj,
getSegmentStatus,
getTableDetails,
getSegmentDetails,
@@ -1318,5 +1335,6 @@ export default {
deleteUser,
updateUser,
getAuthUserNameFromAccessToken,
- getAuthUserEmailFromAccessToken
+ getAuthUserEmailFromAccessToken,
+ fetchServerToSegmentsCountData
};
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
index 81a0f345c1..1edd2176e6 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
@@ -18,8 +18,13 @@
*/
package org.apache.pinot.controller.helix;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.helix.AccessOption;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
@@ -36,6 +41,8 @@ import org.apache.pinot.common.metrics.MetricValueUtils;
import org.apache.pinot.common.utils.LLCSegmentName;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.LeadControllerManager;
+import org.apache.pinot.controller.api.resources.SegmentStatusInfo;
+import org.apache.pinot.controller.api.resources.TableViews;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.controller.util.TableSizeReader;
import org.apache.pinot.spi.config.table.TableConfig;
@@ -43,6 +50,7 @@ import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.metrics.PinotMetricUtils;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.CommonConstants.Segment.Realtime.Status;
+import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.testng.annotations.Test;
@@ -519,4 +527,177 @@ public class SegmentStatusCheckerTest {
runSegmentStatusChecker(resourceManager, 0);
verifyControllerMetrics(OFFLINE_TABLE_NAME, 1, numSegments, numSegments,
0, 0, 0, 99, 0, 246800);
}
+
+ @Test
+ public void testAllSegmentsGoodOnlineOfflineTable() {
+ TableViews.TableView tableViewExternal = new TableViews.TableView();
+ TableViews.TableView tableViewIdeal = new TableViews.TableView();
+ Map<String, Map<String, String>> tableViewExternalOffline = new
TreeMap<>();
+ Map<String, Map<String, String>> tableViewIdealOffline = new TreeMap<>();
+ Map<String, String> testSegment1MapExternal = new LinkedHashMap<>();
+ testSegment1MapExternal.put("Server1", "ONLINE");
+ tableViewExternalOffline.put("TestSegment1", testSegment1MapExternal);
+ tableViewExternalOffline.put("TestSegment2", testSegment1MapExternal);
+ Map<String, String> testSegment1MapIdeal = new LinkedHashMap<>();
+ testSegment1MapIdeal.put("Server1", "ONLINE");
+ tableViewIdealOffline.put("TestSegment1", testSegment1MapIdeal);
+ tableViewIdealOffline.put("TestSegment2", testSegment1MapIdeal);
+ tableViewExternal._offline = tableViewExternalOffline;
+ tableViewIdeal._offline = tableViewIdealOffline;
+ TableViews tableviews = new TableViews();
+ List<SegmentStatusInfo> segmentStatusInfos =
tableviews.getSegmentStatuses(tableViewExternal, tableViewIdeal);
+ assertEquals(segmentStatusInfos.get(0).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.GOOD);
+ assertEquals(segmentStatusInfos.get(1).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.GOOD);
+ }
+
+ @Test
+ public void testAllSegmentsGoodConsumingOfflineTable() {
+ TableViews.TableView tableViewExternal = new TableViews.TableView();
+ TableViews.TableView tableViewIdeal = new TableViews.TableView();
+ Map<String, Map<String, String>> tableViewExternalOffline = new
TreeMap<>();
+ Map<String, Map<String, String>> tableViewIdealOffline = new TreeMap<>();
+ Map<String, String> testSegment1MapExternal = new LinkedHashMap<>();
+ testSegment1MapExternal.put("Server1", "CONSUMING");
+ tableViewExternalOffline.put("TestSegment1", testSegment1MapExternal);
+ tableViewExternalOffline.put("TestSegment2", testSegment1MapExternal);
+ Map<String, String> testSegment1MapIdeal = new LinkedHashMap<>();
+ testSegment1MapIdeal.put("Server1", "CONSUMING");
+ tableViewIdealOffline.put("TestSegment1", testSegment1MapIdeal);
+ tableViewIdealOffline.put("TestSegment2", testSegment1MapIdeal);
+ tableViewExternal._offline = tableViewExternalOffline;
+ tableViewIdeal._offline = tableViewIdealOffline;
+ TableViews tableviews = new TableViews();
+ List<SegmentStatusInfo> segmentStatusInfos =
tableviews.getSegmentStatuses(tableViewExternal, tableViewIdeal);
+ assertEquals(segmentStatusInfos.get(0).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.GOOD);
+ assertEquals(segmentStatusInfos.get(1).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.GOOD);
+ }
+
+ @Test
+ public void testAllSegmentsBadOfflineTable() {
+ TableViews.TableView tableViewExternal = new TableViews.TableView();
+ TableViews.TableView tableViewIdeal = new TableViews.TableView();
+ Map<String, Map<String, String>> tableViewExternalOffline = new
TreeMap<>();
+ Map<String, Map<String, String>> tableViewIdealOffline = new TreeMap<>();
+ Map<String, String> testSegment1MapExternal = new LinkedHashMap<>();
+ testSegment1MapExternal.put("Server1", "ERROR");
+ tableViewExternalOffline.put("TestSegment1", testSegment1MapExternal);
+ tableViewExternalOffline.put("TestSegment2", testSegment1MapExternal);
+ Map<String, String> testSegment1MapIdeal = new LinkedHashMap<>();
+ testSegment1MapIdeal.put("Server1", "ONLINE");
+ tableViewIdealOffline.put("TestSegment1", testSegment1MapIdeal);
+ tableViewIdealOffline.put("TestSegment2", testSegment1MapIdeal);
+ tableViewExternal._offline = tableViewExternalOffline;
+ tableViewIdeal._offline = tableViewIdealOffline;
+ TableViews tableviews = new TableViews();
+ List<SegmentStatusInfo> segmentStatusInfos =
tableviews.getSegmentStatuses(tableViewExternal, tableViewIdeal);
+ assertEquals(segmentStatusInfos.get(0).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.BAD);
+ assertEquals(segmentStatusInfos.get(1).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.BAD);
+ }
+
+ @Test
+ public void testAllSegmentsUpdatingOfflineTable() {
+ TableViews.TableView tableViewExternal = new TableViews.TableView();
+ TableViews.TableView tableViewIdeal = new TableViews.TableView();
+ Map<String, Map<String, String>> tableViewExternalOffline = new
TreeMap<>();
+ Map<String, Map<String, String>> tableViewIdealOffline = new TreeMap<>();
+ Map<String, String> testSegment1MapExternal = new LinkedHashMap<>();
+ testSegment1MapExternal.put("Server1", "OFFLINE");
+ tableViewExternalOffline.put("TestSegment1", testSegment1MapExternal);
+ tableViewExternalOffline.put("TestSegment2", testSegment1MapExternal);
+ Map<String, String> testSegment1MapIdeal = new LinkedHashMap<>();
+ testSegment1MapIdeal.put("Server1", "ONLINE");
+ tableViewIdealOffline.put("TestSegment1", testSegment1MapIdeal);
+ tableViewIdealOffline.put("TestSegment2", testSegment1MapIdeal);
+ tableViewExternal._offline = tableViewExternalOffline;
+ tableViewIdeal._offline = tableViewIdealOffline;
+ TableViews tableviews = new TableViews();
+ List<SegmentStatusInfo> segmentStatusInfos =
tableviews.getSegmentStatuses(tableViewExternal, tableViewIdeal);
+ assertEquals(segmentStatusInfos.get(0).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.UPDATING);
+ assertEquals(segmentStatusInfos.get(1).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.UPDATING);
+ }
+
+ @Test
+ public void testAllSegmentsGoodBadOfflineTable() {
+ TableViews.TableView tableViewExternal = new TableViews.TableView();
+ TableViews.TableView tableViewIdeal = new TableViews.TableView();
+ Map<String, Map<String, String>> tableViewExternalOffline = new
TreeMap<>();
+ Map<String, Map<String, String>> tableViewIdealOffline = new TreeMap<>();
+ Map<String, String> testSegment1MapExternal = new LinkedHashMap<>();
+ Map<String, String> testSegment2MapExternal = new LinkedHashMap<>();
+ testSegment1MapExternal.put("Server1", "OFFLINE");
+ testSegment2MapExternal.put("Server2", "ERROR");
+ tableViewExternalOffline.put("TestSegment1", testSegment1MapExternal);
+ tableViewExternalOffline.put("TestSegment2", testSegment2MapExternal);
+ Map<String, String> testSegment1MapIdeal = new LinkedHashMap<>();
+ testSegment1MapIdeal.put("Server1", "OFFLINE");
+ Map<String, String> testSegment2MapIdeal = new LinkedHashMap<>();
+ testSegment2MapIdeal.put("Server2", "ERROR");
+ tableViewIdealOffline.put("TestSegment1", testSegment1MapIdeal);
+ tableViewIdealOffline.put("TestSegment2", testSegment2MapIdeal);
+ tableViewExternal._offline = tableViewExternalOffline;
+ tableViewIdeal._offline = tableViewIdealOffline;
+ TableViews tableviews = new TableViews();
+ List<SegmentStatusInfo> segmentStatusInfos =
tableviews.getSegmentStatuses(tableViewExternal, tableViewIdeal);
+ assertEquals(segmentStatusInfos.get(0).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.GOOD);
+ assertEquals(segmentStatusInfos.get(1).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.BAD);
+ }
+
+ @Test
+ public void testJsonDeserializationSegmentStatusInfo()
+ throws Exception {
+ // JSON string representing SchemaInfo
+ String json = "[\n" + " {\n" + " \"segmentStatus\": \"GOOD\",\n"
+ + " \"segmentName\": \"airlineStats_OFFLINE_16071_16071_0\"\n" + "
},\n" + " {\n"
+ + " \"segmentStatus\": \"BAD\",\n" + " \"segmentName\":
\"airlineStats_OFFLINE_16072_16072_0\"\n"
+ + " },\n" + " {\n" + " \"segmentStatus\": \"UPDATING\",\n"
+ + " \"segmentName\": \"airlineStats_OFFLINE_16073_16073_0\"\n" + "
}\n" + "]";
+ JsonNode jsonNode = JsonUtils.stringToJsonNode(json);
+ List<SegmentStatusInfo> segmentStatusInfos =
+ JsonUtils.jsonNodeToObject(jsonNode, new
TypeReference<List<SegmentStatusInfo>>() {
+ });
+ // Assertions
+ assertEquals(segmentStatusInfos.size(), 3);
+ assertEquals(segmentStatusInfos.get(0).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.GOOD);
+ assertEquals(segmentStatusInfos.get(0).getSegmentName(),
"airlineStats_OFFLINE_16071_16071_0");
+ assertEquals(segmentStatusInfos.get(1).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.BAD);
+ assertEquals(segmentStatusInfos.get(1).getSegmentName(),
"airlineStats_OFFLINE_16072_16072_0");
+ assertEquals(segmentStatusInfos.get(2).getSegmentStatus(),
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.UPDATING);
+ assertEquals(segmentStatusInfos.get(2).getSegmentName(),
"airlineStats_OFFLINE_16073_16073_0");
+ }
+
+ @Test
+ public void testJsonSerializationSegmentStatusInfo()
+ throws Exception {
+ SegmentStatusInfo statusInfo1 = new
SegmentStatusInfo("airlineStats_OFFLINE_16071_16071_0",
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.GOOD);
+ SegmentStatusInfo statusInfo2 = new
SegmentStatusInfo("airlineStats_OFFLINE_16072_16072_0",
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.BAD);
+ SegmentStatusInfo statusInfo3 = new
SegmentStatusInfo("airlineStats_OFFLINE_16073_16073_0",
+ CommonConstants.Helix.StateModel.DisplaySegmentStatus.UPDATING);
+ List<SegmentStatusInfo> segmentStatusInfoList = new ArrayList<>();
+ segmentStatusInfoList.add(statusInfo1);
+ segmentStatusInfoList.add(statusInfo2);
+ segmentStatusInfoList.add(statusInfo3);
+ String json =
+ "[ {\n" + " \"segmentName\" :
\"airlineStats_OFFLINE_16071_16071_0\",\n" + " \"segmentStatus\" : \"GOOD\"\n"
+ + "}, {\n" + " \"segmentName\" :
\"airlineStats_OFFLINE_16072_16072_0\",\n"
+ + " \"segmentStatus\" : \"BAD\"\n" + "}, {\n"
+ + " \"segmentName\" : \"airlineStats_OFFLINE_16073_16073_0\",\n"
+ " \"segmentStatus\" : \"UPDATING\"\n"
+ + "} ]";
+ String jsonString = JsonUtils.objectToPrettyString(segmentStatusInfoList);
+ assertEquals(jsonString, json);
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java
b/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java
index 62f1b2e4d6..9d0695a285 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/auth/Actions.java
@@ -58,6 +58,7 @@ public class Actions {
public static final String GET_RUNNING_QUERY = "GetRunningQuery";
public static final String GET_SCHEDULER_INFO = "GetSchedulerInfo";
public static final String GET_SCHEMA = "GetSchema";
+ public static final String GET_SCHEMAS_INFO = "GetSchemasInfo";
public static final String GET_SEGMENT = "GetSegment";
public static final String GET_SEGMENT_RELOAD_STATUS =
"GetSegmentReloadStatus";
public static final String GET_SERVER_ROUTING_STATS =
"GetServerRoutingStats";
@@ -66,6 +67,7 @@ public class Actions {
public static final String GET_TABLE_CONFIG = "GetTableConfig";
public static final String GET_TABLE_LEADER = "GetTableLeader";
public static final String GET_TASK = "GetTask";
+ public static final String GET_TASK_COUNT = "GetTaskCount";
public static final String GET_TENANT = "GetTenant";
public static final String GET_USER = "GetUser";
public static final String GET_VERSION = "GetVersion";
@@ -116,6 +118,7 @@ public class Actions {
public static final String GET_CONTROLLER_JOBS = "GetControllerJobs";
public static final String GET_DEBUG_INFO = "GetDebugInfo";
public static final String GET_EXTERNAL_VIEW = "GetExternalView";
+ public static final String GET_SEGMENT_STATUS = "GetSegmentStatus";
public static final String GET_IDEAL_STATE = "GetIdealState";
public static final String GET_INSTANCE = "GetInstance";
public static final String GET_INSTANCE_PARTITIONS =
"GetInstancePartitions";
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/SchemaInfo.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/SchemaInfo.java
new file mode 100644
index 0000000000..5949fcdcad
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/SchemaInfo.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.data;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+
+/**
+ * This class gives the details of a particular schema and the corresponding
column count metrics
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SchemaInfo {
+ @JsonProperty("schemaName")
+ private String _schemaName;
+
+ @JsonProperty("numDimensionFields")
+ private int _numDimensionFields;
+
+ @JsonProperty("numDateTimeFields")
+ private int _numDateTimeFields;
+
+ @JsonProperty("numMetricFields")
+ private int _numMetricFields;
+
+ public String getSchemaName() {
+ return _schemaName;
+ }
+
+ public int getNumDimensionFields() {
+ return _numDimensionFields;
+ }
+
+ public int getNumDateTimeFields() {
+ return _numDateTimeFields;
+ }
+
+ public int getNumMetricFields() {
+ return _numMetricFields;
+ }
+
+ public SchemaInfo() {
+ }
+
+ public SchemaInfo(Schema schema) {
+ _schemaName = schema.getSchemaName();
+
+ //Removed virtual columns($docId, $hostName, $segmentName) from dimension
fields count
+ _numDimensionFields = schema.getDimensionFieldSpecs().size() - 3;
+ _numDateTimeFields = schema.getDateTimeFieldSpecs().size();
+ _numMetricFields = schema.getMetricFieldSpecs().size();
+ }
+}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index 21e5c87ee7..c1c4dbce49 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -139,6 +139,12 @@ public class CommonConstants {
public static final String CONSUMING = "CONSUMING";
}
+ public static class DisplaySegmentStatus {
+ public static final String BAD = "BAD";
+ public static final String GOOD = "GOOD";
+ public static final String UPDATING = "UPDATING";
+ }
+
public static class BrokerResourceStateModel {
public static final String ONLINE = "ONLINE";
public static final String OFFLINE = "OFFLINE";
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaInfoTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaInfoTest.java
new file mode 100644
index 0000000000..5f31e19ed8
--- /dev/null
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaInfoTest.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.data;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.Test;
+
+import static org.apache.pinot.spi.data.FieldSpec.DataType.INT;
+import static org.testng.Assert.assertEquals;
+
+
+public class SchemaInfoTest {
+
+ @Test
+ public void testSchemaInfoSerDeserWithVirtualColumns()
+ throws IOException {
+ // Mock the Schema objects
+ Schema schemaMock =
+ new
Schema.SchemaBuilder().setSchemaName("TestSchema").addDimensionField("dim1",
FieldSpec.DataType.STRING)
+ .addDimensionField("dim2",
FieldSpec.DataType.INT).addDimensionField("dim3", FieldSpec.DataType.INT)
+
.addDimensionField(CommonConstants.Segment.BuiltInVirtualColumn.DOCID,
FieldSpec.DataType.INT)
+
.addDimensionField(CommonConstants.Segment.BuiltInVirtualColumn.HOSTNAME,
FieldSpec.DataType.STRING)
+
.addDimensionField(CommonConstants.Segment.BuiltInVirtualColumn.SEGMENTNAME,
FieldSpec.DataType.STRING)
+ .addDateTimeField("dt1", FieldSpec.DataType.LONG, "1:HOURS:EPOCH",
"1:HOURS")
+ .addDateTimeField("dt2", FieldSpec.DataType.LONG, "1:HOURS:EPOCH",
"1:HOURS").addMetricField("metric", INT)
+ .build();
+ SchemaInfo schemaInfo = new SchemaInfo(schemaMock);
+ List<SchemaInfo> schemaInfoList = new ArrayList<>();
+ schemaInfoList.add(schemaInfo);
+ String response = JsonUtils.objectToPrettyString(schemaInfoList);
+ JsonNode jsonNodeResp = JsonUtils.stringToJsonNode(response);
+
+ // Test deserialization
+ assertEquals(jsonNodeResp.get(0).get("schemaName").asText(), "TestSchema");
+ assertEquals(jsonNodeResp.get(0).get("numDimensionFields").asInt(), 3);
+ assertEquals(jsonNodeResp.get(0).get("numDateTimeFields").asInt(), 2);
+ assertEquals(jsonNodeResp.get(0).get("numMetricFields").asInt(), 1);
+ assertEquals(schemaInfo.getSchemaName(), "TestSchema");
+
+ // Test column count
+ assertEquals(schemaInfo.getNumDimensionFields(), 3); // 6 - 3 virtual
columns = 3
+ assertEquals(schemaInfo.getNumDateTimeFields(), 2);
+ assertEquals(schemaInfo.getNumMetricFields(), 1);
+
+ // Serialize JsonNode back to SchemaInfo
+ List<SchemaInfo> schemaInfoListSer = new ArrayList<>();
+ schemaInfoListSer = JsonUtils.jsonNodeToObject(jsonNodeResp, new
TypeReference<List<SchemaInfo>>() {
+ });
+ SchemaInfo schemaInfo1 = schemaInfoListSer.get(0);
+ // Verify the deserialized object match
+ assertEquals(schemaInfo1.getSchemaName(), "TestSchema");
+ assertEquals(schemaInfo1.getNumDimensionFields(), 3);
+ assertEquals(schemaInfo1.getNumDateTimeFields(), 2);
+ assertEquals(schemaInfo1.getNumMetricFields(), 1);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
