This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c781ff1fff Add a button to view consuming segments info directly in
table UI (#15623)
c781ff1fff is described below
commit c781ff1fff5bd770311d9da85f22c1df9f10077c
Author: Kartik Khare <[email protected]>
AuthorDate: Fri Apr 25 17:56:27 2025 +0530
Add a button to view consuming segments info directly in table UI (#15623)
---
.../app/components/ConsumingSegmentsTable.tsx | 110 +++++++++++++++++++++
.../src/main/resources/app/interfaces/types.d.ts | 25 +++++
.../src/main/resources/app/pages/TenantDetails.tsx | 82 ++++++++++++---
.../src/main/resources/app/requests/index.ts | 11 ++-
.../main/resources/app/utils/PinotMethodUtils.ts | 13 ++-
5 files changed, 226 insertions(+), 15 deletions(-)
diff --git
a/pinot-controller/src/main/resources/app/components/ConsumingSegmentsTable.tsx
b/pinot-controller/src/main/resources/app/components/ConsumingSegmentsTable.tsx
new file mode 100644
index 0000000000..4b940c1731
--- /dev/null
+++
b/pinot-controller/src/main/resources/app/components/ConsumingSegmentsTable.tsx
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import React from 'react';
+import { Typography, List, ListItem, ListItemText } from '@material-ui/core';
+import CustomizedTables from '../components/Table';
+import type { ConsumingSegmentsInfo } from 'Models';
+
+type Props = {
+ info: ConsumingSegmentsInfo;
+};
+
+const ConsumingSegmentsTable: React.FC<Props> = ({ info }) => {
+ const segmentMap = info?._segmentToConsumingInfoMap ?? {};
+ const entries = Object.entries(segmentMap);
+
+ if (entries.length === 0) {
+ return <Typography>No consuming segment data available.</Typography>;
+ }
+
+ const columns = [
+ 'Segment Name',
+ 'Server Details',
+ 'Max Partition Offset Lag',
+ 'Max Partition Availability Lag (ms)',
+ ];
+
+ const records = entries
+ .map(([segment, infos]) => {
+ const list = infos ?? [];
+ if (list.length === 0) {
+ return null;
+ }
+ let segmentMaxOffsetLag = 0;
+ let segmentMaxAvailabilityLag = 0;
+
+ const serverDetails = list.map((item) => {
+ const lags = Object.values(item.partitionOffsetInfo?.recordsLagMap ??
{});
+ const avails =
Object.values(item.partitionOffsetInfo?.availabilityLagMsMap ?? {});
+ const serverLag = lags
+ .filter((lag) => lag != null && Number(lag) > -1)
+ .reduce((max, lag) => Math.max(max, Number(lag)), 0);
+ const serverAvail = avails
+ .filter((av) => av != null && Number(av) > -1)
+ .reduce((max, av) => Math.max(max, Number(av)), 0);
+ segmentMaxOffsetLag = Math.max(segmentMaxOffsetLag, serverLag);
+ segmentMaxAvailabilityLag = Math.max(segmentMaxAvailabilityLag,
serverAvail);
+ return {
+ serverName: item.serverName,
+ consumerState: item.consumerState,
+ lag: serverLag,
+ availabilityMs: serverAvail,
+ };
+ });
+ if (serverDetails.length === 0) {
+ return null;
+ }
+ return [
+ segment,
+ {
+ customRenderer: (
+ <List dense disablePadding style={{ width: '100%' }}>
+ {serverDetails.map((detail, idx) => (
+ <ListItem key={idx} dense disableGutters style={{ padding:
'2px 0' }}>
+ <ListItemText
+ primary={`${detail.serverName}: ${detail.consumerState}`}
+ secondary={`Lag: ${detail.lag}, Availability:
${detail.availabilityMs}ms`}
+ primaryTypographyProps={{ variant: 'body2', style: {
fontWeight: 500 } }}
+ secondaryTypographyProps={{ variant: 'caption', color:
'textSecondary' }}
+ />
+ </ListItem>
+ ))}
+ </List>
+ ),
+ },
+ segmentMaxOffsetLag,
+ segmentMaxAvailabilityLag,
+ ];
+ })
+ .filter(Boolean) as any[];
+
+ if (records.length === 0) {
+ return <Typography>Consuming segment data found, but no server details
available.</Typography>;
+ }
+
+ return (
+ <CustomizedTables
+ title="Consuming Segments Summary"
+ data={{ columns, records }}
+ showSearchBox={true}
+ />
+ );
+};
+
+export default ConsumingSegmentsTable;
\ No newline at end of file
diff --git a/pinot-controller/src/main/resources/app/interfaces/types.d.ts
b/pinot-controller/src/main/resources/app/interfaces/types.d.ts
index 192b83cce8..2dbfb26893 100644
--- a/pinot-controller/src/main/resources/app/interfaces/types.d.ts
+++ b/pinot-controller/src/main/resources/app/interfaces/types.d.ts
@@ -128,6 +128,31 @@ declare module 'Models' {
export type QuerySchemas = Array<string>;
+ /**
+ * Information about a consuming segment on a server
+ */
+ export interface ConsumingInfo {
+ serverName: string;
+ consumerState: string;
+ lastConsumedTimestamp: number;
+ partitionToOffsetMap: Record<string, string>;
+ partitionOffsetInfo: {
+ currentOffsetsMap: Record<string, string>;
+ latestUpstreamOffsetMap: Record<string, string>;
+ recordsLagMap: Record<string, string>;
+ availabilityLagMsMap: Record<string, string>;
+ };
+ }
+
+ /**
+ * Consuming segments information for a table
+ */
+ export interface ConsumingSegmentsInfo {
+ serversFailingToRespond: number;
+ serversUnparsableRespond: number;
+ _segmentToConsumingInfoMap: Record<string, ConsumingInfo[]>;
+ }
+
export type TableSchema = {
dimensionFieldSpecs: Array<schema>;
metricFieldSpecs?: Array<schema>;
diff --git a/pinot-controller/src/main/resources/app/pages/TenantDetails.tsx
b/pinot-controller/src/main/resources/app/pages/TenantDetails.tsx
index d091a8187a..94952c2734 100644
--- a/pinot-controller/src/main/resources/app/pages/TenantDetails.tsx
+++ b/pinot-controller/src/main/resources/app/pages/TenantDetails.tsx
@@ -19,10 +19,10 @@
import React, { useState, useEffect, useRef } from 'react';
import { makeStyles } from '@material-ui/core/styles';
-import { Box, Button, Checkbox, FormControlLabel, Grid, Switch, Tooltip,
Typography } from '@material-ui/core';
+import { Box, Button, Checkbox, FormControlLabel, Grid, Switch, Tooltip,
Typography, CircularProgress } from '@material-ui/core';
import { RouteComponentProps, useHistory, useLocation } from
'react-router-dom';
import { UnControlled as CodeMirror } from 'react-codemirror2';
-import { DISPLAY_SEGMENT_STATUS, InstanceState, TableData, TableSegmentJobs,
TableType } from 'Models';
+import { DISPLAY_SEGMENT_STATUS, InstanceState, TableData, TableSegmentJobs,
TableType, ConsumingSegmentsInfo } from 'Models';
import AppLoader from '../components/AppLoader';
import CustomizedTables from '../components/Table';
import TableToolbar from '../components/TableToolbar';
@@ -37,6 +37,7 @@ import EditConfigOp from
'../components/Homepage/Operations/EditConfigOp';
import ReloadStatusOp from '../components/Homepage/Operations/ReloadStatusOp';
import RebalanceServerTableOp from
'../components/Homepage/Operations/RebalanceServerTableOp';
import Confirm from '../components/Confirm';
+import CustomDialog from '../components/CustomDialog';
import { NotificationContext } from
'../components/Notification/NotificationContext';
import Utils from '../utils/Utils';
import InfoOutlinedIcon from '@material-ui/icons/InfoOutlined';
@@ -47,6 +48,7 @@ import NotFound from '../components/NotFound';
import {
RebalanceServerStatusOp
} from "../components/Homepage/Operations/RebalanceServerStatusOp";
+import ConsumingSegmentsTable from '../components/ConsumingSegmentsTable';
const useStyles = makeStyles((theme) => ({
root: {
@@ -155,6 +157,10 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
const [showRebalanceServerModal, setShowRebalanceServerModal] =
useState(false);
const [schemaJSONFormat, setSchemaJSONFormat] = useState(false);
const [showRebalanceServerStatus, setShowRebalanceServerStatus] =
useState(false);
+ // State for consuming segments info
+ const [showConsumingSegmentsModal, setShowConsumingSegmentsModal] =
useState(false);
+ const [loadingConsumingSegments, setLoadingConsumingSegments] =
useState(false);
+ const [consumingSegmentsInfo, setConsumingSegmentsInfo] =
useState<ConsumingSegmentsInfo | null>(null);
// This is quite hacky, but it's the only way to get this to work with the
dialog.
// The useState variables are simply for the dialog box to know what to
render in
@@ -412,18 +418,18 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
const customMessage = (
<Box>
<Typography variant='inherit'>{result.status}</Typography>
- <Button
- className={classes.copyIdButton}
- variant="outlined"
- color="inherit"
- size="small"
+ <Button
+ className={classes.copyIdButton}
+ variant="outlined"
+ color="inherit"
+ size="small"
onClick={handleCopyReloadJobId}
>
Copy Id
</Button>
</Box>
)
-
+
syncResponse(result, reloadJobId && customMessage);
};
@@ -440,7 +446,7 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
setShowReloadStatusModal(false);
return;
}
-
+
setReloadStatusData(reloadStatusData);
setTableJobsData(tableJobsData);
} catch(error) {
@@ -452,6 +458,20 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
const handleRebalanceTableStatus = () => {
setShowRebalanceServerStatus(true);
};
+ // Handler to view consuming segments info
+ const handleViewConsumingSegments = async () => {
+ setShowConsumingSegmentsModal(true);
+ setLoadingConsumingSegments(true);
+ try {
+ const data = await
PinotMethodUtils.getConsumingSegmentsInfoData(tableName);
+ setConsumingSegmentsInfo(data);
+ } catch (error) {
+ dispatch({ type: 'error', message: `Error fetching consuming segments
info: ${error}` });
+ setShowConsumingSegmentsModal(false);
+ } finally {
+ setLoadingConsumingSegments(false);
+ }
+ };
const handleRebalanceBrokers = () => {
setDialogDetails({
@@ -461,7 +481,7 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
});
setConfirmDialog(true);
};
-
+
const rebalanceBrokers = async () => {
const result = await
PinotMethodUtils.rebalanceBrokersForTableOp(tableName);
syncResponse(result);
@@ -586,6 +606,16 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
Repair Table
</CustomButton>
)}
+ {/* Button to view consuming segments info */}
+ {tableType.toLowerCase() === TableType.REALTIME && (
+ <CustomButton
+ onClick={handleViewConsumingSegments}
+ tooltipTitle="View offset and lag information about consuming
segments"
+ enableTooltip={true}
+ >
+ View Consuming Segments
+ </CustomButton>
+ )}
<Tooltip title="Disabling will disable the table for queries,
consumption and data push" arrow placement="top">
<FormControlLabel
control={
@@ -653,7 +683,7 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
autoCursor={false}
/>
</SimpleAccordion>
- </div>
+ </div>
<CustomizedTables
title={"Segments - " + segmentList.records.length}
data={segmentList}
@@ -743,6 +773,36 @@ const TenantPageDetails = ({ match }:
RouteComponentProps<Props>) => {
tableName={tableName}
/>
)}
+ {/* Consuming Segments Info Dialog */}
+ {showConsumingSegmentsModal && (
+ <CustomDialog
+ open={showConsumingSegmentsModal}
+ handleClose={() => setShowConsumingSegmentsModal(false)}
+ title="Consuming Segments Info"
+ size="lg"
+ showOkBtn={false}
+ btnCancelText="Close"
+ disableBackdropClick
+ >
+ {loadingConsumingSegments && (
+ <Box display="flex" justifyContent="center">
+ <CircularProgress />
+ </Box>
+ )}
+ {!loadingConsumingSegments && consumingSegmentsInfo && (
+ <Box style={{ height: '100%', overflowY: 'auto' }}>
+ <Typography><strong>Servers Failing To Respond:</strong>
{consumingSegmentsInfo?.serversFailingToRespond ?? 'N/A'}</Typography>
+ <Typography><strong>Servers Unparsable Respond:</strong>
{consumingSegmentsInfo?.serversUnparsableRespond ?? 'N/A'}</Typography>
+ <Box mt={2}>
+ <ConsumingSegmentsTable info={consumingSegmentsInfo} />
+ </Box>
+ </Box>
+ )}
+ {!loadingConsumingSegments && !consumingSegmentsInfo && (
+ <Typography>No consuming segments data available.</Typography>
+ )}
+ </CustomDialog>
+ )}
{confirmDialog && dialogDetails && (
<Confirm
openDialog={confirmDialog}
diff --git a/pinot-controller/src/main/resources/app/requests/index.ts
b/pinot-controller/src/main/resources/app/requests/index.ts
index 93a64914b7..2faddda1fa 100644
--- a/pinot-controller/src/main/resources/app/requests/index.ts
+++ b/pinot-controller/src/main/resources/app/requests/index.ts
@@ -45,10 +45,12 @@ import {
SegmentDebugDetails,
QuerySchemas,
TableType,
- InstanceState, SegmentMetadata,
+ InstanceState,
+ SegmentMetadata,
SchemaInfo,
SegmentStatusInfo,
- ServerToSegmentsCount
+ ServerToSegmentsCount,
+ ConsumingSegmentsInfo
} from 'Models';
const headers = {
@@ -110,6 +112,11 @@ export const getServerToSegmentsCount = (name: string,
tableType: TableType, ver
export const getSegmentsStatus = (name: string):
Promise<AxiosResponse<SegmentStatusInfo[]>> =>
baseApi.get(`/tables/${name}/segmentsStatus`);
+// Fetch consuming segments information for a table
+// API: GET /tables/{tableName}/consumingSegmentsInfo
+export const getConsumingSegmentsInfo = (name: string):
Promise<AxiosResponse<ConsumingSegmentsInfo>> =>
+ baseApi.get(`/tables/${name}/consumingSegmentsInfo`);
+
export const getInstances = (): Promise<AxiosResponse<Instances>> =>
baseApi.get('/instances');
diff --git a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
index 48a0f43fdc..5296493676 100644
--- a/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
+++ b/pinot-controller/src/main/resources/app/utils/PinotMethodUtils.ts
@@ -108,6 +108,7 @@ import {
getTaskRuntimeConfig,
getSchemaInfo,
getSegmentsStatus,
+ getConsumingSegmentsInfo,
getServerToSegmentsCount
} from '../requests';
import { baseApi } from './axios-config';
@@ -431,7 +432,14 @@ const getAllSchemaDetails = async (schemaList) => {
columns: allSchemaDetailsColumnHeader,
records: schemaDetails
};
-}
+};
+
+// Fetch consuming segments info for a given table
+// API: /tables/{tableName}/consumingSegmentsInfo
+// Expected Output: ConsumingSegmentsInfo
+const getConsumingSegmentsInfoData = (tableName) => {
+ return getConsumingSegmentsInfo(tableName).then(({ data }) => data);
+};
const allTableDetailsColumnHeader = [
'Table Name',
@@ -1389,5 +1397,6 @@ export default {
updateUser,
getAuthUserNameFromAccessToken,
getAuthUserEmailFromAccessToken,
- fetchServerToSegmentsCountData
+ fetchServerToSegmentsCountData,
+ getConsumingSegmentsInfoData
};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
