This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 436c17c655 List Mapped Instances in Task Details Panel (#22691)
436c17c655 is described below
commit 436c17c655494eff5724df98d1a231ffa2142253
Author: Brent Bovenzi <[email protected]>
AuthorDate: Mon Apr 4 10:33:10 2022 -0500
List Mapped Instances in Task Details Panel (#22691)
* Add table of mapped instances to grid view
* Autorefresh, table width, remove extraneous code
* Switch to icon buttons
* Fix mapped instances url
* Only show dates when they exist and are final
---
airflow/www/package.json | 1 +
airflow/www/static/js/tree/Table.jsx | 176 +++++++++++++++++++++
airflow/www/static/js/tree/api/index.js | 2 +
.../www/static/js/tree/api/useMappedInstances.js | 46 ++++++
.../js/tree/details/content/dagRun/index.jsx | 4 +
.../tree/details/content/taskInstance/Details.jsx | 5 +
.../content/taskInstance/MappedInstances.jsx | 152 ++++++++++++++++++
.../js/tree/details/content/taskInstance/Nav.jsx | 11 --
.../js/tree/details/content/taskInstance/index.jsx | 4 +
airflow/www/static/js/tree/details/index.jsx | 15 +-
airflow/www/templates/airflow/dag.html | 1 +
airflow/www/yarn.lock | 5 +
12 files changed, 400 insertions(+), 22 deletions(-)
diff --git a/airflow/www/package.json b/airflow/www/package.json
index 9ee33c94cd..d15cd05d9e 100644
--- a/airflow/www/package.json
+++ b/airflow/www/package.json
@@ -95,6 +95,7 @@
"react-dom": "^17.0.2",
"react-icons": "^4.3.1",
"react-query": "^3.34.16",
+ "react-table": "^7.7.0",
"redoc": "^2.0.0-rc.63",
"url-search-params-polyfill": "^8.1.0"
},
diff --git a/airflow/www/static/js/tree/Table.jsx
b/airflow/www/static/js/tree/Table.jsx
new file mode 100644
index 0000000000..06eb84cad4
--- /dev/null
+++ b/airflow/www/static/js/tree/Table.jsx
@@ -0,0 +1,176 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Custom wrapper of react-table using Chakra UI components
+*/
+
+import React, { useEffect } from 'react';
+import {
+ Flex,
+ Table as ChakraTable,
+ Thead,
+ Tbody,
+ Tr,
+ Th,
+ Td,
+ IconButton,
+ Text,
+ useColorModeValue,
+} from '@chakra-ui/react';
+import {
+ useTable, useSortBy, usePagination,
+} from 'react-table';
+import {
+ MdKeyboardArrowLeft, MdKeyboardArrowRight,
+} from 'react-icons/md';
+import {
+ TiArrowUnsorted, TiArrowSortedDown, TiArrowSortedUp,
+} from 'react-icons/ti';
+
+const Table = ({
+ data, columns, manualPagination, pageSize = 25, setSortBy, isLoading = false,
+}) => {
+ const { totalEntries, offset, setOffset } = manualPagination || {};
+ const oddColor = useColorModeValue('gray.50', 'gray.900');
+ const hoverColor = useColorModeValue('gray.100', 'gray.700');
+
+ const pageCount = totalEntries ? (Math.ceil(totalEntries / pageSize) || 1) :
data.length;
+
+ const lowerCount = (offset || 0) + 1;
+ const upperCount = lowerCount + data.length - 1;
+
+ const {
+ getTableProps,
+ getTableBodyProps,
+ allColumns,
+ prepareRow,
+ page,
+ canPreviousPage,
+ canNextPage,
+ nextPage,
+ previousPage,
+ state: { pageIndex, sortBy },
+ } = useTable(
+ {
+ columns,
+ data,
+ pageCount,
+ manualPagination: !!manualPagination,
+ manualSortBy: !!setSortBy,
+ initialState: {
+ pageIndex: offset ? offset / pageSize : 0,
+ pageSize,
+ },
+ },
+ useSortBy,
+ usePagination,
+ );
+
+ const handleNext = () => {
+ nextPage();
+ if (setOffset) setOffset((pageIndex + 1) * pageSize);
+ };
+
+ const handlePrevious = () => {
+ previousPage();
+ if (setOffset) setOffset((pageIndex - 1 || 0) * pageSize);
+ };
+
+ useEffect(() => {
+ if (setSortBy) setSortBy(sortBy);
+ }, [sortBy, setSortBy]);
+
+ return (
+ <>
+ <ChakraTable {...getTableProps()}>
+ <Thead>
+ <Tr>
+ {allColumns.map((column) => (
+ <Th
+ {...column.getHeaderProps(column.getSortByToggleProps())}
+ >
+ {column.render('Header')}
+ {column.isSorted && (
+ column.isSortedDesc ? (
+ <TiArrowSortedDown aria-label="sorted descending" style={{
display: 'inline' }} size="1em" />
+ ) : (
+ <TiArrowSortedUp aria-label="sorted ascending" style={{
display: 'inline' }} size="1em" />
+ )
+ )}
+ {(!column.isSorted && column.canSort) && (<TiArrowUnsorted
aria-label="unsorted" style={{ display: 'inline' }} size="1em" />)}
+ </Th>
+ ))}
+ </Tr>
+ </Thead>
+ <Tbody {...getTableBodyProps()}>
+ {!data.length && !isLoading && (
+ <Tr>
+ <Td colSpan={2}>No Data found.</Td>
+ </Tr>
+ )}
+ {page.map((row) => {
+ prepareRow(row);
+ return (
+ <Tr
+ {...row.getRowProps()}
+ _odd={{ backgroundColor: oddColor }}
+ _hover={{ backgroundColor: hoverColor }}
+ >
+ {row.cells.map((cell) => (
+ <Td
+ {...cell.getCellProps()}
+ py={3}
+ >
+ {cell.render('Cell')}
+ </Td>
+ ))}
+ </Tr>
+ );
+ })}
+ </Tbody>
+ </ChakraTable>
+ <Flex alignItems="center" justifyContent="flex-start" my={4}>
+ <IconButton
+ variant="ghost"
+ onClick={handlePrevious}
+ disabled={!canPreviousPage}
+ aria-label="Previous Page"
+ icon={<MdKeyboardArrowLeft />}
+ />
+ <IconButton
+ variant="ghost"
+ onClick={handleNext}
+ disabled={!canNextPage}
+ aria-label="Next Page"
+ icon={<MdKeyboardArrowRight />}
+ />
+ <Text>
+ {lowerCount}
+ -
+ {upperCount}
+ {' of '}
+ {totalEntries}
+ </Text>
+ </Flex>
+ </>
+ );
+};
+
+export default Table;
diff --git a/airflow/www/static/js/tree/api/index.js
b/airflow/www/static/js/tree/api/index.js
index 1c169c91f5..aea1e918f8 100644
--- a/airflow/www/static/js/tree/api/index.js
+++ b/airflow/www/static/js/tree/api/index.js
@@ -33,6 +33,7 @@ import useMarkSuccessTask from './useMarkSuccessTask';
import useExtraLinks from './useExtraLinks';
import useConfirmMarkTask from './useConfirmMarkTask';
import useTreeData from './useTreeData';
+import useMappedInstances from './useMappedInstances';
axios.interceptors.response.use(
(res) => (res.data ? camelcaseKeys(res.data, { deep: true }) : res),
@@ -54,4 +55,5 @@ export {
useExtraLinks,
useConfirmMarkTask,
useTreeData,
+ useMappedInstances,
};
diff --git a/airflow/www/static/js/tree/api/useMappedInstances.js
b/airflow/www/static/js/tree/api/useMappedInstances.js
new file mode 100644
index 0000000000..35eb1decb6
--- /dev/null
+++ b/airflow/www/static/js/tree/api/useMappedInstances.js
@@ -0,0 +1,46 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* global autoRefreshInterval */
+
+import axios from 'axios';
+import { useQuery } from 'react-query';
+
+import { getMetaValue } from '../../utils';
+import { useAutoRefresh } from '../context/autorefresh';
+
+const mappedInstancesUrl = getMetaValue('mapped_instances_api');
+
+export default function useMappedInstances({
+ dagId, runId, taskId, limit, offset, order,
+}) {
+ const url = mappedInstancesUrl.replace('_DAG_RUN_ID_',
runId).replace('_TASK_ID_', taskId);
+ const orderParam = order && order !== 'map_index' ? { order_by: order } : {};
+ const { isRefreshOn } = useAutoRefresh();
+ return useQuery(
+ ['mappedInstances', dagId, runId, taskId, offset, order],
+ () => axios.get(url, {
+ params: { offset, limit, ...orderParam },
+ }),
+ {
+ keepPreviousData: true,
+ refetchInterval: isRefreshOn && autoRefreshInterval * 1000,
+ },
+ );
+}
diff --git a/airflow/www/static/js/tree/details/content/dagRun/index.jsx
b/airflow/www/static/js/tree/details/content/dagRun/index.jsx
index 8a8b3bd705..68e6d6526a 100644
--- a/airflow/www/static/js/tree/details/content/dagRun/index.jsx
+++ b/airflow/www/static/js/tree/details/content/dagRun/index.jsx
@@ -126,16 +126,20 @@ const DagRun = ({ runId }) => {
<Time dateTime={dataIntervalEnd} />
</Text>
<br />
+ {startDate && (
<Text>
Started:
{' '}
<Time dateTime={startDate} />
</Text>
+ )}
+ {endDate && (
<Text>
Ended:
{' '}
<Time dateTime={endDate} />
</Text>
+ )}
</Box>
);
};
diff --git
a/airflow/www/static/js/tree/details/content/taskInstance/Details.jsx
b/airflow/www/static/js/tree/details/content/taskInstance/Details.jsx
index f84996a09f..44294f51f7 100644
--- a/airflow/www/static/js/tree/details/content/taskInstance/Details.jsx
+++ b/airflow/www/static/js/tree/details/content/taskInstance/Details.jsx
@@ -89,6 +89,7 @@ const Details = ({ instance, task }) => {
}
const taskIdTitle = isGroup ? 'Task Group Id: ' : 'Task Id: ';
+ const isStateFinal = ['success', 'failed', 'upstream_failed',
'skipped'].includes(state);
return (
<Flex flexWrap="wrap" justifyContent="space-between">
@@ -144,16 +145,20 @@ const Details = ({ instance, task }) => {
</Text>
</Box>
<Box>
+ {startDate && (
<Text>
Started:
{' '}
<Time dateTime={startDate} />
</Text>
+ )}
+ {endDate && isStateFinal && (
<Text>
Ended:
{' '}
<Time dateTime={endDate} />
</Text>
+ )}
</Box>
</Flex>
);
diff --git
a/airflow/www/static/js/tree/details/content/taskInstance/MappedInstances.jsx
b/airflow/www/static/js/tree/details/content/taskInstance/MappedInstances.jsx
new file mode 100644
index 0000000000..b815f0987f
--- /dev/null
+++
b/airflow/www/static/js/tree/details/content/taskInstance/MappedInstances.jsx
@@ -0,0 +1,152 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useState, useMemo } from 'react';
+import {
+ Flex,
+ Text,
+ Box,
+ Link,
+ IconButton,
+} from '@chakra-ui/react';
+import { snakeCase } from 'lodash';
+import { FaMicroscope } from 'react-icons/fa';
+import { GiLog } from 'react-icons/gi';
+import { HiTemplate } from 'react-icons/hi';
+
+import { getMetaValue } from '../../../../utils';
+import { formatDateTime, formatDuration } from '../../../../datetime_utils';
+import { useMappedInstances } from '../../../api';
+import { SimpleStatus } from '../../../StatusBox';
+import Table from '../../../Table';
+
+const renderedTemplatesUrl = getMetaValue('rendered_templates_url');
+const logUrl = getMetaValue('log_url');
+const taskUrl = getMetaValue('task_url');
+
+const IconLink = (props) => (
+ <IconButton as={Link} variant="outline" colorScheme="blue" {...props} />
+);
+
+const MappedInstances = ({
+ dagId, runId, taskId,
+}) => {
+ const limit = 25;
+ const [offset, setOffset] = useState(0);
+ const [sortBy, setSortBy] = useState([]);
+
+ const sort = sortBy[0];
+
+ const order = sort && (sort.id === 'state' || sort.id === 'mapIndex') ?
`${sort.desc ? '-' : ''}${snakeCase(sort.id)}` : '';
+
+ const {
+ data: { taskInstances, totalEntries } = { taskInstances: [], totalEntries:
0 },
+ isLoading,
+ } = useMappedInstances({
+ dagId, runId, taskId, limit, offset, order,
+ });
+
+ const data = useMemo(
+ () => taskInstances.map((mi) => {
+ const params = new URLSearchParams({
+ dag_id: dagId,
+ task_id: mi.taskId,
+ execution_date: mi.executionDate,
+ map_index: mi.mapIndex,
+ }).toString();
+ const detailsLink = `${taskUrl}&${params}`;
+ const renderedLink = `${renderedTemplatesUrl}&${params}`;
+ const logLink = `${logUrl}&${params}`;
+ return {
+ ...mi,
+ state: (
+ <Flex alignItems="center">
+ <SimpleStatus state={mi.state} mx={2} />
+ {mi.state || 'no status'}
+ </Flex>
+ ),
+ duration: mi.duration && formatDuration(mi.duration),
+ startDate: mi.startDate && formatDateTime(mi.startDate),
+ endDate: mi.endDate && formatDateTime(mi.endDate),
+ links: (
+ <Flex alignItems="center">
+ <IconLink mr={1} title="Rendered Templates" aria-label="Rendered
Templates" icon={<HiTemplate />} href={renderedLink} />
+ <IconLink mr={1} title="Log" aria-label="Log" icon={<GiLog />}
href={logLink} />
+ <IconLink title="Details" aria-label="Details" icon={<FaMicroscope
/>} href={detailsLink} />
+ </Flex>
+ ),
+ };
+ }),
+ [dagId, taskInstances],
+ );
+
+ const columns = useMemo(
+ () => [
+ {
+ Header: 'Map Index',
+ accessor: 'mapIndex',
+ },
+ {
+ Header: 'State',
+ accessor: 'state',
+ },
+ {
+ Header: 'Duration',
+ accessor: 'duration',
+ disableSortBy: true,
+ },
+ {
+ Header: 'Start Date',
+ accessor: 'startDate',
+ disableSortBy: true,
+ },
+ {
+ Header: 'End Date',
+ accessor: 'endDate',
+ disableSortBy: true,
+ },
+ {
+ disableSortBy: true,
+ accessor: 'links',
+ },
+ ],
+ [],
+ );
+
+ return (
+ <Box>
+ <br />
+ <Text as="strong">Mapped Instances</Text>
+ <Table
+ data={data}
+ columns={columns}
+ manualPagination={{
+ offset,
+ setOffset,
+ totalEntries,
+ }}
+ pageSize={limit}
+ setSortBy={setSortBy}
+ isLoading={isLoading}
+ />
+ </Box>
+ );
+};
+
+export default MappedInstances;
diff --git a/airflow/www/static/js/tree/details/content/taskInstance/Nav.jsx
b/airflow/www/static/js/tree/details/content/taskInstance/Nav.jsx
index 6fc8eb6055..d0841d0119 100644
--- a/airflow/www/static/js/tree/details/content/taskInstance/Nav.jsx
+++ b/airflow/www/static/js/tree/details/content/taskInstance/Nav.jsx
@@ -44,7 +44,6 @@ const Nav = ({ instance, isMapped }) => {
const {
taskId,
dagId,
- runId,
operator,
executionDate,
} = instance;
@@ -62,12 +61,6 @@ const Nav = ({ instance, isMapped }) => {
_flt_3_task_id: taskId,
_oc_TaskInstanceModelView: 'dag_run.execution_date',
});
- const mapParams = new URLSearchParams({
- _flt_3_dag_id: dagId,
- _flt_3_task_id: taskId,
- _flt_3_run_id: runId,
- _oc_TaskInstanceModelView: 'map_index',
- });
const subDagParams = new URLSearchParams({
execution_date: executionDate,
}).toString();
@@ -79,7 +72,6 @@ const Nav = ({ instance, isMapped }) => {
}).toString();
const allInstancesLink = `${taskInstancesUrl}?${listParams.toString()}`;
- const mappedInstancesLink = `${taskInstancesUrl}?${mapParams.toString()}`;
const filterUpstreamLink = appendSearchParams(gridUrlNoRoot, filterParams);
const subDagLink = appendSearchParams(gridUrl.replace(dagId,
`${dagId}.${taskId}`), subDagParams);
@@ -103,9 +95,6 @@ const Nav = ({ instance, isMapped }) => {
<LinkButton href={logLink}>Log</LinkButton>
</>
)}
- {isMapped && (
- <LinkButton href={mappedInstancesLink} title="Show the mapped
instances for this DAG run">Mapped Instances</LinkButton>
- )}
<LinkButton href={allInstancesLink} title="View all instances across
all DAG runs">All Instances</LinkButton>
<LinkButton href={filterUpstreamLink}>Filter Upstream</LinkButton>
</Flex>
diff --git a/airflow/www/static/js/tree/details/content/taskInstance/index.jsx
b/airflow/www/static/js/tree/details/content/taskInstance/index.jsx
index 8457d99ca1..1e787c7aba 100644
--- a/airflow/www/static/js/tree/details/content/taskInstance/index.jsx
+++ b/airflow/www/static/js/tree/details/content/taskInstance/index.jsx
@@ -35,6 +35,7 @@ import TaskNav from './Nav';
import Details from './Details';
import { useTreeData } from '../../../api';
+import MappedInstances from './MappedInstances';
const getTask = ({ taskId, runId, task }) => {
if (task.id === taskId) return task;
@@ -100,6 +101,9 @@ const TaskInstance = ({ taskId, runId }) => {
executionDate={executionDate}
extraLinks={task.extraLinks}
/>
+ {task.isMapped && (
+ <MappedInstances dagId={dagId} runId={runId} taskId={taskId} />
+ )}
</Box>
);
};
diff --git a/airflow/www/static/js/tree/details/index.jsx
b/airflow/www/static/js/tree/details/index.jsx
index 1bdf4e2353..ffe8b0ff74 100644
--- a/airflow/www/static/js/tree/details/index.jsx
+++ b/airflow/www/static/js/tree/details/index.jsx
@@ -30,28 +30,21 @@ import DagRunContent from './content/dagRun';
import DagContent from './content/Dag';
import { useSelection } from '../context/selection';
-const Details = ({ isRefreshOn, onToggleRefresh }) => {
+const Details = () => {
const { selected } = useSelection();
return (
- <Flex borderLeftWidth="1px" flexDirection="column" p={3} flexGrow={1}
maxWidth="600px">
+ <Flex borderLeftWidth="1px" flexDirection="column" pl={3} mr={3}
flexGrow={1} maxWidth="750px">
<Header />
<Divider my={2} />
- <Box minWidth="500px">
- {/* TODO: get full instance data from the API */}
+ <Box minWidth="750px">
{!selected.runId && !selected.taskId && <DagContent />}
{selected.runId && !selected.taskId && (
- <DagRunContent
- runId={selected.runId}
- isRefreshOn={isRefreshOn}
- onToggleRefresh={onToggleRefresh}
- />
+ <DagRunContent runId={selected.runId} />
)}
{selected.taskId && (
<TaskInstanceContent
runId={selected.runId}
taskId={selected.taskId}
- isRefreshOn={isRefreshOn}
- onToggleRefresh={onToggleRefresh}
/>
)}
</Box>
diff --git a/airflow/www/templates/airflow/dag.html
b/airflow/www/templates/airflow/dag.html
index ec4f064987..d77d7f2bc0 100644
--- a/airflow/www/templates/airflow/dag.html
+++ b/airflow/www/templates/airflow/dag.html
@@ -67,6 +67,7 @@
<meta name="task_instances_url" content="{{
url_for('TaskInstanceModelView.list') }}">
<meta name="dag_details_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_dag_endpoint_get_dag_details',
dag_id=dag.dag_id) }}">
<meta name="tasks_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_task_endpoint_get_tasks',
dag_id=dag.dag_id) }}">
+ <meta name="mapped_instances_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_task_instance_endpoint_get_mapped_task_instances',
dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_', task_id='_TASK_ID_') }}">
<!-- End Urls -->
<meta name="is_paused" content="{{ dag_is_paused }}">
<meta name="csrf_token" content="{{ csrf_token() }}">
diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock
index 97cfb667ec..9804dc180b 100644
--- a/airflow/www/yarn.lock
+++ b/airflow/www/yarn.lock
@@ -9838,6 +9838,11 @@ react-style-singleton@^2.1.0:
invariant "^2.2.4"
tslib "^1.0.0"
+react-table@^7.7.0:
+ version "7.7.0"
+ resolved
"https://registry.yarnpkg.com/react-table/-/react-table-7.7.0.tgz#e2ce14d7fe3a559f7444e9ecfe8231ea8373f912";
+ integrity
sha512-jBlj70iBwOTvvImsU9t01LjFjy4sXEtclBovl3mTiqjz23Reu0DKnRza4zlLtOPACx6j2/7MrQIthIK1Wi+LIA==
+
react-tabs@^3.2.2:
version "3.2.2"
resolved
"https://registry.yarnpkg.com/react-tabs/-/react-tabs-3.2.2.tgz#07bdc3cdb17bdffedd02627f32a93cd4b3d6e4d0";
