This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12982-0215a8f95562e941f358be6bf92e48196c7089ad in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit f1ab36f0423c8aa17766959dfd6c6b3f5bc1426a Author: Goutam Adwant <[email protected]> AuthorDate: Wed Nov 18 15:43:11 2026 +0000 [Improve][Zeta] Aggregate retained job counts on cluster members (#12982) --- .../engine/server/CoordinatorService.java | 71 +---- .../engine/server/master/JobHistoryService.java | 67 +++-- .../server/master/JobStatusCountAggregator.java | 62 ++++ .../master/JobHistoryServiceJobCountTest.java | 328 +++++++++++++++++++++ .../server/master/JobHistoryServiceTest.java | 25 ++ 5 files changed, 471 insertions(+), 82 deletions(-) diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/CoordinatorService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/CoordinatorService.java index b4919814a2..f36f005cee 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/CoordinatorService.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/CoordinatorService.java @@ -2220,65 +2220,20 @@ public class CoordinatorService { } public JobCounter getJobCountMetrics() { - AtomicLong createdJobCount = new AtomicLong(); - AtomicLong scheduledJobCount = new AtomicLong(); - AtomicLong runningJobCount = new AtomicLong(); - AtomicLong pendingJobCount = new AtomicLong(); - AtomicLong failingJobCount = new AtomicLong(); - AtomicLong failedJobCount = new AtomicLong(); - AtomicLong cancellingJobCount = new AtomicLong(); - AtomicLong canceledJobCount = new AtomicLong(); - AtomicLong finishedJobCount = new AtomicLong(); - - if (jobHistoryService != null) { - jobHistoryService - .getJobStatusData() - .forEach( - jobStatusData -> { - JobStatus jobStatus = jobStatusData.getJobStatus(); - switch (jobStatus) { - case CREATED: - createdJobCount.addAndGet(1); - break; - case PENDING: - pendingJobCount.addAndGet(1); - break; - case SCHEDULED: - scheduledJobCount.addAndGet(1); - break; - case RUNNING: - runningJobCount.addAndGet(1); - break; - case FAILING: - failingJobCount.addAndGet(1); - break; - case FAILED: - failedJobCount.addAndGet(1); - break; - case CANCELING: - cancellingJobCount.addAndGet(1); - break; - case CANCELED: - canceledJobCount.addAndGet(1); - break; - case FINISHED: - finishedJobCount.addAndGet(1); - break; - default: - } - }); - } - + Map<JobStatus, Long> counts = + jobHistoryService == null + ? Collections.emptyMap() + : jobHistoryService.getJobStatusCounts(); return new JobCounter( - createdJobCount.longValue(), - pendingJobCount.longValue(), - scheduledJobCount.longValue(), - runningJobCount.longValue(), - failingJobCount.longValue(), - failedJobCount.longValue(), - cancellingJobCount.longValue(), - canceledJobCount.longValue(), - finishedJobCount.longValue()); + counts.getOrDefault(JobStatus.CREATED, 0L), + counts.getOrDefault(JobStatus.PENDING, 0L), + counts.getOrDefault(JobStatus.SCHEDULED, 0L), + counts.getOrDefault(JobStatus.RUNNING, 0L), + counts.getOrDefault(JobStatus.FAILING, 0L), + counts.getOrDefault(JobStatus.FAILED, 0L), + counts.getOrDefault(JobStatus.CANCELING, 0L), + counts.getOrDefault(JobStatus.CANCELED, 0L), + counts.getOrDefault(JobStatus.FINISHED, 0L)); } public ThreadPoolStatus getThreadPoolStatusMetrics() { diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobHistoryService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobHistoryService.java index 4afeabeb9e..a3c0e119b9 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobHistoryService.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobHistoryService.java @@ -154,33 +154,11 @@ public class JobHistoryService { public List<JobStatusData> getJobStatusData() { List<JobStatusData> status = new ArrayList<>(); - final List<JobState> runningJobStateList = - runningJobMasterMap.values().stream() - .map(master -> toJobStateMapper(master, true)) - .collect(Collectors.toList()); + final List<JobState> runningJobStateList = getRunningJobStates(); Set<Long> runningJonIds = runningJobStateList.stream().map(JobState::getJobId).collect(Collectors.toSet()); - List<JobState> pendingJobStateList = - pendingJobInfoMap.entrySet().stream() - .map( - entry -> { - Long jobId = entry.getKey(); - JobImmutableInformation jobImmutableInformation = - entry.getValue() - .getJobMaster() - .getJobImmutableInformation(); - return new JobState( - jobId, - jobImmutableInformation.getJobName(), - JobStatus.PENDING, - jobImmutableInformation.getCreateTime(), - null, - null, - null, - null); - }) - .collect(Collectors.toList()); + List<JobState> pendingJobStateList = getPendingJobStates(); Set<Long> pendingJobIds = pendingJobStateList.stream().map(JobState::getJobId).collect(Collectors.toSet()); @@ -207,6 +185,47 @@ public class JobHistoryService { return status; } + /** Counts retained states on their owning members without returning full job records. */ + public Map<JobStatus, Long> getJobStatusCounts() { + List<JobState> runningJobStates = getRunningJobStates(); + List<JobState> pendingJobStates = getPendingJobStates(); + Set<Long> activeJobIds = + Stream.concat(runningJobStates.stream(), pendingJobStates.stream()) + .map(JobState::getJobId) + .collect(Collectors.toSet()); + Map<JobStatus, Long> counts = + finishedJobStateImap.aggregate(new JobStatusCountAggregator(activeJobIds)); + Stream.concat(runningJobStates.stream(), pendingJobStates.stream()) + .forEach(state -> counts.merge(state.getJobStatus(), 1L, Long::sum)); + return counts; + } + + private List<JobState> getRunningJobStates() { + return runningJobMasterMap.values().stream() + .map(master -> toJobStateMapper(master, true)) + .collect(Collectors.toList()); + } + + private List<JobState> getPendingJobStates() { + return pendingJobInfoMap.entrySet().stream() + .map( + entry -> { + Long jobId = entry.getKey(); + JobImmutableInformation information = + entry.getValue().getJobMaster().getJobImmutableInformation(); + return new JobState( + jobId, + information.getJobName(), + JobStatus.PENDING, + information.getCreateTime(), + null, + null, + null, + null); + }) + .collect(Collectors.toList()); + } + // Get detailed status of a single job public JobState getJobDetailState(Long jobId) { if (pendingJobInfoMap.containsKey(jobId)) { diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobStatusCountAggregator.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobStatusCountAggregator.java new file mode 100644 index 0000000000..f8c36d8a19 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobStatusCountAggregator.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.master; + +import org.apache.seatunnel.engine.common.job.JobStatus; +import org.apache.seatunnel.engine.server.master.JobHistoryService.JobState; + +import com.hazelcast.aggregation.Aggregator; + +import java.util.EnumMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** Counts retained records on members without transferring their job details. */ +final class JobStatusCountAggregator + implements Aggregator<Map.Entry<Long, JobState>, Map<JobStatus, Long>> { + private static final long serialVersionUID = 1L; + + // Keep filtering in accumulation: a separate predicate can deserialize each record twice. + // Hazelcast also serializes this exclusion set in aggregation responses. + private final Set<Long> activeJobIds; + private final Map<JobStatus, Long> counts = new EnumMap<>(JobStatus.class); + + JobStatusCountAggregator(Set<Long> activeJobIds) { + this.activeJobIds = new HashSet<>(activeJobIds); + } + + @Override + public void accumulate(Map.Entry<Long, JobState> entry) { + JobState state = entry.getValue(); + if (!activeJobIds.contains(state.getJobId())) { + counts.merge(state.getJobStatus(), 1L, Long::sum); + } + } + + @Override + public void combine(Aggregator aggregator) { + ((JobStatusCountAggregator) aggregator) + .counts.forEach((status, count) -> counts.merge(status, count, Long::sum)); + } + + @Override + public Map<JobStatus, Long> aggregate() { + return counts; + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceJobCountTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceJobCountTest.java new file mode 100644 index 0000000000..28665da85f --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceJobCountTest.java @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.master; + +import org.apache.seatunnel.engine.common.job.JobStatus; +import org.apache.seatunnel.engine.common.job.JobStatusData; +import org.apache.seatunnel.engine.core.job.JobImmutableInformation; +import org.apache.seatunnel.engine.core.job.PipelineStatus; +import org.apache.seatunnel.engine.server.CoordinatorService; +import org.apache.seatunnel.engine.server.dag.physical.PipelineLocation; +import org.apache.seatunnel.engine.server.execution.ExecutionState; +import org.apache.seatunnel.engine.server.execution.PendingJobInfo; +import org.apache.seatunnel.engine.server.execution.TaskGroupLocation; +import org.apache.seatunnel.engine.server.master.JobHistoryService.JobState; +import org.apache.seatunnel.engine.server.master.JobHistoryService.PipelineStateData; +import org.apache.seatunnel.engine.server.telemetry.metrics.entity.JobCounter; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.mockito.Mockito; + +import com.hazelcast.config.Config; +import com.hazelcast.core.Hazelcast; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.instance.impl.HazelcastInstanceProxy; +import com.hazelcast.map.IMap; + +import java.lang.reflect.Field; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +/** Exercises count aggregation on remote partitions, independently of the single-server job IT. */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class JobHistoryServiceJobCountTest { + private final List<HazelcastInstance> members = new ArrayList<>(); + private final String clusterName = "job-count-test-" + UUID.randomUUID(); + private final Map<Long, JobMaster> runningMasters = new HashMap<>(); + private final Map<Long, PendingJobInfo> pendingJobs = new HashMap<>(); + private final List<IMap<?, ?>> maps = new ArrayList<>(); + private IMap<Object, Object> runningStates; + private IMap<Long, JobState> finishedStates; + private JobHistoryService history; + private CoordinatorService coordinator; + private int seedPort; + + @BeforeAll + void startMembers() throws Exception { + seedPort = freePort(); + members.add(Hazelcast.newHazelcastInstance(config(seedPort))); + members.add(Hazelcast.newHazelcastInstance(config(freePort()))); + members.add(Hazelcast.newHazelcastInstance(config(freePort()))); + awaitSafeCluster(3); + } + + @AfterAll + void stopMembers() { + members.forEach(member -> member.getLifecycleService().terminate()); + } + + @BeforeEach + void createHistory() throws Exception { + runningMasters.clear(); + pendingJobs.clear(); + runningStates = newMap(); + finishedStates = newMap(); + history = history(finishedStates); + coordinator = Mockito.mock(CoordinatorService.class, Mockito.CALLS_REAL_METHODS); + Field field = CoordinatorService.class.getDeclaredField("jobHistoryService"); + field.setAccessible(true); + field.set(coordinator, history); + } + + @AfterEach + void destroyMaps() { + maps.forEach(IMap::destroy); + maps.clear(); + } + + @Test + void countsAllStatusesAcrossMembersWithoutChangingHistory() { + for (long id = 0; id < JobStatus.values().length * 10; id++) { + finishedStates.put( + id, state(id, JobStatus.values()[(int) (id % JobStatus.values().length)])); + } + assertTrue( + finishedStates.keySet().stream() + .anyMatch( + id -> + !members.get(0) + .getPartitionService() + .getPartition(id) + .getOwner() + .localMember())); + Map<Long, JobState> before = finishedStates.getAll(finishedStates.keySet()); + assertMatchesListing(); + assertEquals(before, finishedStates.getAll(finishedStates.keySet())); + assertEquals(10L, coordinator.getJobCountMetrics().getFinishedJobCount()); + } + + @Test + void activeSnapshotsSuppressRetainedAttemptsAndKeepOverlapBehavior() { + runningMasters.put(1L, master(1L, JobStatus.FAILED)); + runningStates.put(1L, JobStatus.RUNNING); + pendingJobs.put(2L, pending(2L)); + runningMasters.put(3L, master(3L, JobStatus.RUNNING)); + pendingJobs.put(3L, pending(3L)); + runningMasters.put(4L, master(4L, JobStatus.INITIALIZING)); + runningMasters.put(5L, master(5L, JobStatus.FAILING)); + for (long id = 1; id <= 5; id++) { + finishedStates.put(id, state(id, JobStatus.FAILED)); + } + // The existing listing suppresses by the value's job ID, not the IMap key. + finishedStates.put(99L, state(1L, JobStatus.FAILED)); + finishedStates.put(6L, state(6L, JobStatus.FINISHED)); + assertMatchesListing(); + assertEquals(new JobCounter(0, 2, 0, 2, 1, 0, 0, 0, 1), coordinator.getJobCountMetrics()); + + runningMasters.remove(1L); + pendingJobs.remove(2L); + assertMatchesListing(); + assertEquals(3L, coordinator.getJobCountMetrics().getFailedJobCount()); + } + + @Test + void countsEmptyHistoryAndUninitializedCoordinator() throws Exception { + assertMatchesListing(); + assertEquals(new JobCounter(0, 0, 0, 0, 0, 0, 0, 0, 0), coordinator.getJobCountMetrics()); + Field field = CoordinatorService.class.getDeclaredField("jobHistoryService"); + field.setAccessible(true); + field.set(coordinator, null); + assertEquals(new JobCounter(0, 0, 0, 0, 0, 0, 0, 0, 0), coordinator.getJobCountMetrics()); + } + + @Test + void scrapesDoNotExtendRetainedRecordExpiry() { + finishedStates.put(1L, state(1L, JobStatus.FINISHED), 1, TimeUnit.SECONDS); + finishedStates.put(2L, state(2L, JobStatus.CANCELED)); + await().atMost(20, TimeUnit.SECONDS) + .pollInterval(50, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + coordinator.getJobCountMetrics(); + assertFalse(finishedStates.containsKey(1L)); + }); + assertMatchesListing(); + assertEquals(new JobCounter(0, 0, 0, 0, 0, 0, 0, 1, 0), coordinator.getJobCountMetrics()); + } + + @Test + void retainsCountsAfterMemberLossAndPartitionMigration() throws Exception { + for (long id = 0; id < JobStatus.values().length * 10; id++) { + finishedStates.put( + id, state(id, JobStatus.values()[(int) (id % JobStatus.values().length)])); + } + awaitSafeCluster(3); + assertFalse(members.get(2).getMap(finishedStates.getName()).localKeySet().isEmpty()); + JobCounter expected = coordinator.getJobCountMetrics(); + try { + members.get(2).getLifecycleService().terminate(); + awaitSafeCluster(2); + assertEquals(JobStatus.values().length * 10, finishedStates.size()); + assertMatchesListing(); + assertEquals(expected, coordinator.getJobCountMetrics()); + } finally { + members.set(2, Hazelcast.newHazelcastInstance(config(freePort()))); + awaitSafeCluster(3); + } + assertMatchesListing(); + assertEquals(expected, coordinator.getJobCountMetrics()); + } + + @Test + void queryFailurePropagatesInsteadOfReportingZeroCounts() { + IMap<Long, JobState> failedMap = Mockito.spy(finishedStates); + Mockito.doThrow(new IllegalStateException("count query failed")) + .when(failedMap) + .aggregate(any()); + JobHistoryService failedHistory = history(failedMap); + assertThrows(IllegalStateException.class, failedHistory::getJobStatusCounts); + } + + private void assertMatchesListing() { + Map<JobStatus, Long> expected = new EnumMap<>(JobStatus.class); + for (JobStatusData state : history.getJobStatusData()) { + expected.merge(state.getJobStatus(), 1L, Long::sum); + } + assertEquals(expected, history.getJobStatusCounts()); + assertEquals( + new JobCounter( + expected.getOrDefault(JobStatus.CREATED, 0L), + expected.getOrDefault(JobStatus.PENDING, 0L), + expected.getOrDefault(JobStatus.SCHEDULED, 0L), + expected.getOrDefault(JobStatus.RUNNING, 0L), + expected.getOrDefault(JobStatus.FAILING, 0L), + expected.getOrDefault(JobStatus.FAILED, 0L), + expected.getOrDefault(JobStatus.CANCELING, 0L), + expected.getOrDefault(JobStatus.CANCELED, 0L), + expected.getOrDefault(JobStatus.FINISHED, 0L)), + coordinator.getJobCountMetrics()); + } + + private JobMaster master(long id, JobStatus fallback) { + JobMaster master = Mockito.mock(JobMaster.class); + JobImmutableInformation information = Mockito.mock(JobImmutableInformation.class); + when(information.getJobId()).thenReturn(id); + when(information.getJobName()).thenReturn("job-" + id); + when(information.getCreateTime()).thenReturn(1L); + when(master.getJobImmutableInformation()).thenReturn(information); + when(master.getJobStatus()).thenReturn(fallback); + return master; + } + + private PendingJobInfo pending(long id) { + PendingJobInfo pending = Mockito.mock(PendingJobInfo.class); + JobMaster master = master(id, JobStatus.PENDING); + when(pending.getJobMaster()).thenReturn(master); + return pending; + } + + private JobState state(long id, JobStatus status) { + return new JobState( + id, + "job-" + id, + status, + 1L, + 2L, + 3L, + Collections.singletonMap( + new PipelineLocation(id, 1), + new PipelineStateData( + PipelineStatus.FINISHED, + Collections.singletonMap( + new TaskGroupLocation(id, 1, 1), ExecutionState.FINISHED))), + "retained diagnostic"); + } + + private JobHistoryService history(IMap<Long, JobState> finished) { + HazelcastInstanceProxy proxy = (HazelcastInstanceProxy) members.get(0); + return new JobHistoryService( + proxy.getOriginal().node.nodeEngine, + runningStates, + members.get(0).getLoggingService().getLogger(JobHistoryServiceJobCountTest.class), + pendingJobs, + runningMasters, + finished, + newMap(), + newMap(), + 1440); + } + + private <K, V> IMap<K, V> newMap() { + IMap<K, V> map = members.get(0).getMap("job-count-" + UUID.randomUUID()); + maps.add(map); + return map; + } + + private void awaitSafeCluster(int size) { + await().atMost(60, TimeUnit.SECONDS) + .until( + () -> + members.get(0).getCluster().getMembers().size() == size + && members.get(0).getPartitionService().isClusterSafe()); + } + + private Config config(int port) { + Config config = new Config().setClusterName(clusterName); + config.setProperty("hazelcast.logging.type", "none"); + config.setProperty("hazelcast.operation.thread.count", "2"); + config.setProperty("hazelcast.operation.generic.thread.count", "2"); + config.setProperty("hazelcast.io.input.thread.count", "1"); + config.setProperty("hazelcast.io.output.thread.count", "1"); + config.setProperty("hazelcast.query.thread.count", "2"); + config.setProperty("hazelcast.heartbeat.interval.seconds", "1"); + config.setProperty("hazelcast.max.no.heartbeat.seconds", "5"); + config.getJetConfig().setEnabled(false); + config.getNetworkConfig().setPort(port).setPortAutoIncrement(false); + config.getNetworkConfig().getInterfaces().setEnabled(true).addInterface("127.0.0.1"); + config.getNetworkConfig().getJoin().getAutoDetectionConfig().setEnabled(false); + config.getNetworkConfig().getJoin().getMulticastConfig().setEnabled(false); + config.getNetworkConfig() + .getJoin() + .getTcpIpConfig() + .setEnabled(true) + .addMember("127.0.0.1:" + seedPort); + return config; + } + + private int freePort() throws Exception { + try (ServerSocket socket = new ServerSocket(0, 0, InetAddress.getByName("127.0.0.1"))) { + return socket.getLocalPort(); + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceTest.java index a9169b02c9..e3022a2cbf 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/master/JobHistoryServiceTest.java @@ -148,6 +148,31 @@ class JobHistoryServiceTest extends AbstractSeaTunnelServerTest { .contains("FINISHED"))); } + @Test + void testFinishedJobCountsMatchHistory() { + long jobId = System.currentTimeMillis() + 10L; + startJob(jobId, "fake_to_console.conf"); + await().atMost(60, TimeUnit.SECONDS) + .untilAsserted( + () -> { + long listed = + listJob().stream() + .filter(job -> job.getJobStatus() == JobStatus.FINISHED) + .count(); + Assertions.assertEquals( + JobStatus.FINISHED, + server.getCoordinatorService() + .getJobHistoryService() + .getJobDetailState(jobId) + .getJobStatus()); + Assertions.assertEquals( + listed, + server.getCoordinatorService() + .getJobCountMetrics() + .getFinishedJobCount()); + }); + } + private void startJob(Long jobid, String path) { LogicalDag testLogicalDag = TestUtils.createTestLogicalPlan(path, jobid.toString(), jobid);
