Github user GJL commented on a diff in the pull request: https://github.com/apache/flink/pull/5310#discussion_r163922121 --- Diff: flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStoreTest.java --- @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.dispatcher; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.concurrent.ManuallyTriggeredScheduledExecutor; +import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; +import org.apache.flink.runtime.messages.webmonitor.JobDetails; +import org.apache.flink.runtime.messages.webmonitor.JobsOverview; +import org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder; +import org.apache.flink.runtime.testingUtils.TestingUtils; +import org.apache.flink.runtime.webmonitor.WebMonitorUtils; +import org.apache.flink.testutils.category.Flip6; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.TestLogger; + +import org.apache.flink.shaded.guava18.com.google.common.cache.LoadingCache; + +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.Matchers; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.Random; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertThat; + +/** + * Tests for the {@link FileArchivedExecutionGraphStore}. + */ +@Category(Flip6.class) +public class FileArchivedExecutionGraphStoreTest extends TestLogger { + + private static final List<JobStatus> GLOBALLY_TERMINAL_JOB_STATUS = new ArrayList<>(3); + + private static final Random RANDOM = new Random(); + + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @BeforeClass + public static void setup() { + GLOBALLY_TERMINAL_JOB_STATUS.add(JobStatus.FINISHED); + GLOBALLY_TERMINAL_JOB_STATUS.add(JobStatus.FAILED); + GLOBALLY_TERMINAL_JOB_STATUS.add(JobStatus.CANCELED); + } + + /** + * Tests that we can put {@link ArchivedExecutionGraph} into the + * {@link FileArchivedExecutionGraphStore} and that the graph is persisted. + */ + @Test + public void testPut() throws IOException { + final ArchivedExecutionGraph dummyExecutionGraph = new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).build(); + final File rootDir = temporaryFolder.newFolder(); + + try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) { + + final File storageDirectory = executionGraphStore.getStorageDir(); + + // check that the storage directory is empty + assertThat(storageDirectory.listFiles().length, Matchers.equalTo(0)); + + executionGraphStore.put(dummyExecutionGraph); + + // check that we have persisted the given execution graph + assertThat(storageDirectory.listFiles().length, Matchers.equalTo(1)); + + assertThat(executionGraphStore.get(dummyExecutionGraph.getJobID()), new PartialArchivedExecutionGraphMatcher(dummyExecutionGraph)); + } + } + + /** + * Tests that null is returned if we request an unknown JobID. + */ + @Test + public void testUnknownGet() throws IOException { + final File rootDir = temporaryFolder.newFolder(); + + try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) { + assertThat(executionGraphStore.get(new JobID()), Matchers.nullValue()); + } + } + + /** + * Tests that we obtain the correct jobs overview. + */ + @Test + public void testStoredJobsOverview() throws IOException { + final int numberExecutionGraphs = 10; + final Collection<ArchivedExecutionGraph> executionGraphs = generateTerminalExecutionGraphs(numberExecutionGraphs); + + final List<JobStatus> jobStatuses = executionGraphs.stream().map(ArchivedExecutionGraph::getState).collect(Collectors.toList()); + + final JobsOverview expectedJobsOverview = JobsOverview.create(jobStatuses); + + final File rootDir = temporaryFolder.newFolder(); + + try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) { + for (ArchivedExecutionGraph executionGraph : executionGraphs) { + executionGraphStore.put(executionGraph); + } + + assertThat(executionGraphStore.getStoredJobsOverview(), Matchers.equalTo(expectedJobsOverview)); + } + } + + /** + * Tests that we obtain the correct collection of available job details. + */ + @Test + public void testAvailableJobDetails() throws IOException { + final int numberExecutionGraphs = 10; + final Collection<ArchivedExecutionGraph> executionGraphs = generateTerminalExecutionGraphs(numberExecutionGraphs); + + final Collection<JobDetails> jobDetails = executionGraphs.stream().map(WebMonitorUtils::createDetailsForJob).collect(Collectors.toList()); + + final File rootDir = temporaryFolder.newFolder(); + + try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) { + for (ArchivedExecutionGraph executionGraph : executionGraphs) { + executionGraphStore.put(executionGraph); + } + + assertThat(executionGraphStore.getAvailableJobDetails(), Matchers.containsInAnyOrder(jobDetails.toArray())); + } + } + + /** + * Tests that an expired execution graph is removed from the execution graph store. + */ + @Test + public void testExecutionGraphExpiration() throws Exception { + final File rootDir = temporaryFolder.newFolder(); + + final Time expirationTime = Time.milliseconds(1L); + + final ManuallyTriggeredScheduledExecutor scheduledExecutor = new ManuallyTriggeredScheduledExecutor(); + + try (final FileArchivedExecutionGraphStore executionGraphStore = new FileArchivedExecutionGraphStore( + rootDir, + expirationTime, + 10000L, + scheduledExecutor)) { + + final ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).build(); + + executionGraphStore.put(executionGraph); + + // there should one execution graph + assertThat(executionGraphStore.size(), Matchers.equalTo(1)); + + Thread.sleep(expirationTime.toMilliseconds()); --- End diff -- Isn't it possible to use a `Ticker` to test the time based eviction? https://github.com/google/guava/wiki/CachesExplained#testing-timed-eviction
---