SLIDER-270: flex down twice not working. This isn't showing up in a (new) mock test
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/28babb19 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/28babb19 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/28babb19 Branch: refs/heads/feature/SLIDER-151_REST_API Commit: 28babb1923bbfd23cc9bc0148d12497b9e848f29 Parents: 0b52d4b Author: Steve Loughran <ste...@apache.org> Authored: Tue Aug 5 16:05:03 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Aug 5 16:05:03 2014 +0100 ---------------------------------------------------------------------- .../server/appmaster/SliderAppMaster.java | 16 +-- .../slider/server/appmaster/state/AppState.java | 43 +++---- .../appstate/TestAppStateDynamicRoles.groovy | 4 +- .../TestAppStateRebuildOnAMRestart.groovy | 4 +- .../model/appstate/TestFlexDynamicRoles.groovy | 2 + .../model/appstate/TestMockFlexing.groovy | 128 +++++++++++++++++++ .../model/appstate/TestMockRMOperations.groovy | 37 ------ .../model/mock/BaseMockAppStateTest.groovy | 57 ++++++++- .../appmaster/model/mock/MockYarnEngine.groovy | 4 +- .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../management/TestAMManagementWebServices.java | 1 + 11 files changed, 219 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 7e0ae5e..1ad7fa7 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -671,12 +671,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //build the instance appState.buildInstance(instanceDefinition, - providerConf, - providerRoles, - fs.getFileSystem(), - historyDir, - liveContainers, - appInformation); + serviceConf, + providerConf, + providerRoles, + fs.getFileSystem(), + historyDir, + liveContainers, + appInformation); // add the AM to the list of nodes in the cluster @@ -1055,8 +1056,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // non complete containers should not be here assert (status.getState() == ContainerState.COMPLETE); - AppState.NodeCompletionResult result = appState.onCompletedNode( - getConfig(), status); + AppState.NodeCompletionResult result = appState.onCompletedNode(status); if (result.containerFailed) { RoleInstance ri = result.roleInstance; log.error("Role instance {} failed ", ri); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index cc238ff..9f8a39d 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -54,7 +54,6 @@ import org.apache.slider.core.exceptions.ErrorStrings; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderInternalStateException; import org.apache.slider.core.exceptions.TriggerClusterTeardownException; -import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.providers.ProviderRole; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -250,6 +249,8 @@ public class AppState { private long startTimeThreshold; private int failureThreshold = 10; + + private String logServerURL = ""; public AppState(AbstractRecordFactory recordFactory) { this.recordFactory = recordFactory; @@ -432,6 +433,7 @@ public class AppState { /** * Build up the application state * @param instanceDefinition definition of the applicatin instance + * @param appmasterConfig * @param publishedProviderConf any configuration info to be published by a provider * @param providerRoles roles offered by a provider * @param fs filesystem @@ -440,12 +442,12 @@ public class AppState { * @param applicationInfo */ public synchronized void buildInstance(AggregateConf instanceDefinition, - Configuration publishedProviderConf, - List<ProviderRole> providerRoles, - FileSystem fs, - Path historyDir, - List<Container> liveContainers, - Map<String, String> applicationInfo) throws + Configuration appmasterConfig, Configuration publishedProviderConf, + List<ProviderRole> providerRoles, + FileSystem fs, + Path historyDir, + List<Container> liveContainers, + Map<String, String> applicationInfo) throws BadClusterStateException, BadConfigException, IOException { @@ -514,6 +516,10 @@ public class AppState { //rebuild any live containers rebuildModelFromRestart(liveContainers); + // any am config options to pick up + + logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, + ""); //mark as live applicationLive = true; } @@ -1161,18 +1167,6 @@ public class AppState { * @return NodeCompletionResult */ public synchronized NodeCompletionResult onCompletedNode(ContainerStatus status) { - return onCompletedNode(null, status); - } - - /** - * handle completed node in the CD -move something from the live - * server list to the completed server list - * @param amConf YarnConfiguration - * @param status the node that has just completed - * @return NodeCompletionResult - */ - public synchronized NodeCompletionResult onCompletedNode(Configuration amConf, - ContainerStatus status) { ContainerId containerId = status.getContainerId(); NodeCompletionResult result = new NodeCompletionResult(); RoleInstance roleInstance; @@ -1219,11 +1213,8 @@ public class AppState { } String completedLogsUrl = null; Container c = roleInstance.container; - String url = null; - if (amConf != null) { - url = amConf.get(YarnConfiguration.YARN_LOG_SERVER_URL); - } - if (user != null && url != null) { + String url = logServerURL; + if (user != null && SliderUtils.isSet(url)) { completedLogsUrl = url + "/" + c.getNodeId() + "/" + roleInstance.getContainerId() + "/ctx/" + user; } @@ -1255,12 +1246,12 @@ public class AppState { completionOfUnknownContainerEvent.incrementAndGet(); } } - + if (result.surplusNode) { //a surplus node return result; } - + //record the complete node's details; this pulls it from the livenode set //remove the node ContainerId id = status.getContainerId(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy index 6e387d8..6d70885 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy @@ -72,11 +72,13 @@ class TestAppStateDynamicRoles extends BaseMockAppStateTest appState.buildInstance( instance, + new Configuration(), new Configuration(false), factory.ROLES, fs, historyPath, - null, null) + null, + null) } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy index 190e927..8d7ff4f 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy @@ -71,11 +71,13 @@ class TestAppStateRebuildOnAMRestart extends BaseMockAppStateTest //and rebuild appState.buildInstance( factory.newInstanceDefinition(r0, r1, r2), + new Configuration(), new Configuration(false), factory.ROLES, fs, historyPath, - containers, null) + containers, + null) assert appState.getStartedCountainerCount() == clusterSize http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy index 1693365..911880a 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy @@ -73,6 +73,7 @@ class TestFlexDynamicRoles extends BaseMockAppStateTest appState.buildInstance(instance, + new Configuration(), new Configuration(false), factory.ROLES, fs, @@ -176,6 +177,7 @@ class TestFlexDynamicRoles extends BaseMockAppStateTest appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) appState.buildInstance( factory.newInstanceDefinition(0, 0, 0), + new Configuration(), new Configuration(false), factory.ROLES, fs, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy new file mode 100644 index 0000000..d2dc028 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate + +import groovy.util.logging.Slf4j +import org.apache.hadoop.yarn.api.records.Container +import org.apache.hadoop.yarn.api.records.ContainerId +import org.apache.hadoop.yarn.client.api.AMRMClient +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockFactory +import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler +import org.apache.slider.server.appmaster.model.mock.MockRoles +import org.apache.slider.server.appmaster.state.AbstractRMOperation +import org.apache.slider.server.appmaster.state.AppState +import org.apache.slider.server.appmaster.state.ContainerAssignment +import org.apache.slider.server.appmaster.state.ContainerReleaseOperation +import org.apache.slider.server.appmaster.state.ContainerRequestOperation +import org.apache.slider.server.appmaster.state.RMOperationHandler +import org.apache.slider.server.appmaster.state.RoleInstance +import org.junit.Test + +import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority +import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole + +@Slf4j +class TestMockFlexing extends BaseMockAppStateTest implements MockRoles { + + @Override + String getTestName() { + return "TestMockFlexing" + } + + + + @Test + public void testFlexDuringLaunchPhase() throws Throwable { + role0Status.desired = 1 + + List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() + List<Container> allocations = engine.execute(ops) + List<ContainerAssignment> assignments = []; + List<AbstractRMOperation> releases = [] + appState.onContainersAllocated(allocations, assignments, releases) + assert assignments.size() == 1 + ContainerAssignment assigned = assignments[0] + Container target = assigned.container + RoleInstance ri = roleInstance(assigned) + + ops = appState.reviewRequestAndReleaseNodes() + assert ops.empty + + //now this is the start point. + appState.containerStartSubmitted(target, ri); + + ops = appState.reviewRequestAndReleaseNodes() + assert ops.empty + + RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id) + } + + @Test + public void testFlexBeforeAllocationPhase() throws Throwable { + role0Status.desired = 1 + + List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() + assert !ops.empty + List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes() + assert ops2.empty + } + + + @Test + public void testFlexDownTwice() throws Throwable { + int r0 = 6 + int r1 = 0 + int r2 = 0 + role0Status.desired = r0 + role1Status.desired = r1 + role2Status.desired = r2 + List<RoleInstance> instances = createAndStartNodes() + + int clusterSize = r0 + r1 + r2 + assert instances.size() == clusterSize + log.info("shrinking cluster") + r0 = 4 + role0Status.desired = r0 + List<AppState.NodeCompletionResult> completionResults = [] + instances = createStartAndStopNodes(completionResults) + assert instances.size() == 0 + // assert two nodes were released + assert completionResults.size() == 2 + + // no-op review + completionResults = [] + instances = createStartAndStopNodes(completionResults) + assert instances.size() == 0 + // assert two nodes were released + assert completionResults.size() == 0 + + + // now shrink again + role0Status.desired = r0 = 1 + completionResults = [] + instances = createStartAndStopNodes(completionResults) + assert instances.size() == 0 + // assert two nodes were released + assert completionResults.size() == 3 + + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy index 168ac9f..e27827b 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy @@ -173,41 +173,4 @@ class TestMockRMOperations extends BaseMockAppStateTest implements MockRoles { assert ri3 == null } - @Test - public void testFlexDuringLaunchPhase() throws Throwable { - role0Status.desired = 1 - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - List<Container> allocations = engine.execute( - ops) - List<ContainerAssignment> assignments = []; - List<AbstractRMOperation> releases = [] - appState.onContainersAllocated(allocations, assignments, releases) - assert assignments.size() == 1 - ContainerAssignment assigned = assignments[0] - Container target = assigned.container - RoleInstance ri = roleInstance(assigned) - - ops = appState.reviewRequestAndReleaseNodes() - assert ops.empty - - //now this is the start point. - appState.containerStartSubmitted(target, ri); - - ops = appState.reviewRequestAndReleaseNodes() - assert ops.empty - - RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id) - } - - @Test - public void testFlexBeforeAllocationPhase() throws Throwable { - role0Status.desired = 1 - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - assert !ops.empty - List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes() - assert ops2.empty - } - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy index 628c729..4a34e7f 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy @@ -82,6 +82,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) appState.buildInstance( factory.newInstanceDefinition(0, 0, 0), + new Configuration(), new Configuration(false), factory.ROLES, fs, @@ -166,20 +167,72 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles * @return a list of roles */ protected List<RoleInstance> createAndStartNodes() { - List<RoleInstance> instances = createAndSubmitNodes() + return createStartAndStopNodes([]) + } + + /** + * Create, Start and stop nodes + * @param completionResults List filled in with the status on all completed nodes + * @return the nodes + */ + public List<RoleInstance> createStartAndStopNodes( + List<AppState.NodeCompletionResult> completionResults) { + List<ContainerId> released = [] + List<RoleInstance> instances = createAndSubmitNodes(released) for (RoleInstance instance : instances) { assert appState.onNodeManagerContainerStarted(instance.containerId) } + releaseContainers(completionResults, + released, + ContainerState.COMPLETE, + "released", + 0 + ) return instances } /** + * Release a list of containers, updating the completion results + * @param completionResults + * @param containerIds + * @param containerState + * @param exitText + * @param containerExitCode + * @return + */ + public def releaseContainers( + List<AppState.NodeCompletionResult> completionResults, + List<ContainerId> containerIds, + ContainerState containerState, + String exitText, + int containerExitCode) { + containerIds.each { ContainerId id -> + ContainerStatus status = ContainerStatus.newInstance(id, + containerState, + exitText, + containerExitCode) + completionResults << appState.onCompletedNode(status) + + } + } + + /** * Create nodes and submit them * @return a list of roles */ public List<RoleInstance> createAndSubmitNodes() { + return createAndSubmitNodes([]) + } + + /** + * Create nodes and submit them + * @param released a list that is built up of all released nodes + * @return a list of roles allocated + */ + public List<RoleInstance> createAndSubmitNodes( + List<ContainerId> released) { List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - List<Container> allocatedContainers = engine.execute(ops) + List<Container> allocatedContainers = engine.execute(ops, released) List<ContainerAssignment> assignments = []; List<AbstractRMOperation> operations = [] appState.onContainersAllocated(allocatedContainers, assignments, operations) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy index 7ebdf52..763f653 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy @@ -86,9 +86,7 @@ class MockYarnEngine { * @param ops * @return */ - List<Container> execute( - List<AbstractRMOperation> ops - ) { + List<Container> execute(List<AbstractRMOperation> ops) { return execute(ops, []) } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java index 17fbe2b..96a4510 100644 --- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java +++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java @@ -24,7 +24,6 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.test.framework.JerseyTest; import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -43,7 +42,6 @@ import org.apache.slider.server.appmaster.web.WebAppApiImpl; import org.apache.slider.server.appmaster.web.rest.RestPaths; import org.apache.slider.server.services.security.CertificateManager; import org.apache.slider.server.services.security.SecurityUtils; -import org.apache.slider.test.SliderTestBase; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -125,6 +123,7 @@ public class TestAMAgentWebServices { appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES); appState.buildInstance( factory.newInstanceDefinition(0, 0, 0), + new Configuration(), new Configuration(false), factory.ROLES, fs, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java index 134f0bc..96e196f 100644 --- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java +++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java @@ -170,6 +170,7 @@ public class TestAMManagementWebServices extends JerseyTest { appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES); appState.buildInstance( factory.newInstanceDefinition(0, 0, 0), + new Configuration(), new Configuration(false), factory.ROLES, fs,
