This is an automated email from the ASF dual-hosted git repository. yiyang0203 pushed a commit to branch HDDS-5713 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit be9cabb5b946d7cd16bcfb77c4d1415d1c867e1c Author: Symious <[email protected]> AuthorDate: Mon Apr 10 15:51:27 2023 +0800 HDDS-8182. Add volume and container choosing policy (#4408) --- .../scm/storage/DiskBalancerConfiguration.java | 23 ++++++++ .../container/common/volume/MutableVolumeSet.java | 11 ++++ .../diskbalancer/DiskBalancerService.java | 24 ++++++++ .../container/diskbalancer/DiskBalancerYaml.java | 4 +- .../policy/ContainerChoosingPolicy.java | 39 +++++++++++++ .../policy/DefaultContainerChoosingPolicy.java | 59 +++++++++++++++++++ .../policy/DefaultVolumeChoosingPolicy.java | 66 ++++++++++++++++++++++ .../diskbalancer/policy/VolumeChoosingPolicy.java | 41 ++++++++++++++ .../diskbalancer/policy/package-info.java | 22 ++++++++ .../diskbalancer/TestDiskBalancerService.java | 20 +++++++ 10 files changed, 307 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java index ed40fa30f6..7c218cb241 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/DiskBalancerConfiguration.java @@ -126,6 +126,29 @@ public final class DiskBalancerConfiguration { this.diskBalancerTimeout = duration.toMillis(); } + @Config(key = "volume.choosing.policy", type = ConfigType.CLASS, + defaultValue = "org.apache.hadoop.ozone.container.diskbalancer.policy" + + ".DefaultVolumeChoosingPolicy", + tags = {ConfigTag.DISKBALANCER}, + description = "The volume choosing policy of the disk balancer service.") + private Class<?> volumeChoosingPolicyClass; + + public Class<?> getVolumeChoosingPolicyClass() { + return volumeChoosingPolicyClass; + } + + @Config(key = "container.choosing.policy", type = ConfigType.CLASS, + defaultValue = "org.apache.hadoop.ozone.container.diskbalancer.policy" + + ".DefaultContainerChoosingPolicy", + tags = {ConfigTag.DISKBALANCER}, + description = "The container choosing policy of the disk balancer " + + "service.") + private Class<?> containerChoosingPolicyClass; + + public Class<?> getContainerChoosingPolicyClass() { + return containerChoosingPolicyClass; + } + public DiskBalancerConfiguration() { } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java index 3c0b6e618e..b103ca7015 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java @@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; +import org.apache.hadoop.util.Preconditions; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -516,4 +517,14 @@ public class MutableVolumeSet implements VolumeSet { this.readUnlock(); } } + + public double getIdealUsage() { + long totalCapacity = 0L, totalUsed = 0L; + for (StorageVolume volume: volumeMap.values()) { + totalCapacity += volume.getCapacity(); + totalUsed += volume.getUsedSpace(); + } + Preconditions.checkArgument(totalCapacity != 0); + return (double) totalUsed / totalCapacity; + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java index 75e1660029..10980a6b1c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java @@ -35,6 +35,8 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; +import org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy; +import org.apache.hadoop.ozone.container.diskbalancer.policy.VolumeChoosingPolicy; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.util.Time; import org.apache.ratis.util.FileUtils; @@ -89,6 +91,8 @@ public class DiskBalancerService extends BackgroundService { private Map<HddsVolume, Long> deltaSizes; private MutableVolumeSet volumeSet; + private VolumeChoosingPolicy volumeChoosingPolicy; + private ContainerChoosingPolicy containerChoosingPolicy; private final File diskBalancerInfoFile; private DiskBalancerServiceMetrics metrics; @@ -110,6 +114,18 @@ public class DiskBalancerService extends BackgroundService { deltaSizes = new ConcurrentHashMap<>(); volumeSet = ozoneContainer.getVolumeSet(); + try { + volumeChoosingPolicy = (VolumeChoosingPolicy) + conf.getObject(DiskBalancerConfiguration.class) + .getVolumeChoosingPolicyClass().newInstance(); + containerChoosingPolicy = (ContainerChoosingPolicy) + conf.getObject(DiskBalancerConfiguration.class) + .getContainerChoosingPolicyClass().newInstance(); + } catch (Exception e) { + LOG.error("Got exception when initializing DiskBalancerService", e); + throw new RuntimeException(e); + } + metrics = DiskBalancerServiceMetrics.create(); loadDiskBalancerInfo(); @@ -387,6 +403,14 @@ public class DiskBalancerService extends BackgroundService { this.balancedBytesInLastWindow.set(bytes); } + public ContainerChoosingPolicy getContainerChoosingPolicy() { + return containerChoosingPolicy; + } + + public VolumeChoosingPolicy getVolumeChoosingPolicy() { + return volumeChoosingPolicy; + } + @Override public void shutdown() { super.shutdown(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java index d16eb65747..d601bc87d8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.container.diskbalancer; +import org.apache.hadoop.hdds.server.YamlUtils; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; @@ -66,11 +67,10 @@ public final class DiskBalancerYaml { DiskBalancerInfo diskBalancerInfo; try (FileInputStream inputFileStream = new FileInputStream(path)) { - Yaml yaml = new Yaml(); DiskBalancerInfoYaml diskBalancerInfoYaml; try { diskBalancerInfoYaml = - yaml.loadAs(inputFileStream, DiskBalancerInfoYaml.class); + YamlUtils.loadAs(inputFileStream, DiskBalancerInfoYaml.class); } catch (Exception e) { throw new IOException("Unable to parse yaml file.", e); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java new file mode 100644 index 0000000000..61b2eeba28 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.diskbalancer.policy; + +import org.apache.hadoop.ozone.container.common.impl.ContainerData; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; + +import java.util.List; +import java.util.Set; + +/** + * This interface specifies the policy for choosing containers to balance. + */ +public interface ContainerChoosingPolicy { + /** + * Choose a container for balancing. + * + * @return a Container + */ + List<ContainerData> chooseContainer(OzoneContainer ozoneContainer, + HddsVolume volume, Set<Long> inProgressContainerIDs, Long targetSize); +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java new file mode 100644 index 0000000000..da1e3610ad --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.diskbalancer.policy; + +import org.apache.hadoop.ozone.container.common.impl.ContainerData; +import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * Choose a container from specified volume, make sure it's not being balancing. + */ +public class DefaultContainerChoosingPolicy implements ContainerChoosingPolicy { + public static final Logger LOG = LoggerFactory.getLogger( + DefaultContainerChoosingPolicy.class); + + @Override + public List<ContainerData> chooseContainer(OzoneContainer ozoneContainer, + HddsVolume hddsVolume, Set<Long> inProgressContainerIDs, + Long targetSize) { + List<ContainerData> results = new ArrayList<>(); + long sizeTotal = 0L; + + Iterator<Container<?>> itr = ozoneContainer.getController() + .getContainers(hddsVolume); + while (itr.hasNext() && sizeTotal < targetSize) { + ContainerData containerData = itr.next().getContainerData(); + if (!inProgressContainerIDs.contains( + containerData.getContainerID()) && containerData.isClosed()) { + results.add(containerData); + sizeTotal += containerData.getBytesUsed(); + } + } + return results; + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java new file mode 100644 index 0000000000..4e2721c811 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.diskbalancer.policy; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Choose a random volume for balancing. + */ +public class DefaultVolumeChoosingPolicy implements VolumeChoosingPolicy { + + public static final Logger LOG = LoggerFactory.getLogger( + DefaultVolumeChoosingPolicy.class); + + @Override + public Pair<HddsVolume, HddsVolume> chooseVolume(MutableVolumeSet volumeSet, + double threshold, Map<HddsVolume, Long> deltaMap) { + double idealUsage = volumeSet.getIdealUsage(); + + List<HddsVolume> volumes = StorageVolumeUtil + .getHddsVolumesList(volumeSet.getVolumesList()) + .stream() + .filter(volume -> Math.abs( + (double) (volume.getUsedSpace() + deltaMap.getOrDefault(volume, 0L)) + / volume.getCapacity() - idealUsage) >= threshold) + .sorted((v1, v2) -> + Double.compare( + (double) (v2.getUsedSpace() + deltaMap.getOrDefault(v2, 0L)) / + v2.getCapacity(), + (double) (v1.getUsedSpace() + deltaMap.getOrDefault(v1, 0L)) / + v1.getCapacity())) + .collect(Collectors.toList()); + + // Can not generate DiskBalancerTask if we have less than 2 results + if (volumes.size() <= 1) { + LOG.debug("Can not find appropriate Source volume and Dest Volume."); + return null; + } + return Pair.of(volumes.get(0), volumes.get(volumes.size() - 1)); + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/VolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/VolumeChoosingPolicy.java new file mode 100644 index 0000000000..187003bf2f --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/VolumeChoosingPolicy.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.diskbalancer.policy; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; + +import java.util.Map; + +/** + * This interface specifies the policy for choosing volumes to balance. + */ +public interface VolumeChoosingPolicy { + /** + * Choose a pair of volumes for balancing. + * + * @param volumeSet - volumes to choose from. + * @param threshold - the threshold to choose source and dest volumes. + * @param deltaSizes - the sizes changes of inProgress balancing jobs. + * @return Source volume and Dest volume. + */ + Pair<HddsVolume, HddsVolume> chooseVolume(MutableVolumeSet volumeSet, + double threshold, Map<HddsVolume, Long> deltaSizes); +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/package-info.java new file mode 100644 index 0000000000..ab9c5f570b --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + This package contains policy classes related to the DiskBalancer service. + */ +package org.apache.hadoop.ozone.container.diskbalancer.policy; diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java index c03f8cab75..83aff1a971 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java @@ -29,6 +29,8 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; +import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy; +import org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultVolumeChoosingPolicy; import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; @@ -37,6 +39,7 @@ import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import java.io.File; @@ -139,6 +142,23 @@ public class TestDiskBalancerService { svc.shutdown(); } + @Test + public void testPolicyClassInitialization() throws IOException { + ContainerSet containerSet = new ContainerSet(1000); + ContainerMetrics metrics = ContainerMetrics.create(conf); + KeyValueHandler keyValueHandler = + new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet, + metrics, c -> { + }); + DiskBalancerServiceTestImpl svc = + getDiskBalancerService(containerSet, conf, keyValueHandler, null, 1); + + assertTrue(svc.getContainerChoosingPolicy() + instanceof DefaultContainerChoosingPolicy); + assertTrue(svc.getVolumeChoosingPolicy() + instanceof DefaultVolumeChoosingPolicy); + } + private String generateVolumeLocation(String base, int volumeCount) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < volumeCount; i++) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
