mukund-thakur commented on code in PR #6789: URL: https://github.com/apache/hadoop/pull/6789#discussion_r1672860768
########## hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfigurationHelper.java: ########## @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; + +import static java.util.EnumSet.noneOf; +import static org.apache.hadoop.util.Preconditions.checkArgument; +import static org.apache.hadoop.util.StringUtils.getTrimmedStringCollection; + +/** + * Configuration Helper class to provide advanced configuration parsing. + * Private; external code MUST use {@link Configuration} instead + */ +@InterfaceAudience.Private +public final class ConfigurationHelper { + + /** + * Error string if there are multiple enum elements which only differ + * by case: {@value}. + */ + @VisibleForTesting + static final String ERROR_MULTIPLE_ELEMENTS_MATCHING_TO_LOWER_CASE_VALUE = + "has multiple elements matching to lower case value"; + + private ConfigurationHelper() { + } + + /** + * Given a comma separated list of enum values, + * trim the list, map to enum values in the message (case insensitive) + * and return the set. + * Special handling of "*" meaning: all values. + * @param key key for error messages. Review Comment: don't see much use of key param. ########## hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfigurationHelper.java: ########## @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.util.Set; + +import org.assertj.core.api.Assertions; +import org.assertj.core.api.IterableAssert; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.apache.hadoop.util.ConfigurationHelper.ERROR_MULTIPLE_ELEMENTS_MATCHING_TO_LOWER_CASE_VALUE; +import static org.apache.hadoop.util.ConfigurationHelper.mapEnumNamesToValues; +import static org.apache.hadoop.util.ConfigurationHelper.parseEnumSet; + +/** + * Test for {@link ConfigurationHelper}. + */ +public class TestConfigurationHelper extends AbstractHadoopTestBase { + + /** + * Simple Enums. + * "i" is included for case tests, as it is special in turkey. + */ + private enum SimpleEnum { a, b, c, i } + + + /** + * Special case: an enum with no values. + */ + private enum EmptyEnum { } + + /** + * Create assertion about the outcome of + * {@link ConfigurationHelper#parseEnumSet(String, String, Class, boolean)}. + * @param valueString value from Configuration + * @param enumClass class of enum + * @param ignoreUnknown should unknown values be ignored? + * @param <E> enum type + * @return an assertion on the outcome. + * @throws IllegalArgumentException if one of the entries was unknown and ignoreUnknown is false, + * or there are two entries in the enum which differ only by case. + */ + private static <E extends Enum<E>> IterableAssert<E> assertEnumParse( + final String valueString, + final Class<E> enumClass, + final boolean ignoreUnknown) { + final Set<E> enumSet = parseEnumSet("key", valueString, enumClass, ignoreUnknown); + final IterableAssert<E> assertion = Assertions.assertThat(enumSet); + return assertion.describedAs("parsed enum set '%s'", valueString); + } + + + /** + * Create a configuration with the key {@code key} set to a {@code value}. + * @param value value for the key + * @return a configuration with only key set. + */ + private Configuration confWithKey(String value) { + final Configuration conf = new Configuration(false); + conf.set("key", value); + return conf; + } + + @Test + public void testEnumParseAll() throws Throwable { + assertEnumParse("*", SimpleEnum.class, false) + .containsExactly(SimpleEnum.a, SimpleEnum.b, SimpleEnum.c, SimpleEnum.i); + } + + @Test + public void testEnumParse() throws Throwable { + assertEnumParse("a, b,c", SimpleEnum.class, false) + .containsExactly(SimpleEnum.a, SimpleEnum.b, SimpleEnum.c); + } + + @Test + public void testEnumCaseIndependence() throws Throwable { + assertEnumParse("A, B, C, I", SimpleEnum.class, false) + .containsExactly(SimpleEnum.a, SimpleEnum.b, SimpleEnum.c, SimpleEnum.i); + } + + @Test + public void testEmptyArguments() throws Throwable { + assertEnumParse(" ", SimpleEnum.class, false) + .isEmpty(); + } + + @Test + public void testUnknownEnumNotIgnored() throws Throwable { + intercept(IllegalArgumentException.class, "unrecognized", () -> + parseEnumSet("key", "c, unrecognized", SimpleEnum.class, false)); + } + + @Test + public void testUnknownEnumNotIgnoredThroughConf() throws Throwable { + intercept(IllegalArgumentException.class, "unrecognized", () -> + confWithKey("c, unrecognized") + .getEnumSet("key", SimpleEnum.class, false)); + } + + @Test + public void testUnknownEnumIgnored() throws Throwable { + assertEnumParse("c, d", SimpleEnum.class, true) + .containsExactly(SimpleEnum.c); + } + + @Test + public void testStarEnum() throws Throwable { Review Comment: this test is already present above testEnumParseAll ########## hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfigurationHelper.java: ########## @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.util.Set; + +import org.assertj.core.api.Assertions; +import org.assertj.core.api.IterableAssert; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.apache.hadoop.util.ConfigurationHelper.ERROR_MULTIPLE_ELEMENTS_MATCHING_TO_LOWER_CASE_VALUE; +import static org.apache.hadoop.util.ConfigurationHelper.mapEnumNamesToValues; +import static org.apache.hadoop.util.ConfigurationHelper.parseEnumSet; + +/** + * Test for {@link ConfigurationHelper}. + */ +public class TestConfigurationHelper extends AbstractHadoopTestBase { + + /** + * Simple Enums. + * "i" is included for case tests, as it is special in turkey. + */ + private enum SimpleEnum { a, b, c, i } + + + /** + * Special case: an enum with no values. + */ + private enum EmptyEnum { } + + /** + * Create assertion about the outcome of + * {@link ConfigurationHelper#parseEnumSet(String, String, Class, boolean)}. + * @param valueString value from Configuration + * @param enumClass class of enum + * @param ignoreUnknown should unknown values be ignored? + * @param <E> enum type + * @return an assertion on the outcome. + * @throws IllegalArgumentException if one of the entries was unknown and ignoreUnknown is false, + * or there are two entries in the enum which differ only by case. + */ + private static <E extends Enum<E>> IterableAssert<E> assertEnumParse( + final String valueString, + final Class<E> enumClass, + final boolean ignoreUnknown) { + final Set<E> enumSet = parseEnumSet("key", valueString, enumClass, ignoreUnknown); + final IterableAssert<E> assertion = Assertions.assertThat(enumSet); + return assertion.describedAs("parsed enum set '%s'", valueString); + } + + + /** + * Create a configuration with the key {@code key} set to a {@code value}. + * @param value value for the key + * @return a configuration with only key set. + */ + private Configuration confWithKey(String value) { + final Configuration conf = new Configuration(false); + conf.set("key", value); + return conf; + } + + @Test + public void testEnumParseAll() throws Throwable { + assertEnumParse("*", SimpleEnum.class, false) + .containsExactly(SimpleEnum.a, SimpleEnum.b, SimpleEnum.c, SimpleEnum.i); + } + + @Test + public void testEnumParse() throws Throwable { + assertEnumParse("a, b,c", SimpleEnum.class, false) + .containsExactly(SimpleEnum.a, SimpleEnum.b, SimpleEnum.c); + } + + @Test + public void testEnumCaseIndependence() throws Throwable { + assertEnumParse("A, B, C, I", SimpleEnum.class, false) + .containsExactly(SimpleEnum.a, SimpleEnum.b, SimpleEnum.c, SimpleEnum.i); + } + + @Test + public void testEmptyArguments() throws Throwable { + assertEnumParse(" ", SimpleEnum.class, false) + .isEmpty(); + } + + @Test + public void testUnknownEnumNotIgnored() throws Throwable { + intercept(IllegalArgumentException.class, "unrecognized", () -> + parseEnumSet("key", "c, unrecognized", SimpleEnum.class, false)); + } + + @Test + public void testUnknownEnumNotIgnoredThroughConf() throws Throwable { + intercept(IllegalArgumentException.class, "unrecognized", () -> + confWithKey("c, unrecognized") + .getEnumSet("key", SimpleEnum.class, false)); + } + + @Test + public void testUnknownEnumIgnored() throws Throwable { + assertEnumParse("c, d", SimpleEnum.class, true) + .containsExactly(SimpleEnum.c); + } + + @Test + public void testStarEnum() throws Throwable { + assertEnumParse("*", SimpleEnum.class, false) + .containsExactly(SimpleEnum.a, SimpleEnum.b, SimpleEnum.c, SimpleEnum.i); + } + + @Test + public void testUnknownStarEnum() throws Throwable { Review Comment: add a test with repeated values "a, b, a". should pass ########## hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfigurationHelper.java: ########## @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; + +import static java.util.EnumSet.noneOf; +import static org.apache.hadoop.util.Preconditions.checkArgument; +import static org.apache.hadoop.util.StringUtils.getTrimmedStringCollection; + +/** + * Configuration Helper class to provide advanced configuration parsing. + * Private; external code MUST use {@link Configuration} instead + */ +@InterfaceAudience.Private +public final class ConfigurationHelper { + + /** + * Error string if there are multiple enum elements which only differ + * by case: {@value}. + */ + @VisibleForTesting + static final String ERROR_MULTIPLE_ELEMENTS_MATCHING_TO_LOWER_CASE_VALUE = + "has multiple elements matching to lower case value"; + + private ConfigurationHelper() { + } + + /** + * Given a comma separated list of enum values, + * trim the list, map to enum values in the message (case insensitive) + * and return the set. + * Special handling of "*" meaning: all values. + * @param key key for error messages. Review Comment: okay I get it now. maybe mention - configuration key which was used to configure the flags. I got confused initially because of the UT, ########## hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FlagSet.java: ########## @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.impl; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.util.ConfigurationHelper; +import org.apache.hadoop.util.Preconditions; + +import static org.apache.hadoop.util.ConfigurationHelper.mapEnumNamesToValues; + +/** + * A set of flags, constructed from a configuration option or from a string, + * with the semantics of + * {@link ConfigurationHelper#parseEnumSet(String, String, Class, boolean)} + * and implementing {@link StreamCapabilities}. + * <p> + * Thread safety: there is no synchronization on a mutable {@code FlagSet}. + * Once declared immutable, flags cannot be changed, so they + * becomes implicitly thread-safe. + */ +public final class FlagSet<E extends Enum<E>> implements StreamCapabilities { + + /** + * Set of flags. + */ + private final Set<E> flags; + + /** + * Is the set immutable? + */ + private final AtomicBoolean immutable = new AtomicBoolean(false); Review Comment: Isn't this class supposed to be singleton initialized only once during the file system creation while the configuration is being loaded? And read only flags such that callers can know if a falg is enabled or not? wondering what is the use case of updating the flags? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
