[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466158726
##
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatistics.java
##
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics;
+
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * IO Statistics.
+ *
+ * These are low-cost per-instance statistics provided by any Hadoop
+ * I/O class instance.
+ *
+ * Consult the filesystem specification document for the requirements
+ * of an implementation of this interface.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public interface IOStatistics {
+
+ /**
+ * Map of counters.
+ * @return the current map of counters.
+ */
+ Map counters();
+
+ /**
+ * Map of gauges.
+ * @return the current map of gauges.
+ */
+ Map gauges();
+
+ /**
+ * Map of minumums.
+ * @return the current map of minumums.
Review comment:
typos: minumums -> minimums
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466157590
##
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
##
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics.impl;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.MeanStatistic;
+
+/**
+ * Support for implementing IOStatistics interfaces.
+ */
+public final class IOStatisticsBinding {
+
+ /** Pattern used for each entry. */
+ public static final String ENTRY_PATTERN = "(%s=%s)";
+
+ /** String to return when a source is null. */
+ @VisibleForTesting
+ public static final String NULL_SOURCE = "()";
+
+ private IOStatisticsBinding() {
+ }
+
+ /**
+ * Create IOStatistics from a storage statistics instance.
+ * This will be updated as the storage statistics change.
+ * @param storageStatistics source data.
+ * @return an IO statistics source.
+ */
+ public static IOStatistics fromStorageStatistics(
+ StorageStatistics storageStatistics) {
+DynamicIOStatisticsBuilder builder = dynamicIOStatistics();
+Iterator it = storageStatistics
+.getLongStatistics();
+while (it.hasNext()) {
+ StorageStatistics.LongStatistic next = it.next();
+ builder.withLongFunctionCounter(next.getName(),
+ k -> storageStatistics.getLong(k));
+}
+return builder.build();
+ }
+
+ /**
+ * Create a builder for dynamic IO Statistics.
+ * @return a builder to be completed.
+ */
+ public static DynamicIOStatisticsBuilder dynamicIOStatistics() {
+return new DynamicIOStatisticsBuilder();
+ }
+
+ /**
+ * Get the shared instance of the immutable empty statistics
+ * object.
+ * @return an empty statistics object.
+ */
+ public static IOStatistics emptyStatistics() {
+return EmptyIOStatistics.getInstance();
+ }
+
+ /**
+ * Take an IOStatistics instance and wrap it in a source.
+ * @param statistics statistics.
+ * @return a source which will return the values
+ */
+ public static IOStatisticsSource wrap(IOStatistics statistics) {
+return new SourceWrappedStatistics(statistics);
+ }
+
+ /**
+ * Create an a builder for an {@link IOStatisticsStore}.
+ *
+ * @return a builder instance.
+ */
+ public static IOStatisticsStoreBuilder iostatisticsStore() {
+return new IOStatisticsStoreBuilderImpl();
+ }
+
+ /**
+ * Convert an entry to the string format used in logging.
+ *
+ * @param entry entry to evaluate
+ * @param entry type
+ * @return formatted string
+ */
+ public static String entryToString(
+ final Map.Entry entry) {
+return entryToString(entry.getKey(), entry.getValue());
+ }
+
+ /**
+ * Convert entry values to the string format used in logging.
+ *
+ * @param name statistic name
+ * @param value stat value
+ * @return formatted string
+ */
+ public static String entryToString(
+ final String name, final E value) {
+return String.format(
+ENTRY_PATTERN,
+name,
+value);
+ }
+
+ /**
+ * Copy into the dest map all the source entries.
+ * The destination is cleared first.
+ * @param entry type
+ * @param dest destination of the copy
+ * @param source source
+ * @param copyFn function to copy entries
+ * @return the destination.
+ */
+ public static Map copyMap(
+ Map dest,
+ Map source,
+ Function copyFn) {
+// we have to clone the values so that they aren't
+// bound to the original values
+dest.clear();
+source.entrySet()
+.forEach(entry ->
+dest.put(entry.getKey(), copyFn.apply(entry.getValue(;
+
[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466156747
##
File path:
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDynamicIOStatistics.java
##
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.statistics.impl.SourceWrappedStatistics;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static
org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertCounterStatisticIsTracked;
+import static
org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertCounterStatisticIsUntracked;
+import static
org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyCounterStatisticValue;
+import static
org.apache.hadoop.fs.statistics.IOStatisticsLogging.demandStringifyIOStatistics;
+import static
org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToString;
+import static
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.NULL_SOURCE;
+import static
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.dynamicIOStatistics;
+import static
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.emptyStatistics;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * verify dynamic statistics are dynamic, except when you iterate through
+ * them, along with other tests of the class's behavior.
+ */
+public class TestDynamicIOStatistics extends AbstractHadoopTestBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestDynamicIOStatistics.class);
+
+ private static final String ALONG = "along";
+
+ private static final String AINT = "aint";
+
+ private static final String COUNT = "count";
+
+ private static final String EVAL = "eval";
+
+ /**
+ * The statistics.
+ */
+ private IOStatistics statistics = emptyStatistics();
+
+ /**
+ * A source of these statistics.
+ */
+ private IOStatisticsSource statsSource;
+
+ private final AtomicLong aLong = new AtomicLong();
+
+ private final AtomicInteger aInt = new AtomicInteger();
+
+ private final MutableCounterLong counter = new MutableCounterLong(
+ new Info("counter"), 0);
+
+ private long evalLong;
+
+ private static final String[] KEYS = new String[]{ALONG, AINT, COUNT, EVAL};
+
+ @Before
+ public void setUp() throws Exception {
+statistics = dynamicIOStatistics()
+.withAtomicLongCounter(ALONG, aLong)
+.withAtomicIntegerCounter(AINT, aInt)
+.withMutableCounter(COUNT, counter)
+.withLongFunctionCounter(EVAL, x -> evalLong)
+.build();
+statsSource = new SourceWrappedStatistics(statistics);
+ }
+
+ /**
+ * The eval operation is foundational.
+ */
+ @Test
+ public void testEval() throws Throwable {
+verifyCounterStatisticValue(statistics, EVAL, 0);
+evalLong = 10;
+verifyCounterStatisticValue(statistics, EVAL, 10);
+ }
+
+ /**
+ * Atomic Long statistic.
+ */
+ @Test
+ public void testAlong() throws Throwable {
+verifyCounterStatisticValue(statistics, ALONG, 0);
+aLong.addAndGet(1);
+verifyCounterStatisticValue(statistics, ALONG, 1);
+ }
+
+ /**
+ * Atomic Int statistic.
+ */
+ @Test
+ public void testAint() throws Throwable {
+verifyCounterStatisticValue(statistics, AINT, 0);
+aInt.addAndGet(1);
+verifyCounterStatisticValue(statistics, AINT, 1);
+ }
+
+ /**
+ * Metrics2 counter.
+ */
+ @Test
+ public void testCounter() throws Throwable {
+verifyCounterStatisticValue(statistics, COUNT, 0);
+counter.incr();
+verifyCounterStatisticValue(statistics, COUNT, 1);
+ }
+
+ /**
+ * keys() returns all the
[jira] [Created] (HADOOP-17188) Support for AWS STSAssumeRoleWithWebIdentitySessionCredentialsProvider based credential provider to support use of IRSA on deployments on AWS EKS Cluster
Arun Ravi M V created HADOOP-17188: -- Summary: Support for AWS STSAssumeRoleWithWebIdentitySessionCredentialsProvider based credential provider to support use of IRSA on deployments on AWS EKS Cluster Key: HADOOP-17188 URL: https://issues.apache.org/jira/browse/HADOOP-17188 Project: Hadoop Common Issue Type: Improvement Components: fs/s3 Reporter: Arun Ravi M V The latest version of AWS SDK has support to use IRSA for providing credentials to Kubernetes pods which can potentially replace the use of Kube2IAM. For our Apache Spark on Kubernetes use cases, this feature will be useful. The current Hadoop AWS component does support adding custom credential provider but I think if we could add STSAssumeRoleWithWebIdentitySessionCredentialsProvider support to (using roleArn, role session name, web Identity Token File) to the hadoop-aws library, it will be useful for the community as such who use AWS EKS. [https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/STSAssumeRoleWithWebIdentitySessionCredentialsProvider.html] [https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/STSAssumeRoleWithWebIdentitySessionCredentialsProvider.Builder.html ] [https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/] -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466154206
##
File path:
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDynamicIOStatistics.java
##
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.statistics.impl.SourceWrappedStatistics;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static
org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertCounterStatisticIsTracked;
+import static
org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertCounterStatisticIsUntracked;
+import static
org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyCounterStatisticValue;
+import static
org.apache.hadoop.fs.statistics.IOStatisticsLogging.demandStringifyIOStatistics;
+import static
org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToString;
+import static
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.NULL_SOURCE;
+import static
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.dynamicIOStatistics;
+import static
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.emptyStatistics;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * verify dynamic statistics are dynamic, except when you iterate through
+ * them, along with other tests of the class's behavior.
+ */
+public class TestDynamicIOStatistics extends AbstractHadoopTestBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestDynamicIOStatistics.class);
+
+ private static final String ALONG = "along";
+
+ private static final String AINT = "aint";
+
+ private static final String COUNT = "count";
+
+ private static final String EVAL = "eval";
+
+ /**
+ * The statistics.
+ */
+ private IOStatistics statistics = emptyStatistics();
+
+ /**
+ * A source of these statistics.
+ */
+ private IOStatisticsSource statsSource;
+
+ private final AtomicLong aLong = new AtomicLong();
+
+ private final AtomicInteger aInt = new AtomicInteger();
+
+ private final MutableCounterLong counter = new MutableCounterLong(
+ new Info("counter"), 0);
+
+ private long evalLong;
+
+ private static final String[] KEYS = new String[]{ALONG, AINT, COUNT, EVAL};
+
+ @Before
+ public void setUp() throws Exception {
+statistics = dynamicIOStatistics()
+.withAtomicLongCounter(ALONG, aLong)
+.withAtomicIntegerCounter(AINT, aInt)
+.withMutableCounter(COUNT, counter)
+.withLongFunctionCounter(EVAL, x -> evalLong)
+.build();
+statsSource = new SourceWrappedStatistics(statistics);
+ }
+
+ /**
+ * The eval operation is foundational.
+ */
+ @Test
+ public void testEval() throws Throwable {
+verifyCounterStatisticValue(statistics, EVAL, 0);
+evalLong = 10;
+verifyCounterStatisticValue(statistics, EVAL, 10);
+ }
+
+ /**
+ * Atomic Long statistic.
+ */
+ @Test
+ public void testAlong() throws Throwable {
+verifyCounterStatisticValue(statistics, ALONG, 0);
+aLong.addAndGet(1);
+verifyCounterStatisticValue(statistics, ALONG, 1);
+ }
+
+ /**
+ * Atomic Int statistic.
+ */
+ @Test
+ public void testAint() throws Throwable {
+verifyCounterStatisticValue(statistics, AINT, 0);
+aInt.addAndGet(1);
+verifyCounterStatisticValue(statistics, AINT, 1);
+ }
+
+ /**
+ * Metrics2 counter.
+ */
+ @Test
+ public void testCounter() throws Throwable {
+verifyCounterStatisticValue(statistics, COUNT, 0);
+counter.incr();
+verifyCounterStatisticValue(statistics, COUNT, 1);
+ }
+
+ /**
+ * keys() returns all the
[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466153355
##
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSupport.java
##
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Support for working with IOStatistics.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class IOStatisticsSupport {
+
+ private IOStatisticsSupport() {
+ }
+
+ /**
+ * Take a snapshot of the current statistics state.
+ *
+ * This is not an atomic option.
+ *
+ * The instance can be serialized, and its
+ * {@code toString()} method lists all the values.
+ * @param statistics statistics
+ * @return a snapshot of the current values.
+ */
+ public static IOStatisticsSnapshot
+ snapshotIOStatistics(IOStatistics statistics) {
+
+return new IOStatisticsSnapshot(statistics);
+ }
+
+
+ /**
+ * Create a snapsnot statistics instance ready to aggregate data.
Review comment:
typo: snapsnot -> snapshot
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466152928
##
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
##
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics.impl;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.MeanStatistic;
+
+/**
+ * Support for implementing IOStatistics interfaces.
+ */
+public final class IOStatisticsBinding {
+
+ /** Pattern used for each entry. */
+ public static final String ENTRY_PATTERN = "(%s=%s)";
+
+ /** String to return when a source is null. */
+ @VisibleForTesting
+ public static final String NULL_SOURCE = "()";
+
+ private IOStatisticsBinding() {
+ }
+
+ /**
+ * Create IOStatistics from a storage statistics instance.
+ * This will be updated as the storage statistics change.
+ * @param storageStatistics source data.
+ * @return an IO statistics source.
+ */
+ public static IOStatistics fromStorageStatistics(
+ StorageStatistics storageStatistics) {
+DynamicIOStatisticsBuilder builder = dynamicIOStatistics();
+Iterator it = storageStatistics
+.getLongStatistics();
+while (it.hasNext()) {
+ StorageStatistics.LongStatistic next = it.next();
+ builder.withLongFunctionCounter(next.getName(),
+ k -> storageStatistics.getLong(k));
+}
+return builder.build();
+ }
+
+ /**
+ * Create a builder for dynamic IO Statistics.
+ * @return a builder to be completed.
+ */
+ public static DynamicIOStatisticsBuilder dynamicIOStatistics() {
+return new DynamicIOStatisticsBuilder();
+ }
+
+ /**
+ * Get the shared instance of the immutable empty statistics
+ * object.
+ * @return an empty statistics object.
+ */
+ public static IOStatistics emptyStatistics() {
+return EmptyIOStatistics.getInstance();
+ }
+
+ /**
+ * Take an IOStatistics instance and wrap it in a source.
+ * @param statistics statistics.
+ * @return a source which will return the values
+ */
+ public static IOStatisticsSource wrap(IOStatistics statistics) {
+return new SourceWrappedStatistics(statistics);
+ }
+
+ /**
+ * Create an a builder for an {@link IOStatisticsStore}.
+ *
+ * @return a builder instance.
+ */
+ public static IOStatisticsStoreBuilder iostatisticsStore() {
+return new IOStatisticsStoreBuilderImpl();
+ }
+
+ /**
+ * Convert an entry to the string format used in logging.
+ *
+ * @param entry entry to evaluate
+ * @param entry type
+ * @return formatted string
+ */
+ public static String entryToString(
+ final Map.Entry entry) {
+return entryToString(entry.getKey(), entry.getValue());
+ }
+
+ /**
+ * Convert entry values to the string format used in logging.
+ *
+ * @param name statistic name
+ * @param value stat value
+ * @return formatted string
+ */
+ public static String entryToString(
+ final String name, final E value) {
+return String.format(
+ENTRY_PATTERN,
+name,
+value);
+ }
+
+ /**
+ * Copy into the dest map all the source entries.
+ * The destination is cleared first.
+ * @param entry type
+ * @param dest destination of the copy
+ * @param source source
+ * @param copyFn function to copy entries
+ * @return the destination.
+ */
+ public static Map copyMap(
+ Map dest,
+ Map source,
+ Function copyFn) {
+// we have to clone the values so that they aren't
+// bound to the original values
+dest.clear();
+source.entrySet()
+.forEach(entry ->
+dest.put(entry.getKey(), copyFn.apply(entry.getValue(;
+
[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466152709
##
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
##
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics.impl;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.MeanStatistic;
+
+/**
+ * Support for implementing IOStatistics interfaces.
+ */
+public final class IOStatisticsBinding {
+
+ /** Pattern used for each entry. */
+ public static final String ENTRY_PATTERN = "(%s=%s)";
+
+ /** String to return when a source is null. */
+ @VisibleForTesting
+ public static final String NULL_SOURCE = "()";
+
+ private IOStatisticsBinding() {
+ }
+
+ /**
+ * Create IOStatistics from a storage statistics instance.
+ * This will be updated as the storage statistics change.
+ * @param storageStatistics source data.
+ * @return an IO statistics source.
+ */
+ public static IOStatistics fromStorageStatistics(
+ StorageStatistics storageStatistics) {
+DynamicIOStatisticsBuilder builder = dynamicIOStatistics();
+Iterator it = storageStatistics
+.getLongStatistics();
+while (it.hasNext()) {
+ StorageStatistics.LongStatistic next = it.next();
+ builder.withLongFunctionCounter(next.getName(),
+ k -> storageStatistics.getLong(k));
+}
+return builder.build();
+ }
+
+ /**
+ * Create a builder for dynamic IO Statistics.
+ * @return a builder to be completed.
+ */
+ public static DynamicIOStatisticsBuilder dynamicIOStatistics() {
+return new DynamicIOStatisticsBuilder();
+ }
+
+ /**
+ * Get the shared instance of the immutable empty statistics
+ * object.
+ * @return an empty statistics object.
+ */
+ public static IOStatistics emptyStatistics() {
+return EmptyIOStatistics.getInstance();
+ }
+
+ /**
+ * Take an IOStatistics instance and wrap it in a source.
+ * @param statistics statistics.
+ * @return a source which will return the values
+ */
+ public static IOStatisticsSource wrap(IOStatistics statistics) {
+return new SourceWrappedStatistics(statistics);
+ }
+
+ /**
+ * Create an a builder for an {@link IOStatisticsStore}.
+ *
+ * @return a builder instance.
+ */
+ public static IOStatisticsStoreBuilder iostatisticsStore() {
+return new IOStatisticsStoreBuilderImpl();
+ }
+
+ /**
+ * Convert an entry to the string format used in logging.
+ *
+ * @param entry entry to evaluate
+ * @param entry type
+ * @return formatted string
+ */
+ public static String entryToString(
+ final Map.Entry entry) {
+return entryToString(entry.getKey(), entry.getValue());
+ }
+
+ /**
+ * Convert entry values to the string format used in logging.
+ *
+ * @param name statistic name
+ * @param value stat value
+ * @return formatted string
+ */
+ public static String entryToString(
+ final String name, final E value) {
+return String.format(
+ENTRY_PATTERN,
+name,
+value);
+ }
+
+ /**
+ * Copy into the dest map all the source entries.
+ * The destination is cleared first.
+ * @param entry type
+ * @param dest destination of the copy
+ * @param source source
+ * @param copyFn function to copy entries
+ * @return the destination.
+ */
+ public static Map copyMap(
Review comment:
can make this method private.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about
[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466152447
##
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
##
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics.impl;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.MeanStatistic;
+
+/**
+ * Support for implementing IOStatistics interfaces.
+ */
+public final class IOStatisticsBinding {
+
+ /** Pattern used for each entry. */
+ public static final String ENTRY_PATTERN = "(%s=%s)";
+
+ /** String to return when a source is null. */
+ @VisibleForTesting
+ public static final String NULL_SOURCE = "()";
+
+ private IOStatisticsBinding() {
+ }
+
+ /**
+ * Create IOStatistics from a storage statistics instance.
+ * This will be updated as the storage statistics change.
+ * @param storageStatistics source data.
+ * @return an IO statistics source.
+ */
+ public static IOStatistics fromStorageStatistics(
+ StorageStatistics storageStatistics) {
+DynamicIOStatisticsBuilder builder = dynamicIOStatistics();
+Iterator it = storageStatistics
+.getLongStatistics();
+while (it.hasNext()) {
+ StorageStatistics.LongStatistic next = it.next();
+ builder.withLongFunctionCounter(next.getName(),
+ k -> storageStatistics.getLong(k));
+}
+return builder.build();
+ }
+
+ /**
+ * Create a builder for dynamic IO Statistics.
+ * @return a builder to be completed.
+ */
+ public static DynamicIOStatisticsBuilder dynamicIOStatistics() {
+return new DynamicIOStatisticsBuilder();
+ }
+
+ /**
+ * Get the shared instance of the immutable empty statistics
+ * object.
+ * @return an empty statistics object.
+ */
+ public static IOStatistics emptyStatistics() {
+return EmptyIOStatistics.getInstance();
+ }
+
+ /**
+ * Take an IOStatistics instance and wrap it in a source.
+ * @param statistics statistics.
+ * @return a source which will return the values
+ */
+ public static IOStatisticsSource wrap(IOStatistics statistics) {
+return new SourceWrappedStatistics(statistics);
+ }
+
+ /**
+ * Create an a builder for an {@link IOStatisticsStore}.
Review comment:
nit: "Create a builder"
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] bshashikant commented on a change in pull request #2176: HDFS-15492. Make trash root inside each snapshottable directory
bshashikant commented on a change in pull request #2176: URL: https://github.com/apache/hadoop/pull/2176#discussion_r466152617 ## File path: hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java ## @@ -244,6 +244,10 @@ "dfs.namenode.snapshot.capture.openfiles"; boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = false; + String DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED = + "dfs.namenode.snapshot.trashroot.enabled"; + boolean DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT = false; + String DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS = Review comment: If we hide it, i would prefer to not add it in hdfs-default.xml as well. We can make this an internal config which is required only if u want to restrict renames out of snapshottable root which can be turned on only if a requirement like this arises in certain use cases. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466151923
##
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
##
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics.impl;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.MeanStatistic;
+
+/**
+ * Support for implementing IOStatistics interfaces.
+ */
+public final class IOStatisticsBinding {
+
+ /** Pattern used for each entry. */
+ public static final String ENTRY_PATTERN = "(%s=%s)";
Review comment:
can make this private
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] mehakmeet commented on a change in pull request #2069: HADOOP-16830. IOStatistics API.
mehakmeet commented on a change in pull request #2069:
URL: https://github.com/apache/hadoop/pull/2069#discussion_r466152125
##
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
##
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics.impl;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.MeanStatistic;
+
+/**
+ * Support for implementing IOStatistics interfaces.
+ */
+public final class IOStatisticsBinding {
+
+ /** Pattern used for each entry. */
+ public static final String ENTRY_PATTERN = "(%s=%s)";
+
+ /** String to return when a source is null. */
+ @VisibleForTesting
+ public static final String NULL_SOURCE = "()";
+
+ private IOStatisticsBinding() {
Review comment:
Can't find any usage for this constructor.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Commented] (HADOOP-17145) Unauthenticated users are not authorized to access this page message is misleading in HttpServer2.java
[
https://issues.apache.org/jira/browse/HADOOP-17145?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17171889#comment-17171889
]
Hadoop QA commented on HADOOP-17145:
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 9m
46s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} dupname {color} | {color:green} 0m
0s{color} | {color:green} No case conflicting files found. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 2 new or modified test
files. {color} |
|| || || || {color:brown} trunk Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 24m
37s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 24m
34s{color} | {color:green} trunk passed with JDK
Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 20m
39s{color} | {color:green} trunk passed with JDK Private
Build-1.8.0_252-8u252-b09-1~18.04-b09 {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
52s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 1m
34s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} shadedclient {color} | {color:green}
19m 36s{color} | {color:green} branch has no errors when building and testing
our client artifacts. {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
40s{color} | {color:green} trunk passed with JDK
Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
45s{color} | {color:green} trunk passed with JDK Private
Build-1.8.0_252-8u252-b09-1~18.04-b09 {color} |
| {color:blue}0{color} | {color:blue} spotbugs {color} | {color:blue} 2m
32s{color} | {color:blue} Used deprecated FindBugs config; considering
switching to SpotBugs. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 2m
29s{color} | {color:green} trunk passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 0m
59s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 23m
23s{color} | {color:green} the patch passed with JDK
Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 23m
23s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 20m
36s{color} | {color:green} the patch passed with JDK Private
Build-1.8.0_252-8u252-b09-1~18.04-b09 {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 20m
36s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 55s{color} | {color:orange} hadoop-common-project/hadoop-common: The patch
generated 1 new + 84 unchanged - 1 fixed = 85 total (was 85) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 1m
35s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:green}+1{color} | {color:green} shadedclient {color} | {color:green}
17m 31s{color} | {color:green} patch has no errors when building and testing
our client artifacts. {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
36s{color} | {color:green} the patch passed with JDK
Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
41s{color} | {color:green} the patch passed with JDK Private
Build-1.8.0_252-8u252-b09-1~18.04-b09 {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 2m
42s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:red}-1{color} | {color:red} unit {color} | {color:red} 11m 5s{color}
| {color:red} hadoop-common in the patch passed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
56s{color} |
[jira] [Commented] (HADOOP-14040) Use shaded aws-sdk uber-JAR 1.11.86
[ https://issues.apache.org/jira/browse/HADOOP-14040?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17171886#comment-17171886 ] Yongjun Zhang commented on HADOOP-14040: Many thanks [~liuml07]! > Use shaded aws-sdk uber-JAR 1.11.86 > --- > > Key: HADOOP-14040 > URL: https://issues.apache.org/jira/browse/HADOOP-14040 > Project: Hadoop Common > Issue Type: Sub-task > Components: build, fs/s3 >Affects Versions: 2.9.0 >Reporter: Steve Loughran >Assignee: Steve Loughran >Priority: Major > Fix For: 2.9.0, 3.0.0-alpha4 > > Attachments: HADOOP-14040-HADOOP-13345.001.patch, > HADOOP-14040-branch-2-001.patch, HADOOP-14040-branch-2.002.patch, > HADOOP-14040.001.patch > > > AWS SDK now has a (v. large) uberjar shading all dependencies > This ensures that AWS dependency changes (e.g json) don't cause problems > downstream in things like HBase, so enabling backporting if desired. > This will let us addess the org.json don't be evil problem: this SDK version > doesn't have those files. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17145) Unauthenticated users are not authorized to access this page message is misleading in HttpServer2.java
[ https://issues.apache.org/jira/browse/HADOOP-17145?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Andras Bokor updated HADOOP-17145: -- Attachment: HADOOP-17145.004.patch > Unauthenticated users are not authorized to access this page message is > misleading in HttpServer2.java > -- > > Key: HADOOP-17145 > URL: https://issues.apache.org/jira/browse/HADOOP-17145 > Project: Hadoop Common > Issue Type: Bug >Reporter: Andras Bokor >Assignee: Andras Bokor >Priority: Major > Attachments: HADOOP-17145.001.patch, HADOOP-17145.002.patch, > HADOOP-17145.003.patch, HADOOP-17145.004.patch > > > Recently one of the users were misled by the message "Unauthenticated users > are not authorized to access this page" when the user was not an admin user. > At that point the user is authenticated but has no admin access, so it's > actually not an authentication issue but an authorization issue. > Also, 401 as error code would be better. > Something like "User is unauthorized to access the page" would help to users > to find out what is the problem during access an http endpoint. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] aajisaka merged pull request #2183: MAPREDUCE-7288. Fix TestLongLong#testRightShift
aajisaka merged pull request #2183: URL: https://github.com/apache/hadoop/pull/2183 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] JohnZZGithub commented on pull request #2185: HADOOP-15891. provide Regex Based Mount Point In Inode Tree
JohnZZGithub commented on pull request #2185: URL: https://github.com/apache/hadoop/pull/2185#issuecomment-669606794 Here're the UT failed. I didn't see them related. But I could double-check. Thanks [ERROR] Failures: [ERROR] TestBPOfferService.testMissBlocksWhenReregister:350 [ERROR] TestDataNodeErasureCodingMetrics.testFullBlock:97->doTest:205 Wrongly computed block reconstruction work [ERROR] TestNameNodeRetryCacheMetrics.testRetryCacheMetrics:95->checkMetrics:103 CacheHit expected:<2> but was:<0> [ERROR] Errors: [ERROR] TestHDFSContractMultipartUploader>AbstractContractMultipartUploaderTest.testConcurrentUploads:815 ? IllegalArgument [ERROR] TestBlockTokenWithDFSStriped.testRead:92->TestBlockTokenWithDFS.doTestRead:508->isBlockTokenExpired:139->TestBlockTokenWithDFS.isBlockTokenExpired:633 ? NullPointer [ERROR] TestBootstrapStandby.testSuccessfulBaseCase:130->restartNameNodesFromIndex:342 ? Bind [ERROR] TestExternalStoragePolicySatisfier.testChooseInSameDatanodeWithONESSDShouldNotChooseIfNoSpace:1064 ? Timeout [INFO] [ERROR] Tests run: 6500, Failures: 3, Errors: 4, Skipped: 23 [INFO] [ERROR] There are test failures. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #2149: HADOOP-13230. S3A to optionally retain directory markers
hadoop-yetus commented on pull request #2149: URL: https://github.com/apache/hadoop/pull/2149#issuecomment-669490233 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | +0 :ok: | reexec | 0m 31s | Docker mode activated. | ||| _ Prechecks _ | | +1 :green_heart: | dupname | 0m 1s | No case conflicting files found. | | +0 :ok: | markdownlint | 0m 1s | markdownlint was not available. | | +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. | | +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 23 new or modified test files. | ||| _ trunk Compile Tests _ | | +0 :ok: | mvndep | 3m 19s | Maven dependency ordering for branch | | +1 :green_heart: | mvninstall | 25m 53s | trunk passed | | +1 :green_heart: | compile | 19m 24s | trunk passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | compile | 16m 45s | trunk passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | +1 :green_heart: | checkstyle | 2m 39s | trunk passed | | +1 :green_heart: | mvnsite | 2m 22s | trunk passed | | +1 :green_heart: | shadedclient | 19m 42s | branch has no errors when building and testing our client artifacts. | | +1 :green_heart: | javadoc | 1m 14s | trunk passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | javadoc | 2m 17s | trunk passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | +0 :ok: | spotbugs | 1m 15s | Used deprecated FindBugs config; considering switching to SpotBugs. | | +1 :green_heart: | findbugs | 3m 26s | trunk passed | | -0 :warning: | patch | 1m 35s | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. | ||| _ Patch Compile Tests _ | | +0 :ok: | mvndep | 0m 25s | Maven dependency ordering for patch | | -1 :x: | mvninstall | 0m 27s | hadoop-aws in the patch failed. | | -1 :x: | compile | 17m 42s | root in the patch failed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1. | | -1 :x: | javac | 17m 42s | root in the patch failed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1. | | -1 :x: | compile | 15m 47s | root in the patch failed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09. | | -1 :x: | javac | 15m 47s | root in the patch failed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09. | | -0 :warning: | checkstyle | 2m 42s | root: The patch generated 11 new + 67 unchanged - 2 fixed = 78 total (was 69) | | -1 :x: | mvnsite | 0m 49s | hadoop-aws in the patch failed. | | -1 :x: | whitespace | 0m 0s | The patch has 8 line(s) that end in whitespace. Use git apply --whitespace=fix <>. Refer https://git-scm.com/docs/git-apply | | -1 :x: | whitespace | 0m 0s | The patch 3 line(s) with tabs. | | +1 :green_heart: | xml | 0m 1s | The patch has no ill-formed XML file. | | +1 :green_heart: | shadedclient | 13m 59s | patch has no errors when building and testing our client artifacts. | | +1 :green_heart: | javadoc | 1m 12s | the patch passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | javadoc | 2m 17s | the patch passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | -1 :x: | findbugs | 0m 48s | hadoop-aws in the patch failed. | ||| _ Other Tests _ | | +1 :green_heart: | unit | 9m 16s | hadoop-common in the patch passed. | | -1 :x: | unit | 0m 48s | hadoop-aws in the patch failed. | | +1 :green_heart: | asflicense | 0m 53s | The patch does not generate ASF License warnings. | | | | 169m 49s | | | Subsystem | Report/Notes | |--:|:-| | Docker | ClientAPI=1.40 ServerAPI=1.40 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2149/9/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hadoop/pull/2149 | | Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient findbugs checkstyle xml markdownlint | | uname | Linux 51ef73c05764 4.15.0-58-generic #64-Ubuntu SMP Tue Aug 6 11:12:41 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux | | Build tool | maven | | Personality | personality/hadoop.sh | | git revision | trunk / 3f73facd7b3 | | Default Java | Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | mvninstall |
[GitHub] [hadoop] smengcl commented on a change in pull request #2176: HDFS-15492. Make trash root inside each snapshottable directory
smengcl commented on a change in pull request #2176: URL: https://github.com/apache/hadoop/pull/2176#discussion_r465975831 ## File path: hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java ## @@ -244,6 +244,10 @@ "dfs.namenode.snapshot.capture.openfiles"; boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = false; + String DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED = + "dfs.namenode.snapshot.trashroot.enabled"; + boolean DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT = false; + String DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS = Review comment: makes sense. will hide it from the client. but we do want to put this in hdfs-default.xml so people can learn to enable this on NN right? This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] smengcl commented on a change in pull request #2176: HDFS-15492. Make trash root inside each snapshottable directory
smengcl commented on a change in pull request #2176: URL: https://github.com/apache/hadoop/pull/2176#discussion_r465975831 ## File path: hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java ## @@ -244,6 +244,10 @@ "dfs.namenode.snapshot.capture.openfiles"; boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = false; + String DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED = + "dfs.namenode.snapshot.trashroot.enabled"; + boolean DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED_DEFAULT = false; + String DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS = Review comment: makes sense. will hide it from the client This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17168) Remove non-inclusive terminology from Hadoop Common
[ https://issues.apache.org/jira/browse/HADOOP-17168?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Eric Badger updated HADOOP-17168: - Summary: Remove non-inclusive terminology from Hadoop Common (was: Remove non-inclusive terminology from Hadoop) > Remove non-inclusive terminology from Hadoop Common > --- > > Key: HADOOP-17168 > URL: https://issues.apache.org/jira/browse/HADOOP-17168 > Project: Hadoop Common > Issue Type: Improvement >Reporter: Eric Badger >Priority: Major > > http://mail-archives.apache.org/mod_mbox/hadoop-common-dev/202007.mbox/%3CCAAaVJWVXhsv4tn1KOQkKYTaQ441Yb8y7s%2BR_GnESwduB1iFxOA%40mail.gmail.com%3E > This JIRA is to remove offensive and non-inclusive terminology from Hadoop. > The simple ones are whitelist/blacklist and master/slave. However this JIRA > can also serve as a place to fix other non-inclusive terminology (e.g., > binary gendered > examples, "Alice" doing the wrong security thing systematically). > As [~curino] posted in his email, the IETF has created a draft for proposed > alternatives > https://tools.ietf.org/id/draft-knodel-terminology-00.html#rfc.section.1.1.1 -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17170) Remove master/slave terminology from Hadoop Common
[ https://issues.apache.org/jira/browse/HADOOP-17170?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Eric Badger updated HADOOP-17170: - Summary: Remove master/slave terminology from Hadoop Common (was: Remove master/slave terminology from Hadoop) > Remove master/slave terminology from Hadoop Common > -- > > Key: HADOOP-17170 > URL: https://issues.apache.org/jira/browse/HADOOP-17170 > Project: Hadoop Common > Issue Type: Sub-task >Reporter: Eric Badger >Priority: Major > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17169) Remove whitelist/blacklist terminology from Hadoop
[ https://issues.apache.org/jira/browse/HADOOP-17169?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Eric Badger updated HADOOP-17169: - Summary: Remove whitelist/blacklist terminology from Hadoop (was: Remove whitelist/blacklist terminology from Hadoop) > Remove whitelist/blacklist terminology from Hadoop > --- > > Key: HADOOP-17169 > URL: https://issues.apache.org/jira/browse/HADOOP-17169 > Project: Hadoop Common > Issue Type: Sub-task >Reporter: Eric Badger >Assignee: Eric Badger >Priority: Major > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17169) Remove whitelist/blacklist terminology from Hadoop Common
[ https://issues.apache.org/jira/browse/HADOOP-17169?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Eric Badger updated HADOOP-17169: - Summary: Remove whitelist/blacklist terminology from Hadoop Common (was: Remove whitelist/blacklist terminology from Hadoop ) > Remove whitelist/blacklist terminology from Hadoop Common > - > > Key: HADOOP-17169 > URL: https://issues.apache.org/jira/browse/HADOOP-17169 > Project: Hadoop Common > Issue Type: Sub-task >Reporter: Eric Badger >Assignee: Eric Badger >Priority: Major > -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] steveloughran commented on pull request #955: HDFS-14478: Add libhdfs APIs for openFile
steveloughran commented on pull request #955: URL: https://github.com/apache/hadoop/pull/955#issuecomment-669355258 I should warn that in #2046 I'm adding opt/must to take a long as well; we just missed that out in the past. Needed so that applications like yours can pass in the known length of the file (split limit is fine) and we can skip the HEAD from that alone. hive needs that This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Created] (HADOOP-17187) Report non-inclusive language as part of code contribution pre-commit check
Eric Badger created HADOOP-17187: Summary: Report non-inclusive language as part of code contribution pre-commit check Key: HADOOP-17187 URL: https://issues.apache.org/jira/browse/HADOOP-17187 Project: Hadoop Common Issue Type: Sub-task Reporter: Eric Badger -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Commented] (HADOOP-17180) S3Guard: Include 500 DynamoDB system errors in exponential backoff retries
[
https://issues.apache.org/jira/browse/HADOOP-17180?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17171659#comment-17171659
]
Steve Loughran commented on HADOOP-17180:
-
DDB's error reporting is pretty painful. I don't see any support for it in
S3aUtils.isThrottleException in trunk, so upgrading isn't going to make your
life better
.InMemoryFileIndex is something I've heard bad reviews for in terms of S3 load
(https://stackoverflow.com/questions/60590925/spark-making-expensive-s3-api-calls
), probably the trigger of this.
Anyway, if treating 500 as a throttle event from DDB is needed, patch against
trunk welcome, we can go back to 3.3.x as well. We've moved a long way from the
3.1 line though.
Looking at the changelog, you need to be on 3.2.x to get the current DDB
throttle logic in HADOOP-15426
Can you try that and see if it helps?
> S3Guard: Include 500 DynamoDB system errors in exponential backoff retries
> --
>
> Key: HADOOP-17180
> URL: https://issues.apache.org/jira/browse/HADOOP-17180
> Project: Hadoop Common
> Issue Type: Bug
>Affects Versions: 3.1.3
>Reporter: David Kats
>Priority: Major
> Attachments: image-2020-08-03-09-58-54-102.png
>
>
> We get fatal failures from S3guard (that in turn fail our spark jobs) because
> of the inernal DynamoDB system errors.
> {color:#00}com.amazonaws.services.dynamodbv2.model.InternalServerErrorException:
> Internal server error (Service: AmazonDynamoDBv2; Status Code: 500; Error
> Code: InternalServerError; Request ID:
> 00EBRE6J6V8UGD7040C9DUP2MNVV4KQNSO5AEMVJF66Q9ASUAAJG): Internal server error
> (Service: AmazonDynamoDBv2; Status Code: 500; Error Code:
> InternalServerError; Request ID:
> 00EBRE6J6V8UGD7040C9DUP2MNVV4KQNSO5AEMVJF66Q9ASUAAJG){color}
> {color:#00}The DynamoDB has separate statistic for system errors:{color}
> {color:#00}!image-2020-08-03-09-58-54-102.png!{color}
> {color:#00}I contacted the AWS Support and got an explanation that those
> 500 errors are returned to the client once DynamoDB gets overwhelmed with
> client requests.{color}
> {color:#00}So essentially the traffic should had been throttled but it
> didn't and got 500 system errors.{color}
> {color:#00}My point is that the client should handle those errors just
> like throttling exceptions - {color}
> {color:#00}with exponential backoff retries.{color}
>
> {color:#00}Here is more complete exception stack trace:{color}
>
> *{color:#00}org.apache.hadoop.fs.s3a.AWSServiceIOException: get on
> s3a://rem-spark/persisted_step_data/15/0afb1ccb73854f1fa55517a77ec7cc5e__b67e2221-f0e3-4c89-90ab-f49618ea4557__SDTopology/parquet.all_ranges/topo_id=321:
> com.amazonaws.services.dynamodbv2.model.InternalServerErrorException:
> Internal server error (Service: AmazonDynamoDBv2; Status Code: 500; Error
> Code: InternalServerError; Request ID:
> 00EBRE6J6V8UGD7040C9DUP2MNVV4KQNSO5AEMVJF66Q9ASUAAJG): Internal server error
> (Service: AmazonDynamoDBv2; Status Code: 500; Error Code:
> InternalServerError; Request ID:
> 00EBRE6J6V8UGD7040C9DUP2MNVV4KQNSO5AEMVJF66Q9ASUAAJG)
> at{color}*{color:#00}
> org.apache.hadoop.fs.s3a.S3AUtils.translateDynamoDBException(S3AUtils.java:389)
> at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:181)
> at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:111) at
> org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.get(DynamoDBMetadataStore.java:438)
> at
> org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:2110)
> at
> org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:2088)
> at
> org.apache.hadoop.fs.s3a.S3AFileSystem.innerListStatus(S3AFileSystem.java:1889)
> at
> org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$listStatus$9(S3AFileSystem.java:1868)
> at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109) at
> org.apache.hadoop.fs.s3a.S3AFileSystem.listStatus(S3AFileSystem.java:1868) at
> org.apache.spark.sql.execution.datasources.InMemoryFileIndex$.org$apache$spark$sql$execution$datasources$InMemoryFileIndex$$listLeafFiles(InMemoryFileIndex.scala:277)
> at
> org.apache.spark.sql.execution.datasources.InMemoryFileIndex$$anonfun$3$$anonfun$apply$2.apply(InMemoryFileIndex.scala:207)
> at
> org.apache.spark.sql.execution.datasources.InMemoryFileIndex$$anonfun$3$$anonfun$apply$2.apply(InMemoryFileIndex.scala:206)
> at scala.collection.immutable.Stream.map(Stream.scala:418) at
> org.apache.spark.sql.execution.datasources.InMemoryFileIndex$$anonfun$3.apply(InMemoryFileIndex.scala:206)
> at
> org.apache.spark.sql.execution.datasources.InMemoryFileIndex$$anonfun$3.apply(InMemoryFileIndex.scala:204)
> at
>
[jira] [Commented] (HADOOP-17163) ABFS: Add debug log for rename failures
[ https://issues.apache.org/jira/browse/HADOOP-17163?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17171651#comment-17171651 ] Steve Loughran commented on HADOOP-17163: - see also HADOOP-16721 -not done anything there. Tempting > ABFS: Add debug log for rename failures > --- > > Key: HADOOP-17163 > URL: https://issues.apache.org/jira/browse/HADOOP-17163 > Project: Hadoop Common > Issue Type: Sub-task > Components: fs/azure >Affects Versions: 3.3.0 >Reporter: Bilahari T H >Assignee: Bilahari T H >Priority: Major > Fix For: 3.4.0 > > > The JIRA [HADOOP-16281|https://issues.apache.org/jira/browse/HADOOP-16281] > has not yet been concluded. Untill then the logline could help debugging. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17149) ABFS: Test failure: testFailedRequestWhenCredentialsNotCorrect fails when run with SharedKey
[ https://issues.apache.org/jira/browse/HADOOP-17149?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Bilahari T H updated HADOOP-17149: -- Resolution: Fixed Status: Resolved (was: Patch Available) > ABFS: Test failure: testFailedRequestWhenCredentialsNotCorrect fails when run > with SharedKey > > > Key: HADOOP-17149 > URL: https://issues.apache.org/jira/browse/HADOOP-17149 > Project: Hadoop Common > Issue Type: Sub-task > Components: fs/azure >Affects Versions: 3.3.1 >Reporter: Sneha Vijayarajan >Assignee: Bilahari T H >Priority: Minor > Labels: abfsactive > Fix For: 3.4.0 > > > When authentication is set to SharedKey, below test fails. > > [ERROR] > ITestGetNameSpaceEnabled.testFailedRequestWhenCredentialsNotCorrect:161 > Expecting > org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException > with text "Server failed to authenticate the request. Make sure the value of > Authorization header is formed correctly including the signature.", 403 but > got : "void" > > This test fails when the newly introduced config > "fs.azure.account.hns.enabled" is set. This config will avoid network call to > check if namespace is enabled, whereas the test expects thsi call to be made. > > The assert in test to 403 needs check too. Should ideally be 401. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17163) ABFS: Add debug log for rename failures
[ https://issues.apache.org/jira/browse/HADOOP-17163?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Bilahari T H updated HADOOP-17163: -- Resolution: Fixed Status: Resolved (was: Patch Available) > ABFS: Add debug log for rename failures > --- > > Key: HADOOP-17163 > URL: https://issues.apache.org/jira/browse/HADOOP-17163 > Project: Hadoop Common > Issue Type: Sub-task > Components: fs/azure >Affects Versions: 3.3.0 >Reporter: Bilahari T H >Assignee: Bilahari T H >Priority: Major > Fix For: 3.4.0 > > > The JIRA [HADOOP-16281|https://issues.apache.org/jira/browse/HADOOP-16281] > has not yet been concluded. Untill then the logline could help debugging. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17137) ABFS: Tests ITestAbfsNetworkStatistics need to be config setting agnostic
[
https://issues.apache.org/jira/browse/HADOOP-17137?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Bilahari T H updated HADOOP-17137:
--
Resolution: Fixed
Status: Resolved (was: Patch Available)
> ABFS: Tests ITestAbfsNetworkStatistics need to be config setting agnostic
> -
>
> Key: HADOOP-17137
> URL: https://issues.apache.org/jira/browse/HADOOP-17137
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure, test
>Affects Versions: 3.3.0
>Reporter: Sneha Vijayarajan
>Assignee: Bilahari T H
>Priority: Minor
> Labels: abfsactive
> Fix For: 3.4.0
>
>
> Tess in ITestAbfsNetworkStatistics have asserts to a static number of
> network calls made from the start of fileystem instance creation. But this
> number of calls are dependent on the certain configs settings which allow
> creation of container or account is HNS enabled to avoid GetAcl call.
>
> The tests need to be modified to ensure that count asserts are made for the
> requests made by the tests alone.
>
> {code:java}
> [INFO] Running org.apache.hadoop.fs.azurebfs.ITestAbfsNetworkStatistics[INFO]
> Running org.apache.hadoop.fs.azurebfs.ITestAbfsNetworkStatistics[ERROR] Tests
> run: 2, Failures: 2, Errors: 0, Skipped: 0, Time elapsed: 4.148 s <<<
> FAILURE! - in org.apache.hadoop.fs.azurebfs.ITestAbfsNetworkStatistics[ERROR]
> testAbfsHttpResponseStatistics(org.apache.hadoop.fs.azurebfs.ITestAbfsNetworkStatistics)
> Time elapsed: 4.148 s <<< FAILURE!java.lang.AssertionError: Mismatch in
> get_responses expected:<8> but was:<7> at
> org.junit.Assert.fail(Assert.java:88) at
> org.junit.Assert.failNotEquals(Assert.java:834) at
> org.junit.Assert.assertEquals(Assert.java:645) at
> org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest.assertAbfsStatistics(AbstractAbfsIntegrationTest.java:445)
> at
> org.apache.hadoop.fs.azurebfs.ITestAbfsNetworkStatistics.testAbfsHttpResponseStatistics(ITestAbfsNetworkStatistics.java:207)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) at
> org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
> at
> org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266) at
> java.lang.Thread.run(Thread.java:748)
> [ERROR]
> testAbfsHttpSendStatistics(org.apache.hadoop.fs.azurebfs.ITestAbfsNetworkStatistics)
> Time elapsed: 2.987 s <<< FAILURE!java.lang.AssertionError: Mismatch in
> connections_made expected:<6> but was:<5> at
> org.junit.Assert.fail(Assert.java:88) at
> org.junit.Assert.failNotEquals(Assert.java:834) at
> org.junit.Assert.assertEquals(Assert.java:645) at
> org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest.assertAbfsStatistics(AbstractAbfsIntegrationTest.java:445)
> at
> org.apache.hadoop.fs.azurebfs.ITestAbfsNetworkStatistics.testAbfsHttpSendStatistics(ITestAbfsNetworkStatistics.java:91)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) at
>
[GitHub] [hadoop] DadanielZ merged pull request #2174: HADOOP-17149. ABFS: Fixing the testcase ITestGetNameSpaceEnabled#testFailedRequestWhenCre…
DadanielZ merged pull request #2174: URL: https://github.com/apache/hadoop/pull/2174 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17164) UGI loginUserFromKeytab doesn't set the last login time
[ https://issues.apache.org/jira/browse/HADOOP-17164?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Mingliang Liu updated HADOOP-17164: --- Fix Version/s: 2.10.1 Resolution: Fixed Status: Resolved (was: Patch Available) Merged to all fixed versions. Thanks [~sandeep.guggilam] for reporting and filing a patch. Thanks [~ste...@apache.org] for helpful review. > UGI loginUserFromKeytab doesn't set the last login time > --- > > Key: HADOOP-17164 > URL: https://issues.apache.org/jira/browse/HADOOP-17164 > Project: Hadoop Common > Issue Type: Bug > Components: security >Affects Versions: 2.10.0, 3.3.0, 3.2.1, 3.1.3 >Reporter: Sandeep Guggilam >Assignee: Sandeep Guggilam >Priority: Major > Fix For: 3.1.4, 3.2.2, 2.10.1, 3.3.1, 3.4.0 > > Attachments: HADOOP-17164-branch-2.10.001.patch, > HADOOP-17164-branch-2.10.002.patch, HADOOP-17164.001.patch > > > UGI initial login from keytab doesn't set the last login time as a result of > which the relogin can happen even before the configured minimum seconds to > wait before relogin -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] liuml07 merged pull request #2194: HADOOP-17164. UGI loginUserFromKeytab doesn't set the last login time
liuml07 merged pull request #2194: URL: https://github.com/apache/hadoop/pull/2194 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] bilaharith commented on pull request #2192: HADOOP-17183. ABFS: Enabling checkaccess on ABFS
bilaharith commented on pull request #2192: URL: https://github.com/apache/hadoop/pull/2192#issuecomment-669276590 > given a new config option is going in.. > > * what does it do? > * is there any reason _not_ to document it? This was related to the JIRA https://issues.apache.org/jira/browse/HADOOP-16455 The checkaccess config is documented in that commit. Now we are making the default value for the config to be true. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] steveloughran commented on pull request #2192: HADOOP-17183. ABFS: Enabling checkaccess on ABFS
steveloughran commented on pull request #2192: URL: https://github.com/apache/hadoop/pull/2192#issuecomment-669227108 given a new config option is going in.. * what does it do? * is there any reason _not_ to document it? This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #2188: HDFS-15499. Clean up httpfs/pom.xml to remove aws-java-sdk-s3 exclusion
hadoop-yetus commented on pull request #2188: URL: https://github.com/apache/hadoop/pull/2188#issuecomment-669176207 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | +0 :ok: | reexec | 0m 30s | Docker mode activated. | ||| _ Prechecks _ | | +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. | | +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. | | -1 :x: | test4tests | 0m 0s | The patch doesn't appear to include any new or modified tests. Please justify why no new tests are needed for this patch. Also please list what manual steps were performed to verify this patch. | ||| _ trunk Compile Tests _ | | +0 :ok: | mvndep | 3m 22s | Maven dependency ordering for branch | | +1 :green_heart: | mvninstall | 26m 10s | trunk passed | | +1 :green_heart: | compile | 19m 26s | trunk passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | compile | 16m 54s | trunk passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | +1 :green_heart: | mvnsite | 20m 55s | trunk passed | | +1 :green_heart: | shadedclient | 14m 11s | branch has no errors when building and testing our client artifacts. | | +1 :green_heart: | javadoc | 5m 40s | trunk passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | javadoc | 7m 8s | trunk passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | ||| _ Patch Compile Tests _ | | +0 :ok: | mvndep | 0m 33s | Maven dependency ordering for patch | | +1 :green_heart: | mvninstall | 19m 34s | the patch passed | | +1 :green_heart: | compile | 18m 58s | the patch passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | javac | 18m 58s | the patch passed | | +1 :green_heart: | compile | 16m 55s | the patch passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | +1 :green_heart: | javac | 16m 55s | the patch passed | | +1 :green_heart: | mvnsite | 17m 29s | the patch passed | | +1 :green_heart: | shellcheck | 0m 0s | There were no new shellcheck issues. | | +1 :green_heart: | shelldocs | 0m 19s | There were no new shelldocs issues. | | +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. | | +1 :green_heart: | xml | 0m 1s | The patch has no ill-formed XML file. | | +1 :green_heart: | shadedclient | 14m 10s | patch has no errors when building and testing our client artifacts. | | +1 :green_heart: | javadoc | 5m 30s | the patch passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | javadoc | 7m 4s | the patch passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | ||| _ Other Tests _ | | -1 :x: | unit | 569m 10s | root in the patch passed. | | -1 :x: | asflicense | 1m 49s | The patch generated 1 ASF License warnings. | | | | 790m 2s | | | Reason | Tests | |---:|:--| | Failed junit tests | hadoop.yarn.applications.distributedshell.TestDistributedShell | | | hadoop.yarn.server.resourcemanager.scheduler.fair.TestFairSchedulerPreemption | | | hadoop.yarn.server.timelineservice.security.TestTimelineAuthFilterForV2 | | | hadoop.yarn.client.api.impl.TestAMRMClient | | | hadoop.hdfs.server.namenode.TestNameNodeRetryCacheMetrics | | | hadoop.hdfs.server.sps.TestExternalStoragePolicySatisfier | | | hadoop.hdfs.server.blockmanagement.TestUnderReplicatedBlocks | | | hadoop.hdfs.server.blockmanagement.TestBlockTokenWithDFSStriped | | | hadoop.fs.contract.hdfs.TestHDFSContractMultipartUploader | | | hadoop.hdfs.TestGetFileChecksum | | Subsystem | Report/Notes | |--:|:-| | Docker | ClientAPI=1.40 ServerAPI=1.40 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2188/3/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hadoop/pull/2188 | | Optional Tests | dupname asflicense shellcheck shelldocs compile javac javadoc mvninstall mvnsite unit shadedclient xml | | uname | Linux 7fe0132b0320 4.15.0-60-generic #67-Ubuntu SMP Thu Aug 22 16:55:30 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux | | Build tool | maven | | Personality | personality/hadoop.sh | | git revision | trunk / ed3ab4b87d9 | | Default Java | Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | unit | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2188/3/artifact/out/patch-unit-root.txt | | Test Results |
[jira] [Commented] (HADOOP-17145) Unauthenticated users are not authorized to access this page message is misleading in HttpServer2.java
[
https://issues.apache.org/jira/browse/HADOOP-17145?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17171454#comment-17171454
]
Hadoop QA commented on HADOOP-17145:
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 29m
22s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} dupname {color} | {color:green} 0m
0s{color} | {color:green} No case conflicting files found. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:red}-1{color} | {color:red} test4tests {color} | {color:red} 0m
0s{color} | {color:red} The patch doesn't appear to include any new or modified
tests. Please justify why no new tests are needed for this patch. Also please
list what manual steps were performed to verify this patch. {color} |
|| || || || {color:brown} trunk Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 18m
39s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 19m
23s{color} | {color:green} trunk passed with JDK
Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 16m
38s{color} | {color:green} trunk passed with JDK Private
Build-1.8.0_252-8u252-b09-1~18.04-b09 {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
51s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 1m
27s{color} | {color:green} trunk passed {color} |
| {color:green}+1{color} | {color:green} shadedclient {color} | {color:green}
16m 20s{color} | {color:green} branch has no errors when building and testing
our client artifacts. {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
37s{color} | {color:green} trunk passed with JDK
Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
35s{color} | {color:green} trunk passed with JDK Private
Build-1.8.0_252-8u252-b09-1~18.04-b09 {color} |
| {color:blue}0{color} | {color:blue} spotbugs {color} | {color:blue} 2m
14s{color} | {color:blue} Used deprecated FindBugs config; considering
switching to SpotBugs. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 2m
12s{color} | {color:green} trunk passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 0m
52s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 18m
45s{color} | {color:green} the patch passed with JDK
Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 18m
45s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 16m
36s{color} | {color:green} the patch passed with JDK Private
Build-1.8.0_252-8u252-b09-1~18.04-b09 {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 16m
36s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
50s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 1m
26s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:green}+1{color} | {color:green} shadedclient {color} | {color:green}
13m 52s{color} | {color:green} patch has no errors when building and testing
our client artifacts. {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 0m
36s{color} | {color:green} the patch passed with JDK
Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
34s{color} | {color:green} the patch passed with JDK Private
Build-1.8.0_252-8u252-b09-1~18.04-b09 {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 2m
20s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:red}-1{color} | {color:red} unit {color} | {color:red} 9m 42s{color}
| {color:red} hadoop-common in the patch passed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} |
[jira] [Resolved] (HADOOP-17186) Fixing javadoc in ListingOperationCallbacks
[
https://issues.apache.org/jira/browse/HADOOP-17186?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Akira Ajisaka resolved HADOOP-17186.
Fix Version/s: 3.4.0
Hadoop Flags: Reviewed
Resolution: Fixed
Merged the PR into trunk.
> Fixing javadoc in ListingOperationCallbacks
> ---
>
> Key: HADOOP-17186
> URL: https://issues.apache.org/jira/browse/HADOOP-17186
> Project: Hadoop Common
> Issue Type: Bug
> Components: build, documentation
>Reporter: Akira Ajisaka
>Assignee: Mukund Thakur
>Priority: Major
> Labels: newbie
> Fix For: 3.4.0
>
>
> {{mvn javadoc:javadoc -pl hadoop-tools/hadoop-aws}} is failing:
> {noformat}
> [ERROR]
> /Users/aajisaka/git/hadoop/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java:80:
> error: unexpected text
> [ERROR]* {@link this.getMaxKeys()}.
> [ERROR] ^
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17186) Fixing javadoc in ListingOperationCallbacks
[
https://issues.apache.org/jira/browse/HADOOP-17186?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Akira Ajisaka updated HADOOP-17186:
---
Summary: Fixing javadoc in ListingOperationCallbacks (was: mvn
javadoc:javadoc fails in hadoop-aws)
> Fixing javadoc in ListingOperationCallbacks
> ---
>
> Key: HADOOP-17186
> URL: https://issues.apache.org/jira/browse/HADOOP-17186
> Project: Hadoop Common
> Issue Type: Bug
> Components: build, documentation
>Reporter: Akira Ajisaka
>Assignee: Mukund Thakur
>Priority: Major
> Labels: newbie
>
> {{mvn javadoc:javadoc -pl hadoop-tools/hadoop-aws}} is failing:
> {noformat}
> [ERROR]
> /Users/aajisaka/git/hadoop/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java:80:
> error: unexpected text
> [ERROR]* {@link this.getMaxKeys()}.
> [ERROR] ^
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] aajisaka commented on pull request #2196: HADOOP-17186 fixing javadoc in ListingOperationCallbacks
aajisaka commented on pull request #2196: URL: https://github.com/apache/hadoop/pull/2196#issuecomment-669144691 Thank you @mukund-thakur for your fix! This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] aajisaka merged pull request #2196: HADOOP-17186 fixing javadoc in ListingOperationCallbacks
aajisaka merged pull request #2196: URL: https://github.com/apache/hadoop/pull/2196 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #2196: HADOOP-17186 fixing javadoc in ListingOperationCallbacks
hadoop-yetus commented on pull request #2196: URL: https://github.com/apache/hadoop/pull/2196#issuecomment-669144178 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | +0 :ok: | reexec | 4m 2s | Docker mode activated. | ||| _ Prechecks _ | | +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. | | +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. | | -1 :x: | test4tests | 0m 0s | The patch doesn't appear to include any new or modified tests. Please justify why no new tests are needed for this patch. Also please list what manual steps were performed to verify this patch. | ||| _ trunk Compile Tests _ | | +1 :green_heart: | mvninstall | 33m 4s | trunk passed | | +1 :green_heart: | compile | 0m 45s | trunk passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | compile | 0m 34s | trunk passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | +1 :green_heart: | checkstyle | 0m 30s | trunk passed | | +1 :green_heart: | mvnsite | 0m 46s | trunk passed | | +1 :green_heart: | shadedclient | 18m 2s | branch has no errors when building and testing our client artifacts. | | +1 :green_heart: | javadoc | 0m 21s | trunk passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | javadoc | 0m 30s | trunk passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | +0 :ok: | spotbugs | 1m 10s | Used deprecated FindBugs config; considering switching to SpotBugs. | | +1 :green_heart: | findbugs | 1m 8s | trunk passed | ||| _ Patch Compile Tests _ | | +1 :green_heart: | mvninstall | 0m 37s | the patch passed | | +1 :green_heart: | compile | 0m 41s | the patch passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | javac | 0m 41s | the patch passed | | +1 :green_heart: | compile | 0m 30s | the patch passed with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | +1 :green_heart: | javac | 0m 30s | the patch passed | | +1 :green_heart: | checkstyle | 0m 20s | the patch passed | | +1 :green_heart: | mvnsite | 0m 35s | the patch passed | | +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. | | +1 :green_heart: | shadedclient | 16m 5s | patch has no errors when building and testing our client artifacts. | | +1 :green_heart: | javadoc | 0m 18s | the patch passed with JDK Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 | | +1 :green_heart: | javadoc | 0m 26s | hadoop-tools_hadoop-aws-jdkPrivateBuild-1.8.0_252-8u252-b09-1~18.04-b09 with JDK Private Build-1.8.0_252-8u252-b09-1~18.04-b09 generated 0 new + 4 unchanged - 2 fixed = 4 total (was 6) | | +1 :green_heart: | findbugs | 1m 10s | the patch passed | ||| _ Other Tests _ | | +1 :green_heart: | unit | 1m 32s | hadoop-aws in the patch passed. | | +1 :green_heart: | asflicense | 0m 33s | The patch does not generate ASF License warnings. | | | | 84m 39s | | | Subsystem | Report/Notes | |--:|:-| | Docker | ClientAPI=1.40 ServerAPI=1.40 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2196/1/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hadoop/pull/2196 | | Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient findbugs checkstyle | | uname | Linux 0704ed313b67 4.15.0-58-generic #64-Ubuntu SMP Tue Aug 6 11:12:41 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux | | Build tool | maven | | Personality | personality/hadoop.sh | | git revision | trunk / 58def7cecbe | | Default Java | Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.8+10-post-Ubuntu-0ubuntu118.04.1 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_252-8u252-b09-1~18.04-b09 | | Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2196/1/testReport/ | | Max. process+thread count | 423 (vs. ulimit of 5500) | | modules | C: hadoop-tools/hadoop-aws U: hadoop-tools/hadoop-aws | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2196/1/console | | versions | git=2.17.1 maven=3.6.0 findbugs=4.0.6 | | Powered by | Apache Yetus 0.13.0-SNAPSHOT https://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact
[GitHub] [hadoop] mukund-thakur commented on pull request #2196: HADOOP-17186 fixing javadoc in ListingOperationCallbacks
mukund-thakur commented on pull request #2196: URL: https://github.com/apache/hadoop/pull/2196#issuecomment-669098471 CC @aajisaka Please review. Thanks This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] mukund-thakur opened a new pull request #2196: HADOOP-17186 fixing javadoc in ListingOperationCallbacks
mukund-thakur opened a new pull request #2196: URL: https://github.com/apache/hadoop/pull/2196 ## NOTICE Please create an issue in ASF JIRA before opening a pull request, and you need to set the title of the pull request which starts with the corresponding JIRA issue number. (e.g. HADOOP-X. Fix a typo in YYY.) For more details, please see https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Assigned] (HADOOP-17186) mvn javadoc:javadoc fails in hadoop-aws
[
https://issues.apache.org/jira/browse/HADOOP-17186?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Mukund Thakur reassigned HADOOP-17186:
--
Assignee: Mukund Thakur
> mvn javadoc:javadoc fails in hadoop-aws
> ---
>
> Key: HADOOP-17186
> URL: https://issues.apache.org/jira/browse/HADOOP-17186
> Project: Hadoop Common
> Issue Type: Bug
> Components: build, documentation
>Reporter: Akira Ajisaka
>Assignee: Mukund Thakur
>Priority: Major
> Labels: newbie
>
> {{mvn javadoc:javadoc -pl hadoop-tools/hadoop-aws}} is failing:
> {noformat}
> [ERROR]
> /Users/aajisaka/git/hadoop/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java:80:
> error: unexpected text
> [ERROR]* {@link this.getMaxKeys()}.
> [ERROR] ^
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] bshashikant commented on a change in pull request #2176: HDFS-15492. Make trash root inside each snapshottable directory
bshashikant commented on a change in pull request #2176:
URL: https://github.com/apache/hadoop/pull/2176#discussion_r465605682
##
File path:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
##
@@ -2144,4 +2146,180 @@ public void testECCloseCommittedBlock() throws
Exception {
LambdaTestUtils.intercept(IOException.class, "", () -> str.close());
}
}
+
+ @Test
+ public void testGetTrashRoot() throws IOException {
+Configuration conf = getTestConfiguration();
+conf.setBoolean(DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED, true);
+MiniDFSCluster cluster =
+new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+try {
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ Path testDir = new Path("/ssgtr/test1/");
+ Path file0path = new Path(testDir, "file-0");
+ dfs.create(file0path);
+
+ Path trBeforeAllowSnapshot = dfs.getTrashRoot(file0path);
+ String trBeforeAllowSnapshotStr =
trBeforeAllowSnapshot.toUri().getPath();
+ // The trash root should be in user home directory
+ String homeDirStr = dfs.getHomeDirectory().toUri().getPath();
+ assertTrue(trBeforeAllowSnapshotStr.startsWith(homeDirStr));
+
+ dfs.allowSnapshot(testDir);
+
+ Path trAfterAllowSnapshot = dfs.getTrashRoot(file0path);
+ String trAfterAllowSnapshotStr = trAfterAllowSnapshot.toUri().getPath();
+ // The trash root should now be in the snapshot root
+ String testDirStr = testDir.toUri().getPath();
+ assertTrue(trAfterAllowSnapshotStr.startsWith(testDirStr));
+
+ // Cleanup
+ dfs.disallowSnapshot(testDir);
+ dfs.delete(testDir, true);
+} finally {
+ if (cluster != null) {
+cluster.shutdown();
+ }
+}
+ }
+
+ private boolean isPathInUserHome(String pathStr, DistributedFileSystem dfs) {
+String homeDirStr = dfs.getHomeDirectory().toUri().getPath();
+return pathStr.startsWith(homeDirStr);
+ }
+
+ @Test
+ public void testGetTrashRoots() throws IOException {
+Configuration conf = getTestConfiguration();
+conf.setBoolean(DFS_NAMENODE_SNAPSHOT_TRASHROOT_ENABLED, true);
+MiniDFSCluster cluster =
+new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+try {
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ Path testDir = new Path("/ssgtr/test1/");
+ Path file0path = new Path(testDir, "file-0");
+ dfs.create(file0path);
+ // Create user trash
+ Path currUserHome = dfs.getHomeDirectory();
+ Path currUserTrash = new Path(currUserHome, FileSystem.TRASH_PREFIX);
+ dfs.mkdirs(currUserTrash);
+ // Create trash inside test directory
+ Path testDirTrash = new Path(testDir, FileSystem.TRASH_PREFIX);
+ Path testDirTrashCurrUser = new Path(testDirTrash,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ dfs.mkdirs(testDirTrashCurrUser);
+
+ Collection trashRoots = dfs.getTrashRoots(false);
+ // getTrashRoots should only return 1 empty user trash in the home dir
now
+ assertEquals(1, trashRoots.size());
+ FileStatus firstFileStatus = trashRoots.iterator().next();
+ String pathStr = firstFileStatus.getPath().toUri().getPath();
+ assertTrue(isPathInUserHome(pathStr, dfs));
+ // allUsers should not make a difference for now because we have one user
+ Collection trashRootsAllUsers = dfs.getTrashRoots(true);
+ assertEquals(trashRoots, trashRootsAllUsers);
+
+ dfs.allowSnapshot(testDir);
+
+ Collection trashRootsAfter = dfs.getTrashRoots(false);
+ // getTrashRoots should return 1 more trash root inside snapshottable dir
+ assertEquals(trashRoots.size() + 1, trashRootsAfter.size());
+ boolean foundUserHomeTrash = false;
+ boolean foundSnapDirUserTrash = false;
+ String testDirStr = testDir.toUri().getPath();
+ for (FileStatus fileStatus : trashRootsAfter) {
+String currPathStr = fileStatus.getPath().toUri().getPath();
+if (isPathInUserHome(currPathStr, dfs)) {
+ foundUserHomeTrash = true;
+} else if (currPathStr.startsWith(testDirStr)) {
+ foundSnapDirUserTrash = true;
+}
+ }
+ assertTrue(foundUserHomeTrash);
+ assertTrue(foundSnapDirUserTrash);
+ // allUsers should not make a difference for now because we have one user
+ Collection trashRootsAfterAllUsers = dfs.getTrashRoots(true);
+ assertEquals(trashRootsAfter, trashRootsAfterAllUsers);
+
+ // Create trash root for user0
+ UserGroupInformation ugi =
UserGroupInformation.createRemoteUser("user0");
+ String user0HomeStr = DFSUtilClient.getHomeDirectory(conf, ugi);
+ Path user0Trash = new Path(user0HomeStr, FileSystem.TRASH_PREFIX);
+ dfs.mkdirs(user0Trash);
+ // allUsers flag set to false should be unaffected
+ Collection trashRootsAfter2 = dfs.getTrashRoots(false);
+
[GitHub] [hadoop] mukund-thakur commented on pull request #2148: HADOOP-17131. Refactor S3A Listing code for better isolation.
mukund-thakur commented on pull request #2148: URL: https://github.com/apache/hadoop/pull/2148#issuecomment-669095775 I was ignoring java doc issues because of https://issues.apache.org/jira/browse/HADOOP-17091. Looks like it got fixed recently. Sorry. I will create a small patch and fix https://issues.apache.org/jira/browse/HADOOP-17186 Thanks for pointing out. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #2149: HADOOP-13230. S3A to optionally retain directory markers
hadoop-yetus commented on pull request #2149: URL: https://github.com/apache/hadoop/pull/2149#issuecomment-669085983 :broken_heart: **-1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | +0 :ok: | reexec | 0m 0s | Docker mode activated. | | -1 :x: | docker | 8m 48s | Docker failed to build yetus/hadoop:e277d338da2. | | Subsystem | Report/Notes | |--:|:-| | GITHUB PR | https://github.com/apache/hadoop/pull/2149 | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2149/8/console | | versions | git=2.17.1 | | Powered by | Apache Yetus 0.13.0-SNAPSHOT https://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17145) Unauthenticated users are not authorized to access this page message is misleading in HttpServer2.java
[ https://issues.apache.org/jira/browse/HADOOP-17145?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Andras Bokor updated HADOOP-17145: -- Attachment: HADOOP-17145.003.patch > Unauthenticated users are not authorized to access this page message is > misleading in HttpServer2.java > -- > > Key: HADOOP-17145 > URL: https://issues.apache.org/jira/browse/HADOOP-17145 > Project: Hadoop Common > Issue Type: Bug >Reporter: Andras Bokor >Assignee: Andras Bokor >Priority: Major > Attachments: HADOOP-17145.001.patch, HADOOP-17145.002.patch, > HADOOP-17145.003.patch > > > Recently one of the users were misled by the message "Unauthenticated users > are not authorized to access this page" when the user was not an admin user. > At that point the user is authenticated but has no admin access, so it's > actually not an authentication issue but an authorization issue. > Also, 401 as error code would be better. > Something like "User is unauthorized to access the page" would help to users > to find out what is the problem during access an http endpoint. -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] steveloughran commented on pull request #2149: HADOOP-13230. S3A to optionally retain directory markers
steveloughran commented on pull request #2149: URL: https://github.com/apache/hadoop/pull/2149#issuecomment-669070632 +the marker tool audit operation should add a -nonauth option which doesn't worry about markers present in authoritative paths. That way you can do a bucket audit without it failing on /hive or other specific paths. Without that, to audit an entire bucket, you'd need to list all root directories which weren't authoritative and explicitly audit them straightforward to do -simply leave out markers in auth paths from the audit as they come in from the listObjects call This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #2194: HADOOP-17164. UGI loginUserFromKeytab doesn't set the last login time
hadoop-yetus commented on pull request #2194: URL: https://github.com/apache/hadoop/pull/2194#issuecomment-669065207 :confetti_ball: **+1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | +0 :ok: | reexec | 0m 43s | Docker mode activated. | ||| _ Prechecks _ | | +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. | | +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. | | +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. | ||| _ branch-2.10 Compile Tests _ | | +1 :green_heart: | mvninstall | 20m 24s | branch-2.10 passed | | +1 :green_heart: | compile | 16m 41s | branch-2.10 passed | | +1 :green_heart: | checkstyle | 0m 48s | branch-2.10 passed | | +1 :green_heart: | mvnsite | 1m 49s | branch-2.10 passed | | +1 :green_heart: | javadoc | 1m 29s | branch-2.10 passed | | +0 :ok: | spotbugs | 3m 15s | Used deprecated FindBugs config; considering switching to SpotBugs. | | +1 :green_heart: | findbugs | 3m 11s | branch-2.10 passed | ||| _ Patch Compile Tests _ | | +1 :green_heart: | mvninstall | 1m 11s | the patch passed | | +1 :green_heart: | compile | 16m 36s | the patch passed | | +1 :green_heart: | javac | 16m 36s | the patch passed | | -0 :warning: | checkstyle | 0m 50s | hadoop-common-project/hadoop-common: The patch generated 1 new + 95 unchanged - 0 fixed = 96 total (was 95) | | +1 :green_heart: | mvnsite | 1m 56s | the patch passed | | +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. | | +1 :green_heart: | javadoc | 1m 29s | the patch passed | | +1 :green_heart: | findbugs | 3m 35s | the patch passed | ||| _ Other Tests _ | | +1 :green_heart: | unit | 11m 48s | hadoop-common in the patch passed. | | +1 :green_heart: | asflicense | 1m 0s | The patch does not generate ASF License warnings. | | | | 87m 45s | | | Subsystem | Report/Notes | |--:|:-| | Docker | ClientAPI=1.40 ServerAPI=1.40 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2194/3/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hadoop/pull/2194 | | Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient findbugs checkstyle | | uname | Linux 6cc565a20bee 4.15.0-60-generic #67-Ubuntu SMP Thu Aug 22 16:55:30 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux | | Build tool | maven | | Personality | personality/hadoop.sh | | git revision | branch-2.10 / 8d5821b | | Default Java | Oracle Corporation-1.7.0_95-b00 | | checkstyle | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2194/3/artifact/out/diff-checkstyle-hadoop-common-project_hadoop-common.txt | | Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2194/3/testReport/ | | Max. process+thread count | 1352 (vs. ulimit of 5500) | | modules | C: hadoop-common-project/hadoop-common U: hadoop-common-project/hadoop-common | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2194/3/console | | versions | git=2.7.4 maven=3.3.9 findbugs=3.0.1 | | Powered by | Apache Yetus 0.12.0 https://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17186) mvn javadoc:javadoc fails in hadoop-aws
[
https://issues.apache.org/jira/browse/HADOOP-17186?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Akira Ajisaka updated HADOOP-17186:
---
Labels: newbie (was: )
> mvn javadoc:javadoc fails in hadoop-aws
> ---
>
> Key: HADOOP-17186
> URL: https://issues.apache.org/jira/browse/HADOOP-17186
> Project: Hadoop Common
> Issue Type: Bug
> Components: build, documentation
>Reporter: Akira Ajisaka
>Priority: Major
> Labels: newbie
>
> {{mvn javadoc:javadoc -pl hadoop-tools/hadoop-aws}} is failing:
> {noformat}
> [ERROR]
> /Users/aajisaka/git/hadoop/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java:80:
> error: unexpected text
> [ERROR]* {@link this.getMaxKeys()}.
> [ERROR] ^
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17131) Refactor S3A Listing code for better isolation
[ https://issues.apache.org/jira/browse/HADOOP-17131?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Akira Ajisaka updated HADOOP-17131: --- Fix Version/s: 3.4.0 > Refactor S3A Listing code for better isolation > -- > > Key: HADOOP-17131 > URL: https://issues.apache.org/jira/browse/HADOOP-17131 > Project: Hadoop Common > Issue Type: Sub-task > Components: fs/s3 >Affects Versions: 3.3.0 >Reporter: Mukund Thakur >Assignee: Mukund Thakur >Priority: Major > Fix For: 3.3.1, 3.4.0 > > > Currently Listing has a S3AFileSystem instance to use its required methods > thus giving listing access to all of S3AFileSystem methods. Ideal way would > be to use a callback interface in Listing which can in turn call s3Afs > methods thus restricting the listing to methods which are absolutely required. > > CC [~ste...@apache.org] -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] aajisaka commented on pull request #2148: HADOOP-17131. Refactor S3A Listing code for better isolation.
aajisaka commented on pull request #2148: URL: https://github.com/apache/hadoop/pull/2148#issuecomment-669062015 Hi @steveloughran and @mukund-thakur `mvn javadoc:javadoc` fails after this commit. Filed https://issues.apache.org/jira/browse/HADOOP-17186 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Commented] (HADOOP-17186) mvn javadoc:javadoc fails in hadoop-aws
[
https://issues.apache.org/jira/browse/HADOOP-17186?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17171343#comment-17171343
]
Akira Ajisaka commented on HADOOP-17186:
{{@link #getMaxKeys()}} is correct.
> mvn javadoc:javadoc fails in hadoop-aws
> ---
>
> Key: HADOOP-17186
> URL: https://issues.apache.org/jira/browse/HADOOP-17186
> Project: Hadoop Common
> Issue Type: Bug
> Components: build, documentation
>Reporter: Akira Ajisaka
>Priority: Major
>
> {{mvn javadoc:javadoc -pl hadoop-tools/hadoop-aws}} is failing:
> {noformat}
> [ERROR]
> /Users/aajisaka/git/hadoop/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java:80:
> error: unexpected text
> [ERROR]* {@link this.getMaxKeys()}.
> [ERROR] ^
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Commented] (HADOOP-17164) UGI loginUserFromKeytab doesn't set the last login time
[
https://issues.apache.org/jira/browse/HADOOP-17164?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17171341#comment-17171341
]
Hadoop QA commented on HADOOP-17164:
| (x) *{color:red}-1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
| {color:blue}0{color} | {color:blue} reexec {color} | {color:blue} 1m
15s{color} | {color:blue} Docker mode activated. {color} |
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} dupname {color} | {color:green} 0m
0s{color} | {color:green} No case conflicting files found. {color} |
| {color:green}+1{color} | {color:green} @author {color} | {color:green} 0m
0s{color} | {color:green} The patch does not contain any @author tags. {color} |
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green} 0m
0s{color} | {color:green} The patch appears to include 1 new or modified test
files. {color} |
|| || || || {color:brown} branch-2.10 Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 13m
56s{color} | {color:green} branch-2.10 passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 13m
14s{color} | {color:green} branch-2.10 passed with JDK Oracle
Corporation-1.7.0_95-b00 {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 10m
37s{color} | {color:green} branch-2.10 passed with JDK Private
Build-1.8.0_252-8u252-b09-1~16.04-b09 {color} |
| {color:green}+1{color} | {color:green} checkstyle {color} | {color:green} 0m
42s{color} | {color:green} branch-2.10 passed {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 1m
18s{color} | {color:green} branch-2.10 passed {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
14s{color} | {color:green} branch-2.10 passed with JDK Oracle
Corporation-1.7.0_95-b00 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
4s{color} | {color:green} branch-2.10 passed with JDK Private
Build-1.8.0_252-8u252-b09-1~16.04-b09 {color} |
| {color:blue}0{color} | {color:blue} spotbugs {color} | {color:blue} 2m
0s{color} | {color:blue} Used deprecated FindBugs config; considering switching
to SpotBugs. {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 1m
57s{color} | {color:green} branch-2.10 passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} mvninstall {color} | {color:green} 0m
44s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 12m
10s{color} | {color:green} the patch passed with JDK Oracle
Corporation-1.7.0_95-b00 {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 12m
10s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} compile {color} | {color:green} 10m
40s{color} | {color:green} the patch passed with JDK Private
Build-1.8.0_252-8u252-b09-1~16.04-b09 {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green} 10m
40s{color} | {color:green} the patch passed {color} |
| {color:orange}-0{color} | {color:orange} checkstyle {color} | {color:orange}
0m 40s{color} | {color:orange} hadoop-common-project/hadoop-common: The patch
generated 1 new + 95 unchanged - 0 fixed = 96 total (was 95) {color} |
| {color:green}+1{color} | {color:green} mvnsite {color} | {color:green} 1m
15s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} whitespace {color} | {color:green} 0m
0s{color} | {color:green} The patch has no whitespace issues. {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
13s{color} | {color:green} the patch passed with JDK Oracle
Corporation-1.7.0_95-b00 {color} |
| {color:green}+1{color} | {color:green} javadoc {color} | {color:green} 1m
5s{color} | {color:green} the patch passed with JDK Private
Build-1.8.0_252-8u252-b09-1~16.04-b09 {color} |
| {color:green}+1{color} | {color:green} findbugs {color} | {color:green} 2m
5s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:red}-1{color} | {color:red} unit {color} | {color:red} 9m 41s{color}
| {color:red} hadoop-common in the patch failed. {color} |
| {color:green}+1{color} | {color:green} asflicense {color} | {color:green} 0m
48s{color} | {color:green} The patch does not generate ASF License warnings.
{color} |
| {color:black}{color} | {color:black} {color} | {color:black} 89m 8s{color} |
{color:black} {color} |
\\
\\
|| Reason || Tests ||
| Failed junit tests | hadoop.util.TestReadWriteDiskValidator |
| | hadoop.net.TestClusterTopology |
\\
\\
|| Subsystem || Report/Notes ||
| Docker | ClientAPI=1.40
[jira] [Created] (HADOOP-17186) mvn javadoc:javadoc fails in hadoop-aws
Akira Ajisaka created HADOOP-17186:
--
Summary: mvn javadoc:javadoc fails in hadoop-aws
Key: HADOOP-17186
URL: https://issues.apache.org/jira/browse/HADOOP-17186
Project: Hadoop Common
Issue Type: Bug
Components: build, documentation
Reporter: Akira Ajisaka
{{mvn javadoc:javadoc -pl hadoop-tools/hadoop-aws}} is failing:
{noformat}
[ERROR]
/Users/aajisaka/git/hadoop/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java:80:
error: unexpected text
[ERROR]* {@link this.getMaxKeys()}.
[ERROR] ^
{noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #2194: HADOOP-17164. UGI loginUserFromKeytab doesn't set the last login time
hadoop-yetus commented on pull request #2194: URL: https://github.com/apache/hadoop/pull/2194#issuecomment-669045607 :confetti_ball: **+1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | +0 :ok: | reexec | 1m 11s | Docker mode activated. | ||| _ Prechecks _ | | +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. | | +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. | | +1 :green_heart: | test4tests | 0m 0s | The patch appears to include 1 new or modified test files. | ||| _ branch-2.10 Compile Tests _ | | +1 :green_heart: | mvninstall | 16m 25s | branch-2.10 passed | | +1 :green_heart: | compile | 13m 11s | branch-2.10 passed | | +1 :green_heart: | checkstyle | 0m 41s | branch-2.10 passed | | +1 :green_heart: | mvnsite | 1m 30s | branch-2.10 passed | | +1 :green_heart: | javadoc | 1m 27s | branch-2.10 passed | | +0 :ok: | spotbugs | 3m 33s | Used deprecated FindBugs config; considering switching to SpotBugs. | | +1 :green_heart: | findbugs | 3m 28s | branch-2.10 passed | ||| _ Patch Compile Tests _ | | +1 :green_heart: | mvninstall | 1m 13s | the patch passed | | +1 :green_heart: | compile | 14m 46s | the patch passed | | +1 :green_heart: | javac | 14m 46s | the patch passed | | -0 :warning: | checkstyle | 0m 35s | hadoop-common-project/hadoop-common: The patch generated 1 new + 95 unchanged - 0 fixed = 96 total (was 95) | | +1 :green_heart: | mvnsite | 1m 22s | the patch passed | | +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. | | +1 :green_heart: | javadoc | 1m 21s | the patch passed | | +1 :green_heart: | findbugs | 3m 11s | the patch passed | ||| _ Other Tests _ | | +1 :green_heart: | unit | 11m 48s | hadoop-common in the patch passed. | | +1 :green_heart: | asflicense | 0m 58s | The patch does not generate ASF License warnings. | | | | 77m 8s | | | Subsystem | Report/Notes | |--:|:-| | Docker | ClientAPI=1.40 ServerAPI=1.40 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2194/2/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hadoop/pull/2194 | | Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient findbugs checkstyle | | uname | Linux 38d7253d5bf0 4.15.0-60-generic #67-Ubuntu SMP Thu Aug 22 16:55:30 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux | | Build tool | maven | | Personality | personality/hadoop.sh | | git revision | branch-2.10 / 8d5821b | | Default Java | Oracle Corporation-1.7.0_95-b00 | | checkstyle | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2194/2/artifact/out/diff-checkstyle-hadoop-common-project_hadoop-common.txt | | Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2194/2/testReport/ | | Max. process+thread count | 1568 (vs. ulimit of 5500) | | modules | C: hadoop-common-project/hadoop-common U: hadoop-common-project/hadoop-common | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2194/2/console | | versions | git=2.7.4 maven=3.3.9 findbugs=3.0.1 | | Powered by | Apache Yetus 0.12.0 https://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] sguggilam commented on pull request #2194: HADOOP-17164. UGI loginUserFromKeytab doesn't set the last login time
sguggilam commented on pull request #2194: URL: https://github.com/apache/hadoop/pull/2194#issuecomment-669017092 @liuml07 Rebased and pushed the latest code. Also uploaded the patch to JIRA for the QA run This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17164) UGI loginUserFromKeytab doesn't set the last login time
[ https://issues.apache.org/jira/browse/HADOOP-17164?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Sandeep Guggilam updated HADOOP-17164: -- Attachment: HADOOP-17164-branch-2.10.002.patch > UGI loginUserFromKeytab doesn't set the last login time > --- > > Key: HADOOP-17164 > URL: https://issues.apache.org/jira/browse/HADOOP-17164 > Project: Hadoop Common > Issue Type: Bug > Components: security >Affects Versions: 2.10.0, 3.3.0, 3.2.1, 3.1.3 >Reporter: Sandeep Guggilam >Assignee: Sandeep Guggilam >Priority: Major > Fix For: 3.1.4, 3.2.2, 3.3.1, 3.4.0 > > Attachments: HADOOP-17164-branch-2.10.001.patch, > HADOOP-17164-branch-2.10.002.patch, HADOOP-17164.001.patch > > > UGI initial login from keytab doesn't set the last login time as a result of > which the relogin can happen even before the configured minimum seconds to > wait before relogin -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17164) UGI loginUserFromKeytab doesn't set the last login time
[ https://issues.apache.org/jira/browse/HADOOP-17164?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Sandeep Guggilam updated HADOOP-17164: -- Attachment: (was: HADOOP-17164-branch-2.10.002.patch) > UGI loginUserFromKeytab doesn't set the last login time > --- > > Key: HADOOP-17164 > URL: https://issues.apache.org/jira/browse/HADOOP-17164 > Project: Hadoop Common > Issue Type: Bug > Components: security >Affects Versions: 2.10.0, 3.3.0, 3.2.1, 3.1.3 >Reporter: Sandeep Guggilam >Assignee: Sandeep Guggilam >Priority: Major > Fix For: 3.1.4, 3.2.2, 3.3.1, 3.4.0 > > Attachments: HADOOP-17164-branch-2.10.001.patch, > HADOOP-17164.001.patch > > > UGI initial login from keytab doesn't set the last login time as a result of > which the relogin can happen even before the configured minimum seconds to > wait before relogin -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17164) UGI loginUserFromKeytab doesn't set the last login time
[ https://issues.apache.org/jira/browse/HADOOP-17164?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Sandeep Guggilam updated HADOOP-17164: -- Attachment: HADOOP-17164-branch-2.10.002.patch > UGI loginUserFromKeytab doesn't set the last login time > --- > > Key: HADOOP-17164 > URL: https://issues.apache.org/jira/browse/HADOOP-17164 > Project: Hadoop Common > Issue Type: Bug > Components: security >Affects Versions: 2.10.0, 3.3.0, 3.2.1, 3.1.3 >Reporter: Sandeep Guggilam >Assignee: Sandeep Guggilam >Priority: Major > Fix For: 3.1.4, 3.2.2, 3.3.1, 3.4.0 > > Attachments: HADOOP-17164-branch-2.10.001.patch, > HADOOP-17164-branch-2.10.002.patch, HADOOP-17164.001.patch > > > UGI initial login from keytab doesn't set the last login time as a result of > which the relogin can happen even before the configured minimum seconds to > wait before relogin -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17164) UGI loginUserFromKeytab doesn't set the last login time
[ https://issues.apache.org/jira/browse/HADOOP-17164?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Sandeep Guggilam updated HADOOP-17164: -- Attachment: (was: HADOOP-17164-branch-2.10.002.patch) > UGI loginUserFromKeytab doesn't set the last login time > --- > > Key: HADOOP-17164 > URL: https://issues.apache.org/jira/browse/HADOOP-17164 > Project: Hadoop Common > Issue Type: Bug > Components: security >Affects Versions: 2.10.0, 3.3.0, 3.2.1, 3.1.3 >Reporter: Sandeep Guggilam >Assignee: Sandeep Guggilam >Priority: Major > Fix For: 3.1.4, 3.2.2, 3.3.1, 3.4.0 > > Attachments: HADOOP-17164-branch-2.10.001.patch, > HADOOP-17164.001.patch > > > UGI initial login from keytab doesn't set the last login time as a result of > which the relogin can happen even before the configured minimum seconds to > wait before relogin -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Updated] (HADOOP-17164) UGI loginUserFromKeytab doesn't set the last login time
[ https://issues.apache.org/jira/browse/HADOOP-17164?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Sandeep Guggilam updated HADOOP-17164: -- Attachment: HADOOP-17164-branch-2.10.002.patch > UGI loginUserFromKeytab doesn't set the last login time > --- > > Key: HADOOP-17164 > URL: https://issues.apache.org/jira/browse/HADOOP-17164 > Project: Hadoop Common > Issue Type: Bug > Components: security >Affects Versions: 2.10.0, 3.3.0, 3.2.1, 3.1.3 >Reporter: Sandeep Guggilam >Assignee: Sandeep Guggilam >Priority: Major > Fix For: 3.1.4, 3.2.2, 3.3.1, 3.4.0 > > Attachments: HADOOP-17164-branch-2.10.001.patch, > HADOOP-17164-branch-2.10.002.patch, HADOOP-17164.001.patch > > > UGI initial login from keytab doesn't set the last login time as a result of > which the relogin can happen even before the configured minimum seconds to > wait before relogin -- This message was sent by Atlassian Jira (v8.3.4#803005) - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[jira] [Resolved] (HADOOP-17185) Set MAVEN_OPTS in Jenkinsfile
[
https://issues.apache.org/jira/browse/HADOOP-17185?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Akira Ajisaka resolved HADOOP-17185.
Fix Version/s: 2.10.1
2.9.3
Resolution: Fixed
Merged the PR into branch-2.10 and branch-2.9.
> Set MAVEN_OPTS in Jenkinsfile
> -
>
> Key: HADOOP-17185
> URL: https://issues.apache.org/jira/browse/HADOOP-17185
> Project: Hadoop Common
> Issue Type: Bug
> Components: build
>Reporter: Akira Ajisaka
>Assignee: Akira Ajisaka
>Priority: Major
> Fix For: 2.9.3, 2.10.1
>
>
> MAVEN_OPTS is not honored from Dockerfile, so we need to set MAVEN_OPTS in
> Jenkinsfile explicitly.
> Especially, {{-Dhttps.protocols=TLSv1.2}} is required when running with Java
> 7.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] aajisaka commented on pull request #2195: HADOOP-17185. Set MAVEN_OPTS in Jenkinsfile
aajisaka commented on pull request #2195: URL: https://github.com/apache/hadoop/pull/2195#issuecomment-669005341 Merged. Thank you @liuml07 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] aajisaka merged pull request #2195: HADOOP-17185. Set MAVEN_OPTS in Jenkinsfile
aajisaka merged pull request #2195: URL: https://github.com/apache/hadoop/pull/2195 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #2195: HADOOP-17185. Set MAVEN_OPTS in Jenkinsfile
hadoop-yetus commented on pull request #2195: URL: https://github.com/apache/hadoop/pull/2195#issuecomment-669003812 :confetti_ball: **+1 overall** | Vote | Subsystem | Runtime | Comment | |::|--:|:|:| | +0 :ok: | reexec | 31m 6s | Docker mode activated. | ||| _ Prechecks _ | | +1 :green_heart: | dupname | 0m 0s | No case conflicting files found. | | +1 :green_heart: | @author | 0m 0s | The patch does not contain any @author tags. | ||| _ branch-2.10 Compile Tests _ | ||| _ Patch Compile Tests _ | | +1 :green_heart: | shellcheck | 0m 0s | There were no new shellcheck issues. | | +1 :green_heart: | shelldocs | 0m 14s | There were no new shelldocs issues. | | +1 :green_heart: | whitespace | 0m 0s | The patch has no whitespace issues. | ||| _ Other Tests _ | | +1 :green_heart: | asflicense | 0m 36s | The patch does not generate ASF License warnings. | | | | 34m 26s | | | Subsystem | Report/Notes | |--:|:-| | Docker | ClientAPI=1.40 ServerAPI=1.40 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2195/2/artifact/out/Dockerfile | | GITHUB PR | https://github.com/apache/hadoop/pull/2195 | | Optional Tests | dupname asflicense shellcheck shelldocs | | uname | Linux 854b37515e3d 4.15.0-58-generic #64-Ubuntu SMP Tue Aug 6 11:12:41 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux | | Build tool | maven | | Personality | personality/hadoop.sh | | git revision | branch-2.10 / 39c118c | | Max. process+thread count | 41 (vs. ulimit of 5500) | | modules | C: . U: . | | Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-2195/2/console | | versions | git=2.7.4 maven=3.3.9 shellcheck=0.3.7 | | Powered by | Apache Yetus 0.12.0 https://yetus.apache.org | This message was automatically generated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org - To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
