Copilot commented on code in PR #8347: URL: https://github.com/apache/hadoop/pull/8347#discussion_r3036536093
########## hadoop-cloud-storage-project/hadoop-bos/src/main/java/org/apache/hadoop/fs/bos/NonHierarchyBosNativeFileSystemStore.java: ########## @@ -0,0 +1,457 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.bos; + +import com.baidubce.services.bos.model.CompleteMultipartUploadRequest; +import com.baidubce.services.bos.model.DeleteMultipleObjectsRequest; +import com.baidubce.services.bos.model.InitiateMultipartUploadRequest; +import com.baidubce.services.bos.model.InitiateMultipartUploadResponse; +import com.baidubce.services.bos.model.ObjectMetadata; +import com.baidubce.services.bos.model.PartETag; +import com.baidubce.services.bos.model.UploadPartCopyRequest; +import com.baidubce.services.bos.model.UploadPartCopyResponse; +import org.apache.hadoop.conf.Configuration; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +/** + * Store implementation for non-hierarchy (flat) BOS buckets. + * Directories are simulated using key prefixes and delimiter + * conventions. + */ +public class NonHierarchyBosNativeFileSystemStore + extends BosNativeFileSystemStore { + + private final int multipartBlockSize; + private final long copyLargeFileThreshold; + + /** + * Constructs a store for a non-hierarchy bucket. + * + * @param bosClientProxy the BOS client proxy + * @param bucketName the name of the bucket + * @param conf the Hadoop configuration + */ + public NonHierarchyBosNativeFileSystemStore( + BosClientProxy bosClientProxy, + String bucketName, Configuration conf) { + super(bosClientProxy, bucketName, conf); + this.multipartBlockSize = conf.getInt( + BaiduBosConstants.BOS_MULTI_UPLOAD_BLOCK_SIZE, + BaiduBosConstants + .BOS_MULTI_UPLOAD_BLOCK_SIZE_DEFAULT); + this.copyLargeFileThreshold = conf.getLong( + BaiduBosConstants + .BOS_COPY_LARGE_FILE_THRESHOLD, + BaiduBosConstants + .BOS_COPY_LARGE_FILE_THRESHOLD_DEFAULT); + } + + /** + * {@inheritDoc} + * + * Checks whether the key is a directory by looking for + * the directory marker or listing sub-objects. + */ + @Override + public boolean isDirectory(String key) + throws IOException { + try { + key = prefixToDir(key); + bosClientProxy.getObjectMetadata(bucketName, key); + return true; + } catch (FileNotFoundException ex) { + PartialListing listing = + list(key, 1, null, null); + if (listing != null) { + if (listing.getFiles().length > 0) { + return true; + } + } + } + return false; + } + + /** + * {@inheritDoc} + * + * Retrieves metadata for the given key. If the key is not + * found as-is, tries appending a trailing delimiter. If + * sub-objects exist, rebuilds the directory marker. + */ + @Override + public FileMetadata retrieveMetadata(String key) + throws IOException { + boolean isFolder = false; + try { + ObjectMetadata meta = + bosClientProxy.getObjectMetadata( + bucketName, key); + if (key.endsWith( + BaiduBosFileSystem.PATH_DELIMITER)) { + isFolder = true; + } + long lastMod = + meta.getLastModified() != null + ? meta.getLastModified().getTime() : 0L; + return new FileMetadata( + key, meta.getContentLength(), + lastMod, isFolder, meta); + } catch (FileNotFoundException e) { + try { + key = prefixToDir(key); + ObjectMetadata meta = + bosClientProxy.getObjectMetadata( + bucketName, key); + isFolder = true; + long lastMod = + meta.getLastModified() != null + ? meta.getLastModified().getTime() + : 0L; + return new FileMetadata( + key, meta.getContentLength(), + lastMod, isFolder, meta); + } catch (FileNotFoundException ex) { + PartialListing listing = + list(key, 1, null, null); + if (listing != null) { + if (listing.getFiles().length > 0) { + // rebuild dir + this.storeEmptyFile(key); + return new FileMetadata( + key, 0, 0, true); Review Comment: When rebuilding a missing directory marker, this stores the marker at `key` without ensuring it has the trailing delimiter. In the non-hierarchy implementation, directory markers are identified via `prefixToDir(key)` (trailing `/`). Storing the unsuffixed key can create a zero-byte *file* marker instead of a directory marker and cause subsequent `isDirectory()`/metadata lookups to behave incorrectly. Use `storeEmptyFile(prefixToDir(key))` (and return metadata keyed to the directory form) when reconstructing a directory marker from a non-empty listing. ```suggestion String dirKey = prefixToDir(key); this.storeEmptyFile(dirKey); return new FileMetadata( dirKey, 0, 0, true); ``` ########## hadoop-cloud-storage-project/hadoop-bos/src/main/java/org/apache/hadoop/fs/bos/BosInputStream.java: ########## @@ -0,0 +1,587 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.bos; + +import org.apache.hadoop.fs.CanSetReadahead; +import org.apache.hadoop.fs.FSExceptionMessages; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.TimeUnit; + +/** + * Input stream implementation for reading objects from BOS. + * Supports lazy seeking and readahead buffering. + */ +public class BosInputStream extends FSInputStream + implements CanSetReadahead { + + private static final Logger LOG = + LoggerFactory.getLogger(BosInputStream.class); + + private FileSystem.Statistics statistics; + private BosNativeFileSystemStore store; + private static final long DEFAULT_READAHEAD_LEN = + 1024 * 1024; + private volatile boolean closed; + private final String key; + private final long contentLength; + private InputStream in = null; + private long pos = 0; + private FileMetadata fileMetaData; + private long readahead = DEFAULT_READAHEAD_LEN; + + /** + * This is the actual position within the object, used by + * lazy seek to decide whether to seek on the next read or + * not. + */ + private long nextReadPos = 0; + + /** + * The start of the content range of the last request. + */ + private long contentRangeStart = 0; + + /** + * The end of the content range of the last request. + * This is an absolute value of the range, not a length + * field. + */ + private long contentRangeFinish = 0; + + /** + * Constructs a BosInputStream for the given object key. + * + * @param key the object key in BOS + * @param fileMetaData the file metadata + * @param store the native file system store + * @param statistics the file system statistics + */ + public BosInputStream( + String key, FileMetadata fileMetaData, + BosNativeFileSystemStore store, + FileSystem.Statistics statistics) { + this.key = key; + this.fileMetaData = fileMetaData; + this.contentLength = fileMetaData.getLength(); + this.store = store; + this.statistics = statistics; + } + + /** + * Reads a single byte from the stream. + * + * @return the byte read, or -1 if end of stream + * @throws IOException if an I/O error occurs + */ + public synchronized int read() throws IOException { + checkNotClosed(); + + if (this.contentLength == 0 + || (nextReadPos >= contentLength)) { + return -1; + } + + int byteRead; + try { + lazySeek(nextReadPos, 1); + byteRead = in.read(); + } catch (EOFException e) { + return -1; + } catch (IOException e) { + onReadFailure(e, 1); + byteRead = in.read(); + } + + if (byteRead >= 0) { + pos++; + nextReadPos++; + } + + if (statistics != null && byteRead >= 0) { + statistics.incrementBytesRead(1); + } + return byteRead; + } + + /** + * Reads bytes into a buffer from the stream. + * + * @param buf the buffer to read into + * @param off the start offset in the buffer + * @param len the maximum number of bytes to read + * @return the number of bytes read, or -1 if end of stream + * @throws IOException if an I/O error occurs + */ + public synchronized int read( + byte[] buf, int off, int len) throws IOException { + checkNotClosed(); + + if (len == 0) { + return 0; + } + + if (this.contentLength == 0 + || (nextReadPos >= contentLength)) { + return -1; + } + + // Validate and adjust parameters + if (off < 0 || len < 0 + || len > buf.length - off) { + throw new IndexOutOfBoundsException( + String.format( + "Invalid read parameters:" + + " buf.length=%d, off=%d, len=%d", + buf.length, off, len)); + } + + try { + lazySeek(nextReadPos, len); + } catch (EOFException e) { + // the end of the file has moved + return -1; + } + + int bytesRead; + try { + bytesRead = in.read(buf, off, len); + } catch (EOFException e) { + onReadFailure(e, len); + // the base implementation swallows EOFs. + return -1; + } catch (IOException e) { + onReadFailure(e, len); + bytesRead = in.read(buf, off, len); + } catch (Exception e) { + int intervalSeconds = 3; + int retry = 3; + while (true) { + try { + LOG.info( + "Read exception occur:" + + " retry at most {} times.", + retry); + // Reopen the stream at the current position + // to recover from the exception + onReadFailure(e, len); + // Read from the reopened stream + bytesRead = in.read(buf, off, len); + break; + } catch (EOFException eof) { + return -1; + } catch (IOException ioe) { + // For IOException, try to reopen once more + // and retry + LOG.info( + "IOException during retry," + + " attempting to reopen stream"); + onReadFailure(ioe, len); + bytesRead = in.read(buf, off, len); + break; + } catch (Exception ex) { + if (retry <= 0) { + String errorMsg = + ex.getMessage() != null + ? ex.getMessage() + : ex.getClass().getSimpleName(); + throw new IOException( + "Retry " + retry + + " times to read still" + + " exception: " + errorMsg, + ex); + } + + try { + TimeUnit.SECONDS.sleep( + intervalSeconds); + } catch (InterruptedException ite) { + Thread.currentThread().interrupt(); + throw new IOException( + "Thread interrupted during retry", ite); + } + intervalSeconds *= 2; + retry--; + // Update e to ex for the next iteration + e = ex; + } + } + } + + if (bytesRead > 0) { + pos += bytesRead; + nextReadPos += bytesRead; + } + + if (statistics != null && bytesRead >= 0) { + statistics.incrementBytesRead(bytesRead); + } + + return bytesRead; + } + + /** + * Closes this input stream and releases resources. + * + * @throws IOException if an I/O error occurs + */ + public synchronized void close() throws IOException { + if (!closed) { + closed = true; + + // close or abort the stream + closeStream("close() operation"); + // this is actually a no-op + super.close(); + } + } + + /** + * Seeks to the specified position in the stream. This + * performs a lazy seek; the actual stream repositioning + * happens on the next read. + * + * @param targetPos the target position to seek to + * @throws IOException if an I/O error occurs + */ + public synchronized void seek(long targetPos) + throws IOException { + checkNotClosed(); + + // Do not allow negative seek + if (targetPos < 0 || targetPos > contentLength) { + throw new EOFException( + FSExceptionMessages.NEGATIVE_SEEK + + " " + targetPos); + } + + if (this.contentLength <= 0) { + return; + } + + // Lazy seek + nextReadPos = targetPos; + } + + /** + * Returns the current position in the stream. + * + * @return the current position + * @throws IOException if an I/O error occurs + */ + public synchronized long getPos() throws IOException { + return nextReadPos; + } + + /** + * Seeks to a new source. Not supported; always returns + * false. + * + * @param targetPos the target position + * @return false always + * @throws IOException if an I/O error occurs + */ + public boolean seekToNewSource(long targetPos) + throws IOException { + return false; + } + + /** + * Sets the readahead value for this stream. + * + * @param readahead the readahead length, or null to reset + * to default + */ + public synchronized void setReadahead(Long readahead) { + if (readahead == null) { + this.readahead = DEFAULT_READAHEAD_LEN; + } else { + this.readahead = Math.max( + readahead, DEFAULT_READAHEAD_LEN); + } + } + + /** + * Verify that the input stream is open. Non blocking; this + * gives the last state of the volatile {@link #closed} + * field. + * + * @throws IOException if the connection is closed. + */ + private void checkNotClosed() throws IOException { + if (closed) { + throw new IOException( + key + ": " + + FSExceptionMessages.STREAM_IS_CLOSED); + } + } + + /** + * Perform lazy seek and adjust stream to correct position + * for reading. + * + * @param targetPos position from where data should be read + * @param len length of the content that needs to be read + * @throws IOException if an I/O error occurs + */ + private void lazySeek(long targetPos, long len) + throws IOException { + // For lazy seek + try { + seekInStream(targetPos, len); + } catch (Exception e) { + closeStream("seekInStream failed"); + } + // re-open at specific location if needed + if (in == null) { + LOG.debug( + "re-open at specific locaition: {}", Review Comment: Typo in log message: "locaition" → "location". This shows up in debug logs and makes searching/grepping harder. ```suggestion "re-open at specific location: {}", ``` ########## hadoop-cloud-storage-project/hadoop-bos/src/site/markdown/cloud-storage/index.md: ########## @@ -0,0 +1,392 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# Integration of Baidu BOS in Hadoop + +## Overview + +BOS (Baidu Object Storage) is the object storage service provided by Baidu Cloud. Hadoop-BOS is a connector between computing systems and underlying storage. For systems like Hadoop MR, Hive, Spark and Alluxio, hadoop-bos helps them use BOS as the underlying storage system instead of HDFS. + +## Quick Start + +In quick start, we will use hadoop shell command to access a BOS bucket. + +### Requirements + +1. A Baidu Cloud account. Use the account to create a BOS bucket. +2. A dev environment that can access BOS. E.g. a local server or a Baidu Cloud cloud server. +3. Install hadoop to the dev environment. Hadoop is installed at `$HADOOP_HOME`. + +### Usage + +* Compile hadoop-bos bundle tar. The hadoop-bos bundle is not packaged in hadoop final tar file. +So we have to compile it manually. Download the hadoop project, and build it with command below. +```bash +mvn package -DskipTests -pl org.apache.hadoop:hadoop-bos +``` + +* The bundle jar file is placed at +`$HADOOP_HOME/hadoop-cloud-storage-project/hadoop-bos/target/hadoop-bos-{VERSION}.jar`. + +* Copy the bundle jar to hdfs lib path. The hdfs lib path is +`$HADOOP_HOME/share/hadoop/hdfs/lib`. Remember copying to all hadoop nodes. +```bash +cp hadoop-bos-{VERSION}.jar $HADOOP_HOME/share/hadoop/hdfs/lib/ +``` + +* Configure properties below. + +```xml +<properties> + <property> + <name>fs.defaultFS</name> + <value>bos://{your_bucket_name}/</value> + <description> + The name of the default file system. Make it your BOS bucket. + </description> + </property> Review Comment: The Hadoop configuration example is wrapped in a `<properties>` element, but Hadoop config files use `<configuration>` as the root element. Using `<properties>` will make the example invalid if copy/pasted. Please change the example root tag to `<configuration>`. ########## hadoop-cloud-storage-project/hadoop-bos/src/main/java/org/apache/hadoop/fs/bos/BosInputStream.java: ########## @@ -0,0 +1,587 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.bos; + +import org.apache.hadoop.fs.CanSetReadahead; +import org.apache.hadoop.fs.FSExceptionMessages; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.TimeUnit; + +/** + * Input stream implementation for reading objects from BOS. + * Supports lazy seeking and readahead buffering. + */ +public class BosInputStream extends FSInputStream + implements CanSetReadahead { + + private static final Logger LOG = + LoggerFactory.getLogger(BosInputStream.class); + + private FileSystem.Statistics statistics; + private BosNativeFileSystemStore store; + private static final long DEFAULT_READAHEAD_LEN = + 1024 * 1024; + private volatile boolean closed; + private final String key; + private final long contentLength; + private InputStream in = null; + private long pos = 0; + private FileMetadata fileMetaData; + private long readahead = DEFAULT_READAHEAD_LEN; + + /** + * This is the actual position within the object, used by + * lazy seek to decide whether to seek on the next read or + * not. + */ + private long nextReadPos = 0; + + /** + * The start of the content range of the last request. + */ + private long contentRangeStart = 0; + + /** + * The end of the content range of the last request. + * This is an absolute value of the range, not a length + * field. + */ + private long contentRangeFinish = 0; + + /** + * Constructs a BosInputStream for the given object key. + * + * @param key the object key in BOS + * @param fileMetaData the file metadata + * @param store the native file system store + * @param statistics the file system statistics + */ + public BosInputStream( + String key, FileMetadata fileMetaData, + BosNativeFileSystemStore store, + FileSystem.Statistics statistics) { + this.key = key; + this.fileMetaData = fileMetaData; + this.contentLength = fileMetaData.getLength(); + this.store = store; + this.statistics = statistics; + } + + /** + * Reads a single byte from the stream. + * + * @return the byte read, or -1 if end of stream + * @throws IOException if an I/O error occurs + */ + public synchronized int read() throws IOException { + checkNotClosed(); + + if (this.contentLength == 0 + || (nextReadPos >= contentLength)) { + return -1; + } + + int byteRead; + try { + lazySeek(nextReadPos, 1); + byteRead = in.read(); + } catch (EOFException e) { + return -1; + } catch (IOException e) { + onReadFailure(e, 1); + byteRead = in.read(); + } + + if (byteRead >= 0) { + pos++; + nextReadPos++; + } + + if (statistics != null && byteRead >= 0) { + statistics.incrementBytesRead(1); + } + return byteRead; + } + + /** + * Reads bytes into a buffer from the stream. + * + * @param buf the buffer to read into + * @param off the start offset in the buffer + * @param len the maximum number of bytes to read + * @return the number of bytes read, or -1 if end of stream + * @throws IOException if an I/O error occurs + */ + public synchronized int read( + byte[] buf, int off, int len) throws IOException { + checkNotClosed(); + + if (len == 0) { + return 0; + } + + if (this.contentLength == 0 + || (nextReadPos >= contentLength)) { + return -1; + } + + // Validate and adjust parameters + if (off < 0 || len < 0 + || len > buf.length - off) { + throw new IndexOutOfBoundsException( + String.format( + "Invalid read parameters:" + + " buf.length=%d, off=%d, len=%d", + buf.length, off, len)); + } + + try { + lazySeek(nextReadPos, len); + } catch (EOFException e) { + // the end of the file has moved + return -1; + } + + int bytesRead; + try { + bytesRead = in.read(buf, off, len); + } catch (EOFException e) { + onReadFailure(e, len); + // the base implementation swallows EOFs. + return -1; + } catch (IOException e) { + onReadFailure(e, len); + bytesRead = in.read(buf, off, len); + } catch (Exception e) { + int intervalSeconds = 3; + int retry = 3; + while (true) { + try { + LOG.info( + "Read exception occur:" + + " retry at most {} times.", + retry); + // Reopen the stream at the current position + // to recover from the exception + onReadFailure(e, len); + // Read from the reopened stream + bytesRead = in.read(buf, off, len); + break; + } catch (EOFException eof) { + return -1; + } catch (IOException ioe) { + // For IOException, try to reopen once more + // and retry + LOG.info( + "IOException during retry," + + " attempting to reopen stream"); + onReadFailure(ioe, len); + bytesRead = in.read(buf, off, len); + break; + } catch (Exception ex) { + if (retry <= 0) { + String errorMsg = + ex.getMessage() != null + ? ex.getMessage() + : ex.getClass().getSimpleName(); + throw new IOException( + "Retry " + retry + + " times to read still" + + " exception: " + errorMsg, + ex); + } + + try { + TimeUnit.SECONDS.sleep( + intervalSeconds); + } catch (InterruptedException ite) { + Thread.currentThread().interrupt(); + throw new IOException( + "Thread interrupted during retry", ite); + } + intervalSeconds *= 2; + retry--; + // Update e to ex for the next iteration + e = ex; + } + } + } + + if (bytesRead > 0) { + pos += bytesRead; + nextReadPos += bytesRead; + } + + if (statistics != null && bytesRead >= 0) { + statistics.incrementBytesRead(bytesRead); + } + + return bytesRead; + } + + /** + * Closes this input stream and releases resources. + * + * @throws IOException if an I/O error occurs + */ + public synchronized void close() throws IOException { + if (!closed) { + closed = true; + + // close or abort the stream + closeStream("close() operation"); + // this is actually a no-op + super.close(); + } + } + + /** + * Seeks to the specified position in the stream. This + * performs a lazy seek; the actual stream repositioning + * happens on the next read. + * + * @param targetPos the target position to seek to + * @throws IOException if an I/O error occurs + */ + public synchronized void seek(long targetPos) + throws IOException { + checkNotClosed(); + + // Do not allow negative seek + if (targetPos < 0 || targetPos > contentLength) { + throw new EOFException( + FSExceptionMessages.NEGATIVE_SEEK + + " " + targetPos); + } + Review Comment: `seek()` rejects both negative offsets and seeks past EOF, but it always throws an `EOFException` using `FSExceptionMessages.NEGATIVE_SEEK`. This produces an incorrect error message when `targetPos > contentLength` (and also conflates invalid-argument vs EOF). Split the checks and use `FSExceptionMessages.CANNOT_SEEK_PAST_EOF` (or an appropriate message) when seeking beyond the end of the file. ```suggestion if (targetPos < 0) { throw new EOFException( FSExceptionMessages.NEGATIVE_SEEK + " " + targetPos); } if (targetPos > contentLength) { throw new EOFException( FSExceptionMessages.CANNOT_SEEK_PAST_EOF + " " + targetPos); } ``` ########## hadoop-cloud-storage-project/hadoop-bos/src/site/markdown/cloud-storage/index.md: ########## @@ -0,0 +1,392 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# Integration of Baidu BOS in Hadoop + +## Overview + +BOS (Baidu Object Storage) is the object storage service provided by Baidu Cloud. Hadoop-BOS is a connector between computing systems and underlying storage. For systems like Hadoop MR, Hive, Spark and Alluxio, hadoop-bos helps them use BOS as the underlying storage system instead of HDFS. + +## Quick Start + +In quick start, we will use hadoop shell command to access a BOS bucket. + +### Requirements + +1. A Baidu Cloud account. Use the account to create a BOS bucket. +2. A dev environment that can access BOS. E.g. a local server or a Baidu Cloud cloud server. +3. Install hadoop to the dev environment. Hadoop is installed at `$HADOOP_HOME`. + +### Usage + +* Compile hadoop-bos bundle tar. The hadoop-bos bundle is not packaged in hadoop final tar file. +So we have to compile it manually. Download the hadoop project, and build it with command below. +```bash +mvn package -DskipTests -pl org.apache.hadoop:hadoop-bos +``` + +* The bundle jar file is placed at +`$HADOOP_HOME/hadoop-cloud-storage-project/hadoop-bos/target/hadoop-bos-{VERSION}.jar`. + +* Copy the bundle jar to hdfs lib path. The hdfs lib path is +`$HADOOP_HOME/share/hadoop/hdfs/lib`. Remember copying to all hadoop nodes. +```bash +cp hadoop-bos-{VERSION}.jar $HADOOP_HOME/share/hadoop/hdfs/lib/ +``` Review Comment: This section states the hadoop-bos bundle is not packaged in the Hadoop final tar and must be built/copied manually. However, this PR adds hadoop-bos into the cloud-storage dist packaging (`hadoop-bos-package`), and the default dist dependencies already include the connector jar. Please update this documentation to reflect the actual packaging behavior (e.g., whether it is included by default, and what profile flags are needed). ########## hadoop-cloud-storage-project/hadoop-bos/src/test/resources/contract-test-options.xml.template: ########## @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<!-- + Baidu BOS auth-keys.xml Template + + IMPORTANT: + 1. Copy this file to auth-keys.xml in the same directory + 2. Fill in your BOS credentials and bucket information + 3. DO NOT commit auth-keys.xml to version control + + Usage: + cp contract-test-options.xml.template auth-keys.xml + # Edit auth-keys.xml with your settings + mvn test +--> Review Comment: This template file is named `contract-test-options.xml.template`, but its header/instructions say it is an `auth-keys.xml` template and suggests copying it to `auth-keys.xml`. This is confusing given the module also contains a separate `contract-test-options.xml` file and `core-site.xml` includes `auth-keys.xml`. Please align the filename and the instructions so it’s clear which file users should copy/edit for credentials vs contract options. ########## hadoop-cloud-storage-project/hadoop-bos/src/test/resources/README-TEST.md: ########## @@ -0,0 +1,324 @@ +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# BOS FileSystem Testing Guide + +This document provides guidance on running and understanding the BOS FileSystem test suite. + +## Table of Contents + +1. [Prerequisites](#prerequisites) +2. [Test Configuration](#test-configuration) +3. [Running Tests](#running-tests) +4. [Test Structure](#test-structure) +5. [Troubleshooting](#troubleshooting) + +## Prerequisites + +### Required + +- Java 8 or higher +- Maven 3.3 or higher +- Access to a Baidu BOS bucket for testing +- BOS credentials (Access Key ID and Secret Access Key) + +### BOS Bucket Setup + +1. Create a BOS bucket for testing (or use an existing one) +2. Ensure the bucket is accessible with your credentials +3. Note: Tests will create and delete files under `/test` directory in the bucket + +## Test Configuration + +### Step 1: Create Test Configuration File + +Copy the template configuration file: + +```bash +cd src/test/resources +cp contract-test-options.xml.template contract-test-options.xml +``` + +### Step 2: Configure BOS Credentials + +Edit `contract-test-options.xml` and set your BOS configuration: + +```xml +<property> + <name>fs.contract.test.fs.bos</name> + <value>bos://your-bucket-name/test</value> +</property> + +<property> + <name>fs.bos.endpoint</name> + <value>http://bd.bcebos.com</value> +</property> +``` + +### Step 3: Set Authentication + +**Recommended: Use Environment Variables** + +```bash +export BOS_ACCESS_KEY_ID=your_access_key_id +export BOS_SECRET_ACCESS_KEY=your_secret_access_key +``` Review Comment: The environment variable names documented here (BOS_ACCESS_KEY_ID / BOS_SECRET_ACCESS_KEY) do not match what `EnvironmentVariableCredentialsProvider` actually reads (`FS_BOS_ACCESS_KEY` / `FS_BOS_SECRET_ACCESS_KEY` / `FS_BOS_SESSION_TOKEN_KEY` per `BaiduBosConstants`). Update the README to use the correct env var names (or update the implementation/constants to match the documented names) so users can run the tests successfully. ########## hadoop-cloud-storage-project/hadoop-bos/src/test/resources/log4j.properties: ########## @@ -0,0 +1,346 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hadoop.root.logger=INFO,console +hadoop.log.dir=. +hadoop.log.file=hadoop.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hadoop.root.logger}, EventCounter + +# Logging Threshold +log4j.threshold=ALL + +# Null Appender +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# Rolling File Appender - cap space usage at 5gb. +# +hadoop.log.maxfilesize=256MB +hadoop.log.maxbackupindex=20 +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} + +log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# Daily Rolling File Appender +# + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Rollover at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n + +# +# TaskLog Appender +# +log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender + +log4j.appender.TLA.layout=org.apache.log4j.PatternLayout +log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# +# HDFS block state change log from block manager +# +# Uncomment the following to log normal block state change +# messages from BlockManager in NameNode. +#log4j.logger.BlockStateChange=DEBUG + +# +#Security appender +# +hadoop.security.logger=INFO,NullAppender +hadoop.security.log.maxfilesize=256MB +hadoop.security.log.maxbackupindex=20 +log4j.category.SecurityLogger=${hadoop.security.logger} +hadoop.security.log.file=SecurityAuth-${user.name}.audit +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} + +# +# Daily Rolling Security appender +# +log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} +log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd + +# +# hadoop configuration logging +# + +# Uncomment the following line to turn off configuration deprecation warnings. +# log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN + +# +# hdfs audit logging +# +hdfs.audit.logger=INFO,NullAppender +hdfs.audit.log.maxfilesize=256MB +hdfs.audit.log.maxbackupindex=20 +log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} +log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false +log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender +log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log +log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} +log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} + +# +# NameNode metrics logging. +# The default is to retain two namenode-metrics.log files up to 64MB each. +# +namenode.metrics.logger=INFO,NullAppender +log4j.logger.NameNodeMetricsLog=${namenode.metrics.logger} +log4j.additivity.NameNodeMetricsLog=false +log4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender +log4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log +log4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout +log4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n +log4j.appender.NNMETRICSRFA.MaxBackupIndex=1 +log4j.appender.NNMETRICSRFA.MaxFileSize=64MB + +# +# DataNode metrics logging. +# The default is to retain two datanode-metrics.log files up to 64MB each. +# +datanode.metrics.logger=INFO,NullAppender +log4j.logger.DataNodeMetricsLog=${datanode.metrics.logger} +log4j.additivity.DataNodeMetricsLog=false +log4j.appender.DNMETRICSRFA=org.apache.log4j.RollingFileAppender +log4j.appender.DNMETRICSRFA.File=${hadoop.log.dir}/datanode-metrics.log +log4j.appender.DNMETRICSRFA.layout=org.apache.log4j.PatternLayout +log4j.appender.DNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n +log4j.appender.DNMETRICSRFA.MaxBackupIndex=1 +log4j.appender.DNMETRICSRFA.MaxFileSize=64MB + +# Custom Logging levels + +#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG +#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG +#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG + + +# AWS SDK & S3A FileSystem +#log4j.logger.com.amazonaws=ERROR +log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR +#log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN + +# +# Job Summary Appender +# +# Use following logger to send summary to separate file defined by +# hadoop.mapreduce.jobsummary.log.file : +# hadoop.mapreduce.jobsummary.logger=INFO,JSA +# +hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} +hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log +hadoop.mapreduce.jobsummary.log.maxfilesize=256MB +hadoop.mapreduce.jobsummary.log.maxbackupindex=20 +log4j.appender.JSA=org.apache.log4j.RollingFileAppender +log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} +log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} +log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} +log4j.appender.JSA.layout=org.apache.log4j.PatternLayout +log4j.appender.JSA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} +log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false + +# +# shuffle connection log from shuffleHandler +# Uncomment the following line to enable logging of shuffle connections +# log4j.logger.org.apache.hadoop.mapred.ShuffleHandler.audit=DEBUG + +# +# Yarn ResourceManager Application Summary Log +# +# Set the ResourceManager summary log filename +yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log +# Set the ResourceManager summary log level and appender +yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} +#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY + +# To enable AppSummaryLogging for the RM, +# set yarn.server.resourcemanager.appsummary.logger to +# <LEVEL>,RMSUMMARY in hadoop-env.sh + +# Appender for ResourceManager Application Summary Log +# Requires the following properties to be set +# - hadoop.log.dir (Hadoop Log directory) +# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) +# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) + +log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} +log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false +log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender +log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} +log4j.appender.RMSUMMARY.MaxFileSize=256MB +log4j.appender.RMSUMMARY.MaxBackupIndex=20 +log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout +log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n + +# +# YARN ResourceManager audit logging +# +rm.audit.logger=INFO,NullAppender +rm.audit.log.maxfilesize=256MB +rm.audit.log.maxbackupindex=20 +log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=${rm.audit.logger} +log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=false +log4j.appender.RMAUDIT=org.apache.log4j.RollingFileAppender +log4j.appender.RMAUDIT.File=${hadoop.log.dir}/rm-audit.log +log4j.appender.RMAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.RMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.RMAUDIT.MaxFileSize=${rm.audit.log.maxfilesize} +log4j.appender.RMAUDIT.MaxBackupIndex=${rm.audit.log.maxbackupindex} + +# +# YARN NodeManager audit logging +# +nm.audit.logger=INFO,NullAppender +nm.audit.log.maxfilesize=256MB +nm.audit.log.maxbackupindex=20 +log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=${nm.audit.logger} +log4j.additivity.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=false +log4j.appender.NMAUDIT=org.apache.log4j.RollingFileAppender +log4j.appender.NMAUDIT.File=${hadoop.log.dir}/nm-audit.log +log4j.appender.NMAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.NMAUDIT.layout.ConversionPattern=%d{ISO8601}%p %c{2}: %m%n +log4j.appender.NMAUDIT.MaxFileSize=${nm.audit.log.maxfilesize} +log4j.appender.NMAUDIT.MaxBackupIndex=${nm.audit.log.maxbackupindex} + +# HS audit log configs +#mapreduce.hs.audit.logger=INFO,HSAUDIT +#log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} +#log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false +#log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log +#log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout +#log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +#log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd + +# Http Server Request Logs +#log4j.logger.http.requests.namenode=INFO,namenoderequestlog +#log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log +#log4j.appender.namenoderequestlog.RetainDays=3 + +#log4j.logger.http.requests.datanode=INFO,datanoderequestlog +#log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log +#log4j.appender.datanoderequestlog.RetainDays=3 + +#log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog +#log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log +#log4j.appender.resourcemanagerrequestlog.RetainDays=3 + +#log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog +#log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log +#log4j.appender.jobhistoryrequestlog.RetainDays=3 + +#log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog +#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender +#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log +#log4j.appender.nodemanagerrequestlog.RetainDays=3 + + +# WebHdfs request log on datanodes +# Specify -Ddatanode.webhdfs.logger=INFO,HTTPDRFA on datanode startup to +# direct the log to a separate file. +#datanode.webhdfs.logger=INFO,console +#log4j.logger.datanode.webhdfs=${datanode.webhdfs.logger} +#log4j.appender.HTTPDRFA=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.HTTPDRFA.File=${hadoop.log.dir}/hadoop-datanode-webhdfs.log +#log4j.appender.HTTPDRFA.layout=org.apache.log4j.PatternLayout +#log4j.appender.HTTPDRFA.layout.ConversionPattern=%d{ISO8601} %m%n +#log4j.appender.HTTPDRFA.DatePattern=.yyyy-MM-dd + + +# Appender for viewing information for errors and warnings +yarn.ewma.cleanupInterval=300 +yarn.ewma.messageAgeLimitSeconds=86400 +yarn.ewma.maxUniqueMessages=250 +log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender +log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval} +log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds} +log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages} + +# +# Fair scheduler state dump +# +# Use following logger to dump the state to a separate file + +#log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSSTATEDUMP +#log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=false +#log4j.appender.FSSTATEDUMP=org.apache.log4j.RollingFileAppender +#log4j.appender.FSSTATEDUMP.File=${hadoop.log.dir}/fairscheduler-statedump.log +#log4j.appender.FSSTATEDUMP.layout=org.apache.log4j.PatternLayout +#log4j.appender.FSSTATEDUMP.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +#log4j.appender.FSSTATEDUMP.MaxFileSize=${hadoop.log.maxfilesize} +#log4j.appender.FSSTATEDUMP.MaxBackupIndex=${hadoop.log.maxbackupindex} + +# Log levels of third-party libraries +log4j.logger.org.apache.commons.beanutils=WARN + +# log4j.logger.org.apache.hadoop.fs.bos=DEBUG +#log4j.logger.com.baidubce.request=DEBUG + + +#log4j.logger.com.baidubce.http=DEBUG, console + +## All HTTP requests +log4j.logger.bfs.com.baidubce.request=WARN + +log4j.logger.org.apache.hadoop.fs.bos=INFO Review Comment: This logger category (`bfs.com.baidubce.request`) doesn’t match the package relocation used by the shade configuration (which relocates to `org.apache.hadoop.fs.bos.shaded...`) and also won’t match unshaded test classpaths. Use the correct logger name(s) for the actual BOS SDK package present at runtime to ensure request logging can be controlled as intended. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
