[
https://issues.apache.org/jira/browse/HADOOP-19102?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17825185#comment-17825185
]
ASF GitHub Bot commented on HADOOP-19102:
-----------------------------------------
saxenapranav commented on code in PR #6617:
URL: https://github.com/apache/hadoop/pull/6617#discussion_r1519263553
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamReadFooter.java:
##########
@@ -443,27 +575,35 @@ private FutureDataInputStreamBuilder
getParameterizedBuilder(final Path path,
return builder;
}
- private AzureBlobFileSystem getFileSystem(final boolean optimizeFooterRead,
- final int fileSize) throws IOException {
- final AzureBlobFileSystem fs = getFileSystem();
- AzureBlobFileSystemStore store = getAbfsStore(fs);
- store.getAbfsConfiguration().setOptimizeFooterRead(optimizeFooterRead);
- if (fileSize <= store.getAbfsConfiguration().getReadBufferSize()) {
- store.getAbfsConfiguration().setReadSmallFilesCompletely(false);
+ private void changeFooterConfigs(final AzureBlobFileSystem spiedFs,
+ final boolean optimizeFooterRead, final int fileSize,
+ final int readBufferSize) throws IOException {
+ AbfsConfiguration configuration =
spiedFs.getAbfsStore().getAbfsConfiguration();
+
Mockito.doReturn(optimizeFooterRead).when(configuration).optimizeFooterRead();
+ if (fileSize <= readBufferSize) {
+ Mockito.doReturn(false).when(configuration).readSmallFilesCompletely();
}
- return fs;
}
- private AzureBlobFileSystem getFileSystem(final boolean optimizeFooterRead,
- final int fileSize, final int footerReadBufferSize) throws IOException {
- final AzureBlobFileSystem fs = getFileSystem();
- AzureBlobFileSystemStore store = getAbfsStore(fs);
- store.getAbfsConfiguration().setOptimizeFooterRead(optimizeFooterRead);
- store.getAbfsConfiguration().setFooterReadBufferSize(footerReadBufferSize);
- if (fileSize <= store.getAbfsConfiguration().getReadBufferSize()) {
- store.getAbfsConfiguration().setReadSmallFilesCompletely(false);
+ private AzureBlobFileSystem createSpiedFs(Configuration configuration)
throws IOException {
+ AzureBlobFileSystem spiedFs = Mockito.spy((AzureBlobFileSystem)
FileSystem.newInstance(configuration));
+ AzureBlobFileSystemStore store = Mockito.spy(spiedFs.getAbfsStore());
+ Mockito.doReturn(store).when(spiedFs).getAbfsStore();
+ AbfsConfiguration spiedConfig = Mockito.spy(store.getAbfsConfiguration());
+ Mockito.doReturn(spiedConfig).when(store).getAbfsConfiguration();
+ return spiedFs;
+ }
+
+ private void changeFooterConfigs(final AzureBlobFileSystem spiedFs,
+ final boolean optimizeFooterRead, final int fileSize,
+ final int footerReadBufferSize, final int readBufferSize) throws
IOException {
Review Comment:
Fixed it.
> [ABFS]: FooterReadBufferSize should not be greater than readBufferSize
> ----------------------------------------------------------------------
>
> Key: HADOOP-19102
> URL: https://issues.apache.org/jira/browse/HADOOP-19102
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure
> Reporter: Pranav Saxena
> Assignee: Pranav Saxena
> Priority: Major
> Labels: pull-request-available
> Fix For: 3.4.0, 3.5.0
>
>
> The method `optimisedRead` creates a buffer array of size `readBufferSize`.
> If footerReadBufferSize is greater than readBufferSize, abfs will attempt to
> read more data than the buffer array can hold, which causes an exception.
> Change: To avoid this, we will keep footerBufferSize =
> min(readBufferSizeConfig, footerBufferSizeConfig)
>
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
