(hadoop) branch feature-HADOOP-19363-analytics-accelerator-s3 updated: AAL ITest updates

[ahmar]

This is an automated email from the ASF dual-hosted git repository.

ahmar pushed a commit to branch feature-HADOOP-19363-analytics-accelerator-s3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to 
refs/heads/feature-HADOOP-19363-analytics-accelerator-s3 by this push:
     new bae4ed47bd5 AAL ITest updates
bae4ed47bd5 is described below

commit bae4ed47bd5a3ad6a27667fe0833fba58454756b
Author: Ahmar Suhail <ahma...@amazon.co.uk>
AuthorDate: Tue Feb 25 16:40:09 2025 +0000

    AAL ITest updates
---
 .../ITestS3AAnalyticsAcceleratorStreamReading.java | 31 +++++++++++++++-------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
index 1aa8cb1f23a..0afde9352ee 100644
--- 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
+++ 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
@@ -40,7 +40,10 @@
 import software.amazon.s3.analyticsaccelerator.common.ConnectorConfiguration;
 import software.amazon.s3.analyticsaccelerator.util.PrefetchMode;
 
-import static org.apache.hadoop.fs.s3a.Constants.*;
+import static 
org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static 
org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_PARQUET;
+import static 
org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static 
org.apache.hadoop.fs.s3a.Constants.ANALYTICS_ACCELERATOR_CONFIGURATION_PREFIX;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.enableAnalyticsAccelerator;
 import static 
org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static 
org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
@@ -80,19 +83,18 @@ public Configuration createConfiguration() {
   }
 
   @Test
-  public void testConnectorFrameWorkIntegration() throws IOException {
+  public void testConnectorFrameWorkIntegration() throws Throwable {
     describe("Verify S3 connector framework integration");
 
-    Configuration conf = new Configuration(getConfiguration());
-    removeBaseAndBucketOverrides(conf, INPUT_FADVISE);
-    conf.set(INPUT_FADVISE, "whole-file");
-
     S3AFileSystem fs =
-        (S3AFileSystem) FileSystem.get(externalTestFile.toUri(), conf);
+        (S3AFileSystem) FileSystem.get(externalTestFile.toUri(), 
getConfiguration());
     byte[] buffer = new byte[500];
     IOStatistics ioStats;
 
-    try (FSDataInputStream inputStream = fs.open(externalTestFile)) {
+    try (FSDataInputStream inputStream =
+        fs.openFile(externalTestFile)
+            .must(FS_OPTION_OPENFILE_READ_POLICY, 
FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+            .build().get()) {
       ioStats = inputStream.getIOStatistics();
       inputStream.seek(5);
       inputStream.read(buffer, 0, 500);
@@ -142,7 +144,7 @@ public void testMalformedParquetFooter() throws IOException 
{
     * @throws IOException any IO problem
    */
  @Test
- public void testMultiRowGroupParquet() throws IOException {
+ public void testMultiRowGroupParquet() throws Throwable {
     describe("A parquet file is read successfully");
 
     Path dest = path("multi_row_group.parquet");
@@ -162,6 +164,15 @@ public void testMultiRowGroupParquet() throws IOException {
    }
 
    verifyStatisticCounterValue(ioStats, STREAM_READ_ANALYTICS_OPENED, 1);
+
+   try (FSDataInputStream inputStream = getFileSystem().openFile(dest)
+       
.must(FS_OPTION_OPENFILE_READ_POLICY,FS_OPTION_OPENFILE_READ_POLICY_PARQUET)
+       .build().get()) {
+     ioStats = inputStream.getIOStatistics();
+     inputStream.readFully(buffer, 0, (int) fileStatus.getLen());
+   }
+
+   verifyStatisticCounterValue(ioStats, STREAM_READ_ANALYTICS_OPENED, 1);
  }
 
   @Test
@@ -197,7 +208,7 @@ public void testConnectorFrameworkConfigurable() {
   public void testInvalidConfigurationThrows() throws Exception {
     describe("Verify S3 connector framework throws with invalid 
configuration");
 
-    Configuration conf = getConfiguration();
+    Configuration conf = new Configuration(getConfiguration()) ;
     removeBaseAndBucketOverrides(conf);
     //Disable Sequential Prefetching
     conf.setInt(ANALYTICS_ACCELERATOR_CONFIGURATION_PREFIX +


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org