bhattmanish98 commented on code in PR #8611: URL: https://github.com/apache/hadoop/pull/8611#discussion_r3850266562
########## hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsPhotonListStatus.java: ########## @@ -0,0 +1,249 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; + +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_FALLBACK_COUNT; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_LISTING_LATENCY; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_PARSE_FAILURE_COUNT; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_REQUEST_COUNT; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.PHOTON_RESPONSE_COUNT; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_LIST_MAX_RESULTS; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_PHOTON; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Live integration tests for the Photon (Apache Arrow based ListBlob) listing + * path against a real Blob-endpoint account. These cover the integration + * scenarios from the design document that are not verifiable with unit tests: + * <ul> + * <li>XML-vs-Photon {@link FileStatus} parity (identical results regardless of + * the wire format).</li> + * <li>Graceful XML fallback when Arrow is requested but the account returns + * XML (asserted via the Photon telemetry classification).</li> + * <li>Pagination across multiple Photon responses.</li> + * <li>Photon telemetry counters and listing-latency tracker are emitted.</li> + * </ul> + * + * <p>Photon is only offered on the Blob endpoint, so every test asserts a Blob + * service type before running.</p> + */ +public class ITestAbfsPhotonListStatus extends AbstractAbfsIntegrationTest { + + private static final Logger LOG = + LoggerFactory.getLogger(ITestAbfsPhotonListStatus.class); + + private static final String[] CHILD_FILES = { + "file-a.txt", + "file-b.txt", + "name with space.txt", + "\u6587\u4ef6-unicode.txt", + }; + + private static final String CHILD_DIR = "subdir"; + + public ITestAbfsPhotonListStatus() throws Exception { + } + + /** + * Create a filesystem instance with Photon explicitly toggled and, optionally, + * a reduced ListBlobs page size to force multi-page pagination. + */ + private AzureBlobFileSystem newFileSystem(final boolean photonEnabled, + final int listMaxResults) throws IOException { + Configuration conf = new Configuration(getRawConfiguration()); + conf.setBoolean(FS_AZURE_ENABLE_PHOTON, photonEnabled); + if (listMaxResults > 0) { + conf.setInt(AZURE_LIST_MAX_RESULTS, listMaxResults); + } + return (AzureBlobFileSystem) FileSystem.newInstance(conf); + } + + /** + * Populate a directory with a mix of files (including special-character and + * Unicode names) and a subdirectory, so a listing exercises files and + * directories together. + */ + private void createTree(final AzureBlobFileSystem fs, final Path baseDir) + throws IOException { + fs.mkdirs(baseDir); + for (int i = 0; i < CHILD_FILES.length; i++) { + try (FSDataOutputStream out = fs.create(new Path(baseDir, CHILD_FILES[i]))) { + // Give the files distinct, non-zero sizes to make parity meaningful. + out.write(new byte[i + 1]); + } + } + fs.mkdirs(new Path(baseDir, CHILD_DIR)); + } + + private static List<FileStatus> sortedByName(final FileStatus[] statuses) { + return Arrays.stream(statuses) + .sorted(Comparator.comparing(s -> s.getPath().getName())) + .collect(Collectors.toList()); + } + + /** + * Verify that listing the same directory over XML (Photon disabled) and over + * Arrow (Photon enabled) yields identical {@link FileStatus} results, covering + * the doc's XML/Photon parity and identical-FileStatus scenarios. + */ + @Test + public void testPhotonAndXmlListingParity() throws Exception { Review Comment: Added a modification-time parity assertion (isCloseTo(xml.getModificationTime(), within(1000L))) so the hand-written fastIsoUtcToRfc1123() / Sakamoto math has an end-to-end guard against XML, with a second of tolerance for precision differences. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
