mehakmeet commented on a change in pull request #1899: HADOOP-16914 Adding Output Stream Counters in ABFS URL: https://github.com/apache/hadoop/pull/1899#discussion_r405059806
########## File path: hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsOutputStream.java ########## @@ -133,58 +133,60 @@ public void testAbfsOutputStreamTimeSpendOnWaitTask() throws IOException { public void testAbfsOutputStreamQueueShrink() throws IOException { describe("Testing Queue Shrink calls in AbfsOutputStream"); final AzureBlobFileSystem fs = getFileSystem(); - Path TEST_PATH = new Path("AbfsOutputStreamStatsPath"); + Path queueShrinkFilePath = new Path("AbfsOutputStreamStatsPath"); AzureBlobFileSystemStore abfss = fs.getAbfsStore(); abfss.getAbfsConfiguration().setDisableOutputStreamFlush(false); FileSystem.Statistics statistics = fs.getFsStatistics(); String testQueueShrink = "testQueue"; - AbfsOutputStream outForOneOp = null; try { - outForOneOp = (AbfsOutputStream) abfss.createFile(TEST_PATH, statistics, - true, - FsPermission.getDefault(), FsPermission.getUMask(fs.getConf())); + outForOneOp = + (AbfsOutputStream) abfss.createFile(queueShrinkFilePath, statistics, + true, + FsPermission.getDefault(), FsPermission.getUMask(fs.getConf())); //Test for shrinking Queue zero time - Assert.assertEquals("Mismatch in number of queueShrink() Calls", 0, + assertValues("number of queueShrink() Calls", 0, outForOneOp.getOutputStreamStatistics().queueShrink); outForOneOp.write(testQueueShrink.getBytes()); // Queue is shrunk 2 times when outStream is flushed outForOneOp.flush(); //Test for shrinking Queue 2 times - Assert.assertEquals("Mismatch in number of queueShrink() Calls", 2, + assertValues("number of queueShrink() Calls", 2, outForOneOp.getOutputStreamStatistics().queueShrink); } finally { - if(outForOneOp != null){ + if (outForOneOp != null) { outForOneOp.close(); } } AbfsOutputStream outForLargeOps = null; try { - outForLargeOps = (AbfsOutputStream) abfss.createFile(TEST_PATH, + outForLargeOps = (AbfsOutputStream) abfss.createFile(queueShrinkFilePath, statistics, true, FsPermission.getDefault(), FsPermission.getUMask(fs.getConf())); + int largeValue = 1000; //QueueShrink is called 2 times in 1 flush(), hence 1000 flushes must // give 2000 QueueShrink calls - for (int i = 0; i < 1000; i++) { + for (int i = 0; i < largeValue; i++) { outForLargeOps.write(testQueueShrink.getBytes()); //Flush is quite expensive so 1000 calls only which takes 1 min+ outForLargeOps.flush(); Review comment: No I can't, basically calling flush after I write means the write task is done. This would trigger the shrinkWriteOperationQueue() method and we need to do it after each read to get 20 operations. If I flush after the loop, it would take all the write calls as 1 write operation and only 1 time the shrinkWriteOperationQueue() method is triggered. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org