[
https://issues.apache.org/jira/browse/HADOOP-17272?focusedWorklogId=493054&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-493054
]
ASF GitHub Bot logged work on HADOOP-17272:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 30/Sep/20 16:28
Start Date: 30/Sep/20 16:28
Worklog Time Spent: 10m
Work Description: steveloughran commented on a change in pull request
#2353:
URL: https://github.com/apache/hadoop/pull/2353#discussion_r497636885
##########
File path:
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsOutputStreamStatistics.java
##########
@@ -219,6 +226,31 @@ public void testAbfsOutputStreamWriteBuffer() throws
IOException {
}
}
+ /**
+ * Test to check correct value of time spent on a PUT request in
+ * AbfsOutputStream.
+ */
+ @Test
+ public void testAbfsOutputStreamDurationTrackerPutRequest() throws
IOException {
+ describe("Testing to check if DurationTracker for PUT request is working "
+ + "correctly.");
+ AzureBlobFileSystem fs = getFileSystem();
+ Path pathForPutRequest = path(getMethodName());
+
+ try(AbfsOutputStream outputStream =
+ createAbfsOutputStreamWithFlushEnabled(fs, pathForPutRequest)) {
+ outputStream.write('a');
+ outputStream.hflush();
+
+ AbfsOutputStreamStatisticsImpl abfsOutputStreamStatistics =
+ getAbfsOutputStreamStatistics(outputStream);
+
+
Assertions.assertThat(abfsOutputStreamStatistics.getTimeSpentOnPutRequest())
+ .describedAs("Mismatch in timeTakenForPutRequest DurationTracker")
Review comment:
timeSpentOnPutRequest.
##########
File path:
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStreamStatisticsImpl.java
##########
@@ -125,31 +120,63 @@ public void queueShrunk() {
*/
@Override
public void writeCurrentBuffer() {
- writeCurrentBufferOperations++;
+
ioStatisticsStore.incrementCounter(getStatName(WRITE_CURRENT_BUFFER_OPERATIONS));
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * A getter for IOStatisticsStore instance which extends IOStatistics.
+ *
+ * @return IOStatisticsStore instance.
+ */
+ @Override
+ public IOStatistics getIOStatistics() {
+ return ioStatisticsStore;
}
+ @VisibleForTesting
public long getBytesToUpload() {
- return bytesToUpload;
+ return
ioStatisticsStore.getCounterReference(getStatName(BYTES_TO_UPLOAD)).get();
Review comment:
counters.get(stat-name) will do this too
##########
File path:
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
##########
@@ -557,6 +567,11 @@ int getMaxRequestsThatCanBeQueued() {
return maxRequestsThatCanBeQueued;
}
+ @VisibleForTesting
+ public IOStatistics getIoStatistics() {
Review comment:
declare implementation of IOStatisticsSource, implement accessor there
##########
File path:
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStreamStatisticsImpl.java
##########
@@ -125,31 +120,63 @@ public void queueShrunk() {
*/
@Override
public void writeCurrentBuffer() {
- writeCurrentBufferOperations++;
+
ioStatisticsStore.incrementCounter(getStatName(WRITE_CURRENT_BUFFER_OPERATIONS));
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * A getter for IOStatisticsStore instance which extends IOStatistics.
+ *
+ * @return IOStatisticsStore instance.
+ */
+ @Override
+ public IOStatistics getIOStatistics() {
+ return ioStatisticsStore;
}
+ @VisibleForTesting
public long getBytesToUpload() {
- return bytesToUpload;
+ return
ioStatisticsStore.getCounterReference(getStatName(BYTES_TO_UPLOAD)).get();
}
+ @VisibleForTesting
public long getBytesUploadSuccessful() {
- return bytesUploadSuccessful;
+ return
ioStatisticsStore.getCounterReference(getStatName(BYTES_UPLOAD_SUCCESSFUL)).get();
}
+ @VisibleForTesting
public long getBytesUploadFailed() {
- return bytesUploadFailed;
+ return
ioStatisticsStore.getCounterReference(getStatName(BYTES_UPLOAD_FAILED)).get();
}
+ @VisibleForTesting
public long getTimeSpentOnTaskWait() {
- return timeSpentOnTaskWait;
+ return
ioStatisticsStore.getCounterReference(getStatName(TIME_SPENT_ON_TASK_WAIT)).get();
}
+ @VisibleForTesting
public long getQueueShrunkOps() {
- return queueShrunkOps;
+ return
ioStatisticsStore.getCounterReference(getStatName(QUEUE_SHRUNK_OPS)).get();
}
+ @VisibleForTesting
public long getWriteCurrentBufferOperations() {
- return writeCurrentBufferOperations;
+ return
ioStatisticsStore.getCounterReference(getStatName(WRITE_CURRENT_BUFFER_OPERATIONS)).get();
+ }
+
+ @VisibleForTesting
+ public double getTimeSpentOnPutRequest() {
+ return
ioStatisticsStore.getMeanStatistic(getStatName(TIME_TAKEN_TO_PUT_REQUEST) +
".mean").mean();
Review comment:
StoreStatisticsNames.SUFFIX_MEAN has that string value there
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 493054)
Time Spent: 0.5h (was: 20m)
> ABFS Streams to support IOStatistics API
> -----------------------------------------
>
> Key: HADOOP-17272
> URL: https://issues.apache.org/jira/browse/HADOOP-17272
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure
> Affects Versions: 3.3.1
> Reporter: Steve Loughran
> Assignee: Mehakmeet Singh
> Priority: Major
> Labels: pull-request-available
> Time Spent: 0.5h
> Remaining Estimate: 0h
>
> ABFS input/output streams to support IOStatistics API
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
