KYLIN-2490 Couldn't get cube size on Azure HDInsight
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f1154c55 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f1154c55 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f1154c55 Branch: refs/heads/master-hbase0.98 Commit: f1154c55ebcc87280dc8d126e36587bc913dacc4 Parents: 67d30a0 Author: shaofengshi <shaofeng...@apache.org> Authored: Wed Mar 8 11:23:29 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Wed Mar 8 11:23:29 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/engine/mr/common/HadoopCmdOutput.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f1154c55/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java index 2a480e6..5da1947 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.TaskCounter; @@ -93,9 +94,16 @@ public class HadoopCmdOutput { } this.output.append(counters.toString()).append("\n"); logger.debug(counters.toString()); + + String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN"; + String fsScheme = FileSystem.get(job.getConfiguration()).getScheme(); + if (("wasb").equalsIgnoreCase(fsScheme)) { + // for Azure blob store + bytsWrittenCounterName = "WASB_BYTES_WRITTEN"; + } mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); - hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue()); + hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", bytsWrittenCounterName).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); } catch (Exception e) { logger.error(e.getLocalizedMessage(), e);
