Repository: kylin Updated Branches: refs/heads/KYLIN-2351 aab7178c4 -> 2ac1743fc
KYLIN-2351 more enforce Path schema Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ac1743f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ac1743f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ac1743f Branch: refs/heads/KYLIN-2351 Commit: 2ac1743fc2c42cd0346576e8dcd6674990fc6bc6 Parents: aab7178 Author: Li Yang <liy...@apache.org> Authored: Thu Jan 5 11:35:44 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Thu Jan 5 11:35:44 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java | 2 +- .../kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/2ac1743f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java index 88f6ba2..af86181 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java @@ -123,7 +123,7 @@ public class MergeStatisticsStep extends AbstractExecutable { averageSamplingPercentage = averageSamplingPercentage / CubingExecutableUtil.getMergingSegmentIds(this.getParams()).size(); CubeStatsWriter.writeCuboidStatistics(conf, new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams())), cuboidHLLMap, averageSamplingPercentage); Path statisticsFilePath = new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME); - FileSystem fs = statisticsFilePath.getFileSystem(conf); + FileSystem fs = HadoopUtil.getFileSystem(statisticsFilePath, conf); FSDataInputStream is = fs.open(statisticsFilePath); try { // put the statistics to metadata store http://git-wip-us.apache.org/repos/asf/kylin/blob/2ac1743f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java index fca91a6..ee8da6b 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java @@ -33,6 +33,7 @@ import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.kylin.engine.mr.HadoopUtil; import org.apache.kylin.engine.mr.KylinMapper; /** @@ -89,7 +90,7 @@ public class RowKeyDistributionCheckerMapper extends KylinMapper<Text, Text, Tex List<byte[]> rowkeyList = new ArrayList<byte[]>(); SequenceFile.Reader reader = null; try { - reader = new SequenceFile.Reader(path.getFileSystem(conf), path, conf); + reader = new SequenceFile.Reader(HadoopUtil.getFileSystem(path, conf), path, conf); Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) {
