KYLIN-2766 rework, fix bugs in origin commits
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3177d79c Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3177d79c Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3177d79c Branch: refs/heads/master Commit: 3177d79ca5cd8533164319acda8676684a6d307e Parents: b4ad6f0 Author: shaofengshi <shaofeng...@apache.org> Authored: Mon Sep 18 11:28:05 2017 +0800 Committer: liyang-gmt8 <liy...@apache.org> Committed: Mon Sep 18 16:17:01 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/engine/mr/common/HadoopCmdOutput.java | 4 ++-- .../org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java | 2 +- .../org/apache/kylin/storage/hbase/HBaseResourceStore.java | 6 +++--- .../org/apache/kylin/storage/hbase/steps/CreateHTableJob.java | 2 +- .../storage/hbase/steps/HDFSPathGarbageCollectionStep.java | 2 +- .../apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java | 5 ++--- 6 files changed, 10 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/3177d79c/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java index 0bff511..11d6d2c 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java @@ -22,11 +22,11 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.FileSystemCounter; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.TaskCounter; -import org.apache.kylin.common.util.HadoopUtil; import org.apache.kylin.engine.mr.steps.FactDistinctColumnsMapper.RawDataCounter; import org.apache.kylin.job.constant.ExecutableConstants; import org.slf4j.Logger; @@ -99,7 +99,7 @@ public class HadoopCmdOutput { mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); - String fsScheme = HadoopUtil.getWorkingFileSystem().getScheme(); + String fsScheme = FileSystem.get(job.getConfiguration()).getScheme(); long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue(); if (bytesWritten == 0) { bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); http://git-wip-us.apache.org/repos/asf/kylin/blob/3177d79c/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java index 28f99fb..5f49391 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java @@ -59,8 +59,8 @@ public class SaveStatisticsStep extends AbstractExecutable { ResourceStore rs = ResourceStore.getStore(kylinConf); try { - FileSystem fs = HadoopUtil.getWorkingFileSystem(); Path statisticsDir = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams())); + FileSystem fs = HadoopUtil.getFileSystem(statisticsDir); Path statisticsFilePath = HadoopUtil.getFilterOnlyPath(fs, statisticsDir, BatchConstants.CFG_OUTPUT_STATISTICS); if (statisticsFilePath == null) { throw new IOException("fail to find the statistics file in base dir: " + statisticsDir); http://git-wip-us.apache.org/repos/asf/kylin/blob/3177d79c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index bcdb06a..45e6195 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -248,7 +248,7 @@ public class HBaseResourceStore extends ResourceStore { byte[] value = r.getValue(B_FAMILY, B_COLUMN); if (value.length == 0) { Path redirectPath = bigCellHDFSPath(resPath); - FileSystem fileSystem = HadoopUtil.getWorkingFileSystem(HBaseConnection.getCurrentHBaseConfiguration()); + FileSystem fileSystem = HadoopUtil.getFileSystem(redirectPath, HBaseConnection.getCurrentHBaseConfiguration()); try { return fileSystem.open(redirectPath); @@ -341,7 +341,7 @@ public class HBaseResourceStore extends ResourceStore { if (hdfsResourceExist) { // remove hdfs cell value Path redirectPath = bigCellHDFSPath(resPath); - FileSystem fileSystem = HadoopUtil.getWorkingFileSystem(HBaseConnection.getCurrentHBaseConfiguration()); + FileSystem fileSystem = HadoopUtil.getFileSystem(redirectPath, HBaseConnection.getCurrentHBaseConfiguration()); if (fileSystem.exists(redirectPath)) { fileSystem.delete(redirectPath, true); @@ -389,7 +389,7 @@ public class HBaseResourceStore extends ResourceStore { private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException { Path redirectPath = bigCellHDFSPath(resPath); - FileSystem fileSystem = HadoopUtil.getWorkingFileSystem(HBaseConnection.getCurrentHBaseConfiguration()); + FileSystem fileSystem = HadoopUtil.getFileSystem(redirectPath, HBaseConnection.getCurrentHBaseConfiguration()); if (fileSystem.exists(redirectPath)) { fileSystem.delete(redirectPath, true); http://git-wip-us.apache.org/repos/asf/kylin/blob/3177d79c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java index e1f8bb1..8696a3b 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java @@ -275,7 +275,7 @@ public class CreateHTableJob extends AbstractHadoopJob { // note read-write separation, respect HBase FS here Configuration hbaseConf = HBaseConnection.getCurrentHBaseConfiguration(); - FileSystem fs = HadoopUtil.getWorkingFileSystem(hbaseConf); + FileSystem fs = HadoopUtil.getFileSystem(outputFolder, hbaseConf); if (fs.exists(outputFolder) == false) { fs.mkdirs(outputFolder); } http://git-wip-us.apache.org/repos/asf/kylin/blob/3177d79c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java index 981672c..86e8e6b 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java @@ -62,7 +62,7 @@ public class HDFSPathGarbageCollectionStep extends AbstractExecutable { dropHdfsPathOnCluster(toDeletePaths, HadoopUtil.getWorkingFileSystem()); if (StringUtils.isNotEmpty(context.getConfig().getHBaseClusterFs())) { - dropHdfsPathOnCluster(toDeletePaths, HadoopUtil.getWorkingFileSystem(HBaseConnection.getCurrentHBaseConfiguration())); + dropHdfsPathOnCluster(toDeletePaths, FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration())); } } catch (IOException e) { logger.error("job:" + getId() + " execute finished with exception", e); http://git-wip-us.apache.org/repos/asf/kylin/blob/3177d79c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java index 110a51a..c437e66 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinVersion; import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.HadoopUtil; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; @@ -87,7 +86,7 @@ public class DeployCoprocessorCLI { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); - FileSystem fileSystem = HadoopUtil.getWorkingFileSystem(hconf); + FileSystem fileSystem = FileSystem.get(hconf); Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); @@ -218,7 +217,7 @@ public class DeployCoprocessorCLI { private static void initHTableCoprocessor(HTableDescriptor desc) throws IOException { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); - FileSystem fileSystem = HadoopUtil.getWorkingFileSystem(hconf); + FileSystem fileSystem = FileSystem.get(hconf); String localCoprocessorJar = kylinConfig.getCoprocessorLocalJar(); Path hdfsCoprocessorJar = DeployCoprocessorCLI.uploadCoprocessorJar(localCoprocessorJar, fileSystem, null);
