This is an automated email from the ASF dual-hosted git repository. nic pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 981cc273be31bb3099c3de34c78f88f986b0db8d Author: Zhong, Yanghong <nju_y...@apache.org> AuthorDate: Tue Mar 10 16:11:34 2020 +0800 KYLIN-4414 add a property jobTempDir for AbstractHadoopJob to indicate the directory for all of the temporary files --- .../kylin/engine/mr/common/AbstractHadoopJob.java | 48 ++++++++++------------ 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index fd4d413..411caa7 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -157,6 +157,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { protected OptionsHelper optionsHelper = new OptionsHelper(); protected Job job; + private File jobTempDir; public AbstractHadoopJob() { super(HadoopUtil.getCurrentConfiguration()); @@ -595,10 +596,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { protected void dumpKylinPropsAndMetadata(String prj, Set<String> dumpList, KylinConfig kylinConfig, Configuration conf) throws IOException { - File tmp = File.createTempFile("kylin_job_meta", ""); - FileUtils.forceDelete(tmp); // we need a directory, so delete the file first - - File metaDir = new File(tmp, "meta"); + File metaDir = new File(getJobTempDir(), "meta"); metaDir.mkdirs(); // write kylin.properties @@ -628,33 +626,29 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { } protected void cleanupTempConfFile(Configuration conf) { - String[] tempfiles = StringUtils.split(conf.get("tmpfiles"), ","); - if (tempfiles == null) { - return; - } - for (String tempMetaFileString : tempfiles) { - logger.trace("tempMetaFileString is : " + tempMetaFileString); - if (tempMetaFileString != null) { - if (tempMetaFileString.startsWith("file://")) { - tempMetaFileString = tempMetaFileString.substring("file://".length()); - File tempMetaFile = new File(tempMetaFileString); - if (tempMetaFile.exists()) { - try { - FileUtils.forceDelete(tempMetaFile.getParentFile()); - - } catch (IOException e) { - logger.warn("error when deleting " + tempMetaFile, e); - } - } else { - logger.info("" + tempMetaFileString + " does not exist"); - } - } else { - logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString); - } + String tmpFilesString = conf.get("tmpfiles"); + logger.info("tmpFilesString is : " + tmpFilesString); + if (jobTempDir != null) { + try { + FileUtils.forceDelete(jobTempDir); + } catch (IOException e) { + logger.warn("error when deleting " + jobTempDir, e); } } } + // It's not thread safe + protected File getJobTempDir() throws IOException { + if (jobTempDir != null && jobTempDir.isDirectory()) { + return jobTempDir; + } + jobTempDir = File.createTempFile("kylin_job_meta", ""); + FileUtils.forceDelete(jobTempDir); // we need a directory, so delete the file first + + jobTempDir.mkdirs(); + return jobTempDir; + } + protected void deletePath(Configuration conf, Path path) throws IOException { HadoopUtil.deletePath(conf, path); }