Repository: kylin Updated Branches: refs/heads/master 6f6dcf00c -> 4018e6671
KYLIN-1758 should set hive conf when creating materialized tables Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4018e667 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4018e667 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4018e667 Branch: refs/heads/master Commit: 4018e6671f6f4e23f4493286881fa47f302c6372 Parents: 6f6dcf0 Author: shaofengshi <shaofeng...@apache.org> Authored: Sat Jun 4 20:58:30 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Sat Jun 4 20:58:30 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/job/JoinedFlatTable.java | 25 +++++++++----------- .../kylin/provision/BuildIIWithStream.java | 7 +----- .../apache/kylin/source/hive/HiveMRInput.java | 20 ++++------------ 3 files changed, 16 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4018e667/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index d625ad7..24032fa 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -55,15 +55,16 @@ public class JoinedFlatTable { return storageDfsDir + "/" + intermediateTableDesc.getTableName(); } - public static String generateHiveSetStatements(JobEngineConfig engineConfig) throws IOException { + public static String generateHiveSetStatements(JobEngineConfig engineConfig) { StringBuilder buffer = new StringBuilder(); - File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath()); - if (hadoopPropertiesFile.exists()) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder; - Document doc; - try { + try { + File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath()); + + if (hadoopPropertiesFile.exists()) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder; + Document doc; builder = factory.newDocumentBuilder(); doc = builder.parse(hadoopPropertiesFile); NodeList nl = doc.getElementsByTagName("property"); @@ -74,14 +75,10 @@ public class JoinedFlatTable { buffer.append("SET " + name + "=" + value + ";\n"); } } - - } catch (ParserConfigurationException e) { - throw new IOException(e); - } catch (SAXException e) { - throw new IOException(e); } + } catch (Exception e) { + throw new RuntimeException("Failed to parse hive conf file ", e); } - return buffer.toString(); } @@ -114,7 +111,7 @@ public class JoinedFlatTable { return ddl.toString(); } - public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, JobEngineConfig engineConfig) throws IOException { + public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, JobEngineConfig engineConfig) { StringBuilder sql = new StringBuilder(); sql.append(generateHiveSetStatements(engineConfig)); sql.append("INSERT OVERWRITE TABLE " + intermediateTableDesc.getTableName() + " " + generateSelectDataStatement(intermediateTableDesc) + ";").append("\n"); http://git-wip-us.apache.org/repos/asf/kylin/blob/4018e667/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java index c0f6f63..1a335c6 100644 --- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java +++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java @@ -148,12 +148,7 @@ public class BuildIIWithStream { final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc); final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, JobBuilderSupport.getJobWorkingDir(jobEngineConfig, uuid)); String insertDataHqls; - try { - insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobEngineConfig); - } catch (IOException e1) { - e1.printStackTrace(); - throw new RuntimeException("Failed to generate insert data SQL for intermediate table."); - } + insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobEngineConfig); ShellExecutable step = new ShellExecutable(); HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); http://git-wip-us.apache.org/repos/asf/kylin/blob/4018e667/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java index 0c33c11..76dda34 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java @@ -120,7 +120,6 @@ public class HiveMRInput implements IMRInput { } jobFlow.addTask(createCountHiveTableStep(conf, flatHiveTableDesc, jobFlow.getId(), rowCountOutputDir)); jobFlow.addTask(createFlatHiveTableStep(conf, flatHiveTableDesc, jobFlow.getId(), cubeName, rowCountOutputDir)); -// jobFlow.addTask(createFlatHiveTableStep(conf, flatHiveTableDesc, jobFlow.getId())); AbstractExecutable task = createLookupHiveViewMaterializationStep(jobFlow.getId()); if(task != null) { jobFlow.addTask(task); @@ -131,11 +130,7 @@ public class HiveMRInput implements IMRInput { final ShellExecutable step = new ShellExecutable(); final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); - try { - hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf)); - } catch (IOException e) { - throw new RuntimeException("Failed to generate hive set statements for createCountHiveTableStep", e); - } + hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf)); hiveCmdBuilder.addStatement("set hive.exec.compress.output=false;\n"); hiveCmdBuilder.addStatement(JoinedFlatTable.generateCountDataStatement(flatTableDesc, rowCountOutputDir)); @@ -168,6 +163,7 @@ public class HiveMRInput implements IMRInput { } final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";"; hiveCmdBuilder.addStatement(useDatabaseHql); + hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf)); for(TableDesc lookUpTableDesc : lookupViewsTables) { if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(lookUpTableDesc.getTableType())) { StringBuilder createIntermediateTableHql = new StringBuilder(); @@ -190,21 +186,13 @@ public class HiveMRInput implements IMRInput { public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName, String rowCountOutputDir) { StringBuilder hiveInitBuf = new StringBuilder(); - try { - hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf)); - } catch (IOException e) { - throw new RuntimeException("Failed to generate hive set statements for RedistributeFlatHiveTableStep", e); - } + hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf)); final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";"; final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatTableDesc); final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatTableDesc, JobBuilderSupport.getJobWorkingDir(conf, jobId)); String insertDataHqls; - try { - insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, conf); - } catch (IOException e) { - throw new RuntimeException("Failed to generate insert data SQL for intermediate table.", e); - } + insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, conf); CreateFlatHiveTableStep step = new CreateFlatHiveTableStep(); step.setInitStatement(hiveInitBuf.toString());
