Repository: kylin Updated Branches: refs/heads/1.x-staging 474dbfa67 -> 85d55f4f8
KYLIN-1402 StringIndexOutOfBoundsException in Kylin Hive Column Cardinality Job Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85d55f4f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85d55f4f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85d55f4f Branch: refs/heads/1.x-staging Commit: 85d55f4f8e4de440c7764cd1820917ddbef3836b Parents: 474dbfa Author: shaofengshi <[email protected]> Authored: Fri Feb 5 16:35:25 2016 +0800 Committer: shaofengshi <[email protected]> Committed: Fri Feb 5 16:35:25 2016 +0800 ---------------------------------------------------------------------- .../HiveColumnCardinalityUpdateJob.java | 44 +++++++++++--------- 1 file changed, 25 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/85d55f4f/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java index 115753e..cad3c0b 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java @@ -18,14 +18,6 @@ package org.apache.kylin.job.hadoop.cardinality; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; @@ -41,11 +33,21 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.job.hadoop.AbstractHadoopJob; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; /** * This job will update save the cardinality result into Kylin table metadata store. - * @author shaoshi * + * @author shaoshi */ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob { public static final String JOB_TITLE = "Kylin Hive Column Cardinality Update Job"; @@ -53,6 +55,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob { @SuppressWarnings("static-access") protected static final Option OPTION_TABLE = OptionBuilder.withArgName("table name").hasArg().isRequired(true).withDescription("The hive table name").create("table"); + private static final Logger logger = LoggerFactory.getLog(HiveColumnCardinalityUpdateJob.class); private String table; public HiveColumnCardinalityUpdateJob() { @@ -73,7 +76,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob { this.table = getOptionValue(OPTION_TABLE).toUpperCase(); // start job String jobName = JOB_TITLE + getOptionsAsString(); - System.out.println("Starting: " + jobName); + logger.info("Starting: " + jobName); Configuration conf = getConf(); Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); @@ -91,30 +94,33 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob { try { columns = readLines(new Path(outPath), config); } catch (Exception e) { - e.printStackTrace(); - System.out.println("Failed to resolve cardinality for " + tableName + " from " + outPath); + logger.error("Failed to resolve cardinality for " + tableName + " from " + outPath, e); return; } StringBuffer cardi = new StringBuffer(); Iterator<String> it = columns.iterator(); while (it.hasNext()) { - String string = (String) it.next(); + String string = it.next(); String[] ss = StringUtils.split(string, "\t"); if (ss.length != 2) { - System.out.println("The hadoop cardinality value is not valid " + string); + logger.info("The hadoop cardinality value is not valid " + string); continue; } cardi.append(ss[1]); cardi.append(","); } String scardi = cardi.toString(); - scardi = scardi.substring(0, scardi.length() - 1); - MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); - Map<String, String> tableExd = metaMgr.getTableDescExd(tableName); - tableExd.put(MetadataConstants.TABLE_EXD_CARDINALITY, scardi); - metaMgr.saveTableExd(tableName.toUpperCase(), tableExd); + if (scardi.length() > 0) { + scardi = scardi.substring(0, scardi.length() - 1); + MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + Map<String, String> tableExd = metaMgr.getTableDescExd(tableName); + tableExd.put(MetadataConstants.TABLE_EXD_CARDINALITY, scardi); + metaMgr.saveTableExd(tableName.toUpperCase(), tableExd); + } else { + throw new IllegalArgumentException("No cardinality data is collected for table " + tableName); + } } private static List<String> readLines(Path location, Configuration conf) throws Exception {
