[kylin] 10/11: KYLIN-4342 Build Global Dict by MR/Hive New Version, fix some potential bugs, such as null pointer exceptions

[xxyu]

This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit a2489aaf4560adf7f415629519d6e4b617967dce
Author: wangxiaojing <wangxiaoj...@didichuxing.com>
AuthorDate: Wed May 6 15:24:04 2020 +0800

    KYLIN-4342 Build Global Dict by MR/Hive New Version,  fix
     some potential bugs, such as null pointer exceptions
---
 .../steps/BuildGlobalHiveDicPartBuildReducer.java  | 10 ++++-----
 .../steps/BuildGlobalHiveDictTotalBuildMapper.java | 24 ++++++++++++++++------
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDicPartBuildReducer.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDicPartBuildReducer.java
index 7ecf16d..8cdd8f1 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDicPartBuildReducer.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDicPartBuildReducer.java
@@ -62,14 +62,12 @@ public class BuildGlobalHiveDicPartBuildReducer extends 
KylinReducer<Text, LongW
             colIndex = key.getBytes()[0];//col index
             colName = dicCols[colIndex];
         }
-        logAFewRows(key.toString());
-        mos.write(colIndex+"", new LongWritable(count), new Text(keyBytes), 
"part_sort/"+colIndex);
-    }
 
-    private void logAFewRows(String value) {
         if(count<10){
-            logger.info("key:{}, temp dict num :{},colIndex:{},colName:{}", 
value, count, colIndex, colName);
+            logger.info("key:{}, temp dict num :{}, colIndex:{}, colName:{}", 
key.toString(), count, colIndex, colName);
         }
+
+        mos.write(colIndex+"", new LongWritable(count), new Text(keyBytes), 
"part_sort/"+colIndex);
     }
 
     @Override
@@ -79,7 +77,7 @@ public class BuildGlobalHiveDicPartBuildReducer extends 
KylinReducer<Text, LongW
         String partition = conf.get(MRJobConfig.TASK_PARTITION);
         mos.write(colIndex + "", new LongWritable(count), new Text(partition), 
"reduce_stats/" + colIndex);
         mos.close();
-        logger.info("Reduce partition num {} finish,this reduce done item 
count is {}" , partition, count);
+        logger.info("Reduce partition num {} finish, this reduce done item 
count is {}" , partition, count);
     }
 
 }
diff --git 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDictTotalBuildMapper.java
 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDictTotalBuildMapper.java
index 6efac31..b2252c0 100644
--- 
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDictTotalBuildMapper.java
+++ 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDictTotalBuildMapper.java
@@ -159,8 +159,12 @@ public class BuildGlobalHiveDictTotalBuildMapper<KEYIN, 
Object> extends KylinMap
         } catch (IOException e) {
             e.printStackTrace();
         }finally {
-            buffer.close();
-            in.close();
+            if(buffer!=null) {
+                buffer.close();
+            }
+            if(in!=null) {
+                in.close();
+            }
         }
         return stat.toString();
     }
@@ -168,7 +172,7 @@ public class BuildGlobalHiveDictTotalBuildMapper<KEYIN, 
Object> extends KylinMap
     /**
      *
      * @param conf
-     * @param lastMaxDicValuePath, eg: 
/user/kylin/warehouse/db/kylin_intermediate_kylin_sales_cube_mr_6222c210_ce2d_e8ce_dd0f_f12c38fa9115__group_by/dict_column=KYLIN_MAX_DISTINCT_COUNT/part-00000-450ee120-39ff-4806-afaf-ed482ceffc68-c000
+     * @param lastMaxDicValuePath eg: 
/user/kylin/warehouse/db/kylin_intermediate_kylin_sales_cube_mr_6222c210_ce2d_e8ce_dd0f_f12c38fa9115__group_by/dict_column=KYLIN_MAX_DISTINCT_COUNT/part-00000-450ee120-39ff-4806-afaf-ed482ceffc68-c000
      *        remotePath content is dict colum stats info of per column: dic 
column name,extract distinct value count,last max dic value
      * @return this colIndex's last max dic value
      * @throws IOException
@@ -186,7 +190,11 @@ public class BuildGlobalHiveDictTotalBuildMapper<KEYIN, 
Object> extends KylinMap
 
             }
         }
-        return  map.get(colIndex)==null?0L:map.get(colIndex);
+        if(map == null){
+            return 0L;
+        }else{
+            return  map.get(colIndex)==null?0L:map.get(colIndex);
+        }
     }
 
     /**
@@ -222,8 +230,12 @@ public class BuildGlobalHiveDictTotalBuildMapper<KEYIN, 
Object> extends KylinMap
         } catch (IOException e) {
             e.printStackTrace();
         }finally {
-            buffer.close();
-            in.close();
+            if(buffer!=null) {
+                buffer.close();
+            }
+            if(in!=null) {
+                in.close();
+            }
         }
         logger.info("BuildMaxCountMap map="+map);
         return map;