kylin git commit: minor, use angularjs directive to convert GMT0 timestamp for partition_date

2016-01-20 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 1342e881b -> 6485324a2


minor,use angularjs directive to convert GMT0 timestamp for partition_date


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6485324a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6485324a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6485324a

Branch: refs/heads/2.0-rc
Commit: 6485324a23d2dfb31f4e10489f45f74fd7945749
Parents: 1342e88
Author: jian 
Authored: Thu Jan 21 14:49:03 2016 +0800
Committer: jian 
Committed: Thu Jan 21 14:49:03 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js   | 14 --
 webapp/app/js/controllers/cubes.js  | 29 ++--
 webapp/app/js/directives/directives.js  | 22 +++
 .../partials/cubeDesigner/refresh_settings.html |  2 +-
 webapp/app/partials/jobs/job_merge.html |  2 +-
 webapp/app/partials/jobs/job_refresh.html   |  2 +-
 webapp/app/partials/jobs/job_submit.html|  4 +--
 7 files changed, 29 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/6485324a/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index e2c8fef..4f7d67d 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -164,11 +164,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 CubeDescService.query({cube_name: $routeParams.cubeName}, function 
(detail) {
   if (detail.length > 0) {
 $scope.cubeMetaFrame = detail[0];
-//convert GMT mills ,to make sure partition date show GMT Date
-if($scope.cubeMetaFrame.partition_date_start)
-{
-  $scope.cubeMetaFrame.partition_date_start+=new 
Date().getTimezoneOffset()*6;
-}
 
 $scope.metaModel = {};
 
@@ -221,11 +216,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 reGenerateRowKey();
 
 if ($scope.metaModel.model.partition_desc.partition_date_column && 
($scope.cubeMetaFrame.partition_date_start | 
$scope.cubeMetaFrame.partition_date_start == 0)) {
-  var dateStart = new Date($scope.cubeMetaFrame.partition_date_start);
-  dateStart = (dateStart.getFullYear() + "-" + (dateStart.getMonth() + 1) 
+ "-" + dateStart.getDate());
-  //switch selected time to utc timestamp
-  $scope.cubeMetaFrame.partition_date_start = new 
Date(moment.utc(dateStart, "-MM-DD").format()).getTime();
-
 
   if 
($scope.metaModel.model.partition_desc.partition_date_column.indexOf(".") == 
-1) {
 $scope.metaModel.model.partition_desc.partition_date_column = 
$scope.metaModel.model.fact_table + "." + 
$scope.metaModel.model.partition_desc.partition_date_column;
@@ -459,11 +449,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   }
 
 
-//reverse the date
   $scope.saveCubeRollBack = function () {
-if ($scope.metaModel.model && ($scope.cubeMetaFrame.partition_date_start 
|| $scope.cubeMetaFrame.partition_date_start == 0)) {
-  $scope.cubeMetaFrame.partition_date_start += new 
Date().getTimezoneOffset() * 6;
-}
   }
 
   $scope.updateMandatory = function (rowkey_column) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/6485324a/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js 
b/webapp/app/js/controllers/cubes.js
index df01338..ca53dc5 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -413,19 +413,13 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
   };
   $scope.message = "";
 
-  $scope.rebuild = function (jobsubmit) {
+  $scope.rebuild = function () {
 
 $scope.message = null;
-$scope.jobBuildRequest.startTime = new 
Date($scope.jobBuildRequest.startTime).getTime();
-$scope.jobBuildRequest.endTime = new 
Date($scope.jobBuildRequest.endTime).getTime();
 
 if ($scope.jobBuildRequest.startTime >= $scope.jobBuildRequest.endTime) {
   $scope.message = "WARNING: End time should be later than the start 
time.";
 
-  //rollback date setting
-  if (jobsubmit) {
-$scope.rebuildRollback();
-  }
   return;
 }
 
@@ -437,11 +431,6 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
   SweetAlert.swal('Success!', 'Rebuild job was submitted successfully', 
'success');
 }, function (e) {
 
-  //rollback date setting
-  if (jobsubmit) {
-

kylin git commit: minor, use angularjs directive to convert GMT0 timestamp for partition_date, rm jobEndTime rollback action

2016-01-20 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging eea691651 -> b6c057609


minor,use angularjs directive to convert GMT0 timestamp for partition_date,rm 
jobEndTime rollback action


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b6c05760
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b6c05760
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b6c05760

Branch: refs/heads/1.x-staging
Commit: b6c05760962b2fe5287d331e95e337752dda73c4
Parents: eea6916
Author: jian 
Authored: Thu Jan 21 14:30:18 2016 +0800
Committer: jian 
Committed: Thu Jan 21 14:30:36 2016 +0800

--
 webapp/app/js/controllers/cubes.js| 17 ++---
 webapp/app/js/directives/directives.js|  2 +-
 webapp/app/partials/jobs/job_merge.html   |  2 +-
 webapp/app/partials/jobs/job_refresh.html |  2 +-
 webapp/app/partials/jobs/job_submit.html  |  2 +-
 5 files changed, 6 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/b6c05760/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js 
b/webapp/app/js/controllers/cubes.js
index 2896a1a..6801561 100755
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -454,17 +454,13 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
   };
   $scope.message = "";
 
-  $scope.rebuild = function (jobsubmit) {
+  $scope.rebuild = function () {
 
 $scope.message = null;
 
 if ($scope.jobBuildRequest.startTime >= $scope.jobBuildRequest.endTime) {
   $scope.message = "WARNING: End time should be later than the start 
time.";
 
-  //rollback date setting
-  if (jobsubmit) {
-$scope.rebuildRollback();
-  }
   return;
 }
 
@@ -476,11 +472,6 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
   SweetAlert.swal('Success!', 'Rebuild job was submitted successfully', 
'success');
 }, function (e) {
 
-  //rollback date setting
-  if (jobsubmit) {
-$scope.rebuildRollback();
-  }
-
   loadingRequest.hide();
   if (e.data && e.data.exception) {
 var message = e.data.exception;
@@ -497,7 +488,7 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
   }, function (isConfirm) {
 if (isConfirm) {
   $scope.jobBuildRequest.forceMergeEmptySegment = true;
-  $scope.rebuild(jobsubmit);
+  $scope.rebuild();
 }
   });
   return;
@@ -511,10 +502,6 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
 });
   };
 
-  $scope.rebuildRollback = function () {
-$scope.jobBuildRequest.endTime += new Date().getTimezoneOffset() * 6;
-  }
-
   // used by cube segment refresh
   $scope.segmentSelected = function (selectedSegment) {
 $scope.jobBuildRequest.startTime = 0;

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6c05760/webapp/app/js/directives/directives.js
--
diff --git a/webapp/app/js/directives/directives.js 
b/webapp/app/js/directives/directives.js
index f73b8a8..7b47137 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -228,4 +228,4 @@ KylinApp.directive('kylinPagination', function ($parse, $q) 
{
});
   }
 };
-  });;
+  });

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6c05760/webapp/app/partials/jobs/job_merge.html
--
diff --git a/webapp/app/partials/jobs/job_merge.html 
b/webapp/app/partials/jobs/job_merge.html
index c55b280..f5c3965 100644
--- a/webapp/app/partials/jobs/job_merge.html
+++ b/webapp/app/partials/jobs/job_merge.html
@@ -124,7 +124,7 @@
 
 
 Close
-
 Submit
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6c05760/webapp/app/partials/jobs/job_refresh.html
--
diff --git a/webapp/app/partials/jobs/job_refresh.html 
b/webapp/app/partials/jobs/job_refresh.html
index b8ffa6e..aa8f930 100644
--- a/webapp/app/partials/jobs/job_refresh.html
+++ b/webapp/app/partials/jobs/job_refresh.html
@@ -92,7 +92,7 @@
 
 
 Close
-Submit
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b6c05760/webapp/app/partials/jobs/job_submit.html
--
diff --git a/webapp/app/partials/jobs/job_submit.html 
b/webapp/app/partials/jobs/job_submit.html
index

kylin git commit: KYLIN-1352 Fix storage cache issue in high-concurrency query situation

2016-01-20 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging e5653278c -> 368430527


KYLIN-1352 Fix storage cache issue in high-concurrency query situation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/36843052
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/36843052
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/36843052

Branch: refs/heads/2.x-staging
Commit: 36843052712c286b93af36f73d1e8ab582b27021
Parents: e565327
Author: lidongsjtu 
Authored: Thu Jan 21 10:52:36 2016 +0800
Committer: lidongsjtu 
Committed: Thu Jan 21 14:07:01 2016 +0800

--
 .../apache/kylin/storage/cache/AbstractCacheFledgedQuery.java   | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/36843052/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
index 745ddd1..18011e4 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
@@ -58,7 +58,7 @@ public abstract class AbstractCacheFledgedQuery implements 
IStorageQuery, TeeTup
 //maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).//
 persistence(new 
PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
 
-CACHE_MANAGER.addCache(storageCache);
+CACHE_MANAGER.addCacheIfAbsent(storageCache);
 }
 
 protected StreamSQLResult getStreamSQLResult(StreamSQLDigest 
streamSQLDigest) {
@@ -103,8 +103,7 @@ public abstract class AbstractCacheFledgedQuery implements 
IStorageQuery, TeeTup
 //maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), 
MemoryUnit.BYTES).//using pooled size
 persistence(templateConf.getPersistenceConfiguration()));
 
-CACHE_MANAGER.addCache(storageCache);
-
+CACHE_MANAGER.addCacheIfAbsent(storageCache);
 }
 }
 }

kylin git commit: KYLIN-1297 correct LoadingCache settings

2016-01-20 Thread mahongbin 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 2f5d89a8e -> eea691651


KYLIN-1297 correct LoadingCache settings


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eea69165
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eea69165
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eea69165

Branch: refs/heads/1.x-staging
Commit: eea6916518a59ce137ac6d26bd539e39bb871a6c
Parents: 2f5d89a
Author: honma 
Authored: Thu Jan 21 10:28:54 2016 +0800
Committer: honma 
Committed: Thu Jan 21 10:29:05 2016 +0800

--
 .../org/apache/kylin/common/KylinConfig.java|  5 
 .../apache/kylin/dict/DictionaryManager.java| 31 +++-
 .../storage/hbase/CubeSegmentTupleIterator.java |  1 -
 3 files changed, 22 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/eea69165/common/src/main/java/org/apache/kylin/common/KylinConfig.java
--
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java 
b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index e0f774a..790e7a5 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -466,6 +466,11 @@ public class KylinConfig {
 return this.getOptional("kylin.server.mode", "all");
 }
 
+public int getCachedDictMaxEntrySize() {
+return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", 
"3000"));
+}
+
+
 public int getDictionaryMaxCardinality() {
 return 
Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "500"));
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/eea69165/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
--
diff --git 
a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java 
b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index aa0bc5d..9294e03 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -28,9 +28,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -49,6 +46,10 @@ import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
 public class DictionaryManager {
 
 private static final Logger logger = 
LoggerFactory.getLogger(DictionaryManager.class);
@@ -84,17 +85,19 @@ public class DictionaryManager {
 
 private DictionaryManager(KylinConfig config) {
 this.config = config;
-this.dictCache = 
CacheBuilder.newBuilder().weakValues().expireAfterWrite(10, 
TimeUnit.MINUTES).build(new CacheLoader() {
-@Override
-public DictionaryInfo load(String key) throws Exception {
-DictionaryInfo dictInfo = DictionaryManager.this.load(key, 
true);
-if (dictInfo == null) {
-return NONE_INDICATOR;
-} else {
-return dictInfo;
-}
-}
-});
+this.dictCache = 
CacheBuilder.newBuilder().maximumSize(KylinConfig.getInstanceFromEnv().getCachedDictMaxEntrySize())//
+.expireAfterWrite(1, TimeUnit.DAYS).build(new 
CacheLoader() {
+
+@Override
+public DictionaryInfo load(String key) throws Exception {
+DictionaryInfo dictInfo = 
DictionaryManager.this.load(key, true);
+if (dictInfo == null) {
+return NONE_INDICATOR;
+} else {
+return dictInfo;
+}
+}
+});
 }
 
 public Dictionary getDictionary(String resourcePath) throws IOException 
{

http://git-wip-us.apache.org/repos/asf/kylin/blob/eea69165/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
--
diff --git

kylin git commit: KYLIN-1352 Fix storage cache issue in high-concurrency query situation

2016-01-20 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 58efbca87 -> 1342e881b


KYLIN-1352 Fix storage cache issue in high-concurrency query situation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1342e881
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1342e881
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1342e881

Branch: refs/heads/2.0-rc
Commit: 1342e881bc01a612743891bf057426c44b58f6f3
Parents: 58efbca
Author: lidongsjtu 
Authored: Thu Jan 21 10:52:36 2016 +0800
Committer: lidongsjtu 
Committed: Thu Jan 21 10:52:43 2016 +0800

--
 .../apache/kylin/storage/cache/AbstractCacheFledgedQuery.java   | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/1342e881/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
--
diff --git 
a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
 
b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
index 745ddd1..18011e4 100644
--- 
a/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
+++ 
b/core-storage/src/main/java/org/apache/kylin/storage/cache/AbstractCacheFledgedQuery.java
@@ -58,7 +58,7 @@ public abstract class AbstractCacheFledgedQuery implements 
IStorageQuery, TeeTup
 //maxBytesLocalHeap(10, MemoryUnit.MEGABYTES).//
 persistence(new 
PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
 
-CACHE_MANAGER.addCache(storageCache);
+CACHE_MANAGER.addCacheIfAbsent(storageCache);
 }
 
 protected StreamSQLResult getStreamSQLResult(StreamSQLDigest 
streamSQLDigest) {
@@ -103,8 +103,7 @@ public abstract class AbstractCacheFledgedQuery implements 
IStorageQuery, TeeTup
 //maxBytesLocalHeap(templateConf.getMaxBytesLocalHeap(), 
MemoryUnit.BYTES).//using pooled size
 persistence(templateConf.getPersistenceConfiguration()));
 
-CACHE_MANAGER.addCache(storageCache);
-
+CACHE_MANAGER.addCacheIfAbsent(storageCache);
 }
 }
 }

kylin git commit: minor, use angularjs directive to convert GMT0 timestamp for partition_date

2016-01-20 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 368430527 -> d829340c5


minor,use angularjs directive to convert GMT0 timestamp for partition_date


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d829340c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d829340c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d829340c

Branch: refs/heads/2.x-staging
Commit: d829340c5facc1de1f44f114f9af9b06f51996e4
Parents: 3684305
Author: jian 
Authored: Thu Jan 21 15:08:39 2016 +0800
Committer: jian 
Committed: Thu Jan 21 15:08:39 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js   | 23 +++-
 webapp/app/js/controllers/cubes.js  | 28 ++--
 webapp/app/js/directives/directives.js  | 22 +++
 .../partials/cubeDesigner/refresh_settings.html |  2 +-
 webapp/app/partials/jobs/job_merge.html |  2 +-
 webapp/app/partials/jobs/job_refresh.html   |  2 +-
 webapp/app/partials/jobs/job_submit.html|  4 +--
 7 files changed, 32 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/d829340c/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index ea9e822..8259ab1 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -165,11 +165,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 CubeDescService.query({cube_name: $routeParams.cubeName}, function 
(detail) {
   if (detail.length > 0) {
 $scope.cubeMetaFrame = detail[0];
-//convert GMT mills ,to make sure partition date show GMT Date
-  if($scope.cubeMetaFrame.partition_date_start)
-  {
-$scope.cubeMetaFrame.partition_date_start+=new 
Date().getTimezoneOffset()*6;
-  }
 $scope.metaModel = {};
 
 //get model from API when page refresh
@@ -221,21 +216,12 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 reGenerateRowKey();
 
 if ($scope.metaModel.model.partition_desc.partition_date_column && 
($scope.cubeMetaFrame.partition_date_start | 
$scope.cubeMetaFrame.partition_date_start == 0)) {
-  var dateStart = new Date($scope.cubeMetaFrame.partition_date_start);
-  dateStart = (dateStart.getFullYear() + "-" + (dateStart.getMonth() + 1) 
+ "-" + dateStart.getDate());
-  //switch selected time to utc timestamp
-  $scope.cubeMetaFrame.partition_date_start = new 
Date(moment.utc(dateStart, "-MM-DD").format()).getTime();
-
 
-  //if 
($scope.metaModel.model.partition_desc.partition_date_column.indexOf(".") == 
-1) {
-  //  $scope.metaModel.model.partition_desc.partition_date_column = 
$scope.metaModel.model.fact_table + "." + 
$scope.metaModel.model.partition_desc.partition_date_column;
-  //}
+  if 
($scope.metaModel.model.partition_desc.partition_date_column.indexOf(".") == 
-1) {
+$scope.metaModel.model.partition_desc.partition_date_column = 
$scope.metaModel.model.fact_table + "." + 
$scope.metaModel.model.partition_desc.partition_date_column;
+  }
 
 }
-//use cubedesc name as model name
-//if ($scope.metaModel.model.name === "" || 
angular.isUndefined($scope.metaModel.model.name)) {
-//  $scope.metaModel.model.name = $scope.cubeMetaFrame.name;
-//}
 
 //set model ref for cubeDesc
 if ($scope.cubeMetaFrame.model_name === "" || 
angular.isUndefined($scope.cubeMetaFrame.model_name)) {
@@ -463,9 +449,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 
 //reverse the date
   $scope.saveCubeRollBack = function () {
-if ($scope.metaModel.model && ($scope.cubeMetaFrame.partition_date_start 
|| $scope.cubeMetaFrame.partition_date_start == 0)) {
-  $scope.cubeMetaFrame.partition_date_start += new 
Date().getTimezoneOffset() * 6;
-}
   }
 
   $scope.updateMandatory = function (rowkey_column) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/d829340c/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js 
b/webapp/app/js/controllers/cubes.js
index 4cd4217..b9a0f13 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -412,19 +412,13 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
   };
   $scope.message = "";
 
-  $scope.rebuild = function (jobsubmit) {
+  $scope.rebuild = function () {
 
 $scope.message = null;
-$scope.jobBuildRequest.startTime = new

kylin git commit: minor, use angularjs to convert GMT0 timestamp for partition_date

2016-01-20 Thread zhongjian 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging f86314869 -> 2f5d89a8e


minor,use angularjs to convert GMT0 timestamp for partition_date


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f5d89a8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f5d89a8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f5d89a8

Branch: refs/heads/1.x-staging
Commit: 2f5d89a8eda0db57e669063c6af887293f31575d
Parents: f863148
Author: jian 
Authored: Thu Jan 21 01:32:48 2016 +0800
Committer: jian 
Committed: Thu Jan 21 01:33:12 2016 +0800

--
 webapp/app/js/controllers/cubeEdit.js   | 16 -
 webapp/app/js/controllers/cubes.js  | 12 --
 webapp/app/js/directives/directives.js  | 24 +++-
 webapp/app/js/model/metaModel.js|  5 
 .../app/partials/cubeDesigner/incremental.html  |  3 +--
 webapp/app/partials/jobs/job_submit.html| 11 ++---
 6 files changed, 26 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/2f5d89a8/webapp/app/js/controllers/cubeEdit.js
--
diff --git a/webapp/app/js/controllers/cubeEdit.js 
b/webapp/app/js/controllers/cubeEdit.js
index 1178bee..0095fde 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -95,13 +95,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
   if (model) {
 MetaModel.setMetaModel(model);
 $scope.metaModel = MetaModel;
-
-//use
-//convert GMT mills ,to make sure partition date show GMT Date
-//should run only one time
-if (model.partition_desc && 
model.partition_desc.partition_date_start) {
-  MetaModel.converDateToGMT();
-}
   }
 });
 $scope.state.cubeSchema = angular.toJson($scope.cubeMetaFrame, true);
@@ -129,11 +122,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 
 
 if ($scope.metaModel.model.partition_desc.partition_date_column && 
($scope.metaModel.model.partition_desc.partition_date_start | 
$scope.metaModel.model.partition_desc.partition_date_start == 0)) {
-  var dateStart = new 
Date($scope.metaModel.model.partition_desc.partition_date_start);
-  dateStart = (dateStart.getFullYear() + "-" + (dateStart.getMonth() + 1) 
+ "-" + dateStart.getDate());
-  //switch selected time to utc timestamp
-  $scope.metaModel.model.partition_desc.partition_date_start = new 
Date(moment.utc(dateStart, "-MM-DD").format()).getTime();
-
 
   if 
($scope.metaModel.model.partition_desc.partition_date_column.indexOf(".") == 
-1) {
 $scope.metaModel.model.partition_desc.partition_date_column = 
$scope.metaModel.model.fact_table + "." + 
$scope.metaModel.model.partition_desc.partition_date_column;
@@ -286,10 +274,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, 
$routeParams, $locatio
 
 //reverse the date
   $scope.saveCubeRollBack = function () {
-if ($scope.metaModel.model && 
($scope.metaModel.model.partition_desc.partition_date_start || 
$scope.metaModel.model.partition_desc.partition_date_start == 0)) {
-  $scope.metaModel.model.partition_desc.partition_date_start += new 
Date().getTimezoneOffset() * 6;
-}
-
 //update model last modified
 $scope.statusRefresh();
   }

http://git-wip-us.apache.org/repos/asf/kylin/blob/2f5d89a8/webapp/app/js/controllers/cubes.js
--
diff --git a/webapp/app/js/controllers/cubes.js 
b/webapp/app/js/controllers/cubes.js
index d277dc1..2896a1a 100755
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -457,8 +457,6 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
   $scope.rebuild = function (jobsubmit) {
 
 $scope.message = null;
-$scope.jobBuildRequest.startTime = new 
Date($scope.jobBuildRequest.startTime).getTime();
-$scope.jobBuildRequest.endTime = new 
Date($scope.jobBuildRequest.endTime).getTime();
 
 if ($scope.jobBuildRequest.startTime >= $scope.jobBuildRequest.endTime) {
   $scope.message = "WARNING: End time should be later than the start 
time.";
@@ -548,16 +546,6 @@ var jobSubmitCtrl = function ($scope, $modalInstance, 
CubeService, MessageServic
 }
   };
 
-  $scope.updateDate = function () {
-$scope.jobBuildRequest.endTime = 
$scope.formatDate($scope.jobBuildRequest.endTime);
-  };
-
-  $scope.formatDate = function (timestemp) {
-var dateStart = new Date(timestemp);
-dateStart =

kylin git commit: KYLIN-1329 Bug fix for ACL exception when deleting model

2016-01-20 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 80171d6ae -> 8eb289d52


KYLIN-1329 Bug fix for ACL exception when deleting model


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8eb289d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8eb289d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8eb289d5

Branch: refs/heads/1.x-staging
Commit: 8eb289d52955b4ee7ae65acfbb7ba03f27e8f2d4
Parents: 80171d6
Author: lidongsjtu 
Authored: Wed Jan 20 16:02:43 2016 +0800
Committer: lidongsjtu 
Committed: Wed Jan 20 16:02:43 2016 +0800

--
 .../java/org/apache/kylin/rest/service/AccessService.java   | 9 ++---
 1 file changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/8eb289d5/server/src/main/java/org/apache/kylin/rest/service/AccessService.java
--
diff --git 
a/server/src/main/java/org/apache/kylin/rest/service/AccessService.java 
b/server/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 216c454..2f0e05a 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -56,10 +56,9 @@ import org.springframework.util.Assert;
 public class AccessService {
 
 @Autowired
-private AclService aclService;
-
-@Autowired
 UserService userService;
+@Autowired
+private AclService aclService;
 
 // ~ Methods to manage acl life circle of domain objects ~
 
@@ -219,6 +218,10 @@ public class AccessService {
 public void clean(AclEntity ae, boolean deleteChildren) {
 Assert.notNull(ae, "Acl domain object required");
 
+// For those may have null uuid, like DataModel, won't delete Acl.
+if (ae.getId() == null)
+return;
+
 ObjectIdentity objectIdentity = new ObjectIdentityImpl(ae.getClass(), 
ae.getId());
 
 try {

[2/2] kylin git commit: KYLIN-1312 Enhance DeployCoprocessorCLI to support HTable/Cube filter

2016-01-20 Thread lidong 
KYLIN-1312 Enhance DeployCoprocessorCLI to support HTable/Cube filter


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8631486
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8631486
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8631486

Branch: refs/heads/1.x-staging
Commit: f863148696771526e8e7d360159174dc37c9d4eb
Parents: e90763d
Author: lidongsjtu 
Authored: Wed Jan 20 16:24:02 2016 +0800
Committer: lidongsjtu 
Committed: Wed Jan 20 16:24:02 2016 +0800

--
 .../kylin/job/tools/DeployCoprocessorCLI.java   | 77 +++-
 1 file changed, 74 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/f8631486/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java 
b/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
index 5482684..bf655a0 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/DeployCoprocessorCLI.java
@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -54,15 +55,19 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.Lists;
+
 /**
  * @author yangli9
  */
 public class DeployCoprocessorCLI {
 
-private static final Logger logger = 
LoggerFactory.getLogger(DeployCoprocessorCLI.class);
-
+public static final String CubeObserverClassV2 = 
"org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.AggregateRegionObserver";
+public static final String CubeEndpointClassV2 = 
"org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.CubeVisitService";
+public static final String IIEndpointClassV2 = 
"org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.IIEndpoint";
 public static final String OBSERVER_CLS_NAME = 
"org.apache.kylin.storage.hbase.coprocessor.observer.AggregateRegionObserver";
 public static final String ENDPOINT_CLS_NAMAE = 
"org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint";
+private static final Logger logger = 
LoggerFactory.getLogger(DeployCoprocessorCLI.class);
 
 public static void main(String[] args) throws IOException {
 KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
@@ -76,6 +81,21 @@ public class DeployCoprocessorCLI {
 List tableNames = getHTableNames(kylinConfig);
 logger.info("Identify tables " + tableNames);
 
+if (args.length <= 1) {
+printUsageAndExit();
+}
+
+String filterType = args[1].toLowerCase();
+if (filterType.equals("-table")) {
+tableNames = filterByTables(tableNames, 
Arrays.asList(args).subList(2, args.length));
+} else if (filterType.equals("-cube")) {
+tableNames = filterByCubes(tableNames, 
Arrays.asList(args).subList(2, args.length));
+} else if (!filterType.equals("all")) {
+printUsageAndExit();
+}
+
+logger.info("Will execute tables " + tableNames);
+
 Set oldJarPaths = getCoprocessorJarPaths(hbaseAdmin, 
tableNames);
 logger.info("Old coprocessor jar: " + oldJarPaths);
 
@@ -93,6 +113,45 @@ public class DeployCoprocessorCLI {
 logger.info("Active coprocessor jar: " + hdfsCoprocessorJar);
 }
 
+private static void printUsageAndExit() {
+logger.warn("Probe run, exiting.");
+logger.info("Usage: bin/kylin.sh 
org.apache.kylin.job.tools.DeployCoprocessorCLI JAR_FILE all|-cube CUBE1 
CUBE2|-table TABLE1 TABLE2");
+System.exit(0);
+}
+
+private static List filterByCubes(List allTableNames, 
List cubeNames) {
+CubeManager cubeManager = 
CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+List result = Lists.newArrayList();
+for (String c : cubeNames) {
+c = c.trim();
+if (c.endsWith(","))
+c = c.substring(0, c.length() - 1);
+
+CubeInstance cubeInstance = cubeManager.getCube(c);
+for (CubeSegment segment : cubeInstance.getSegments()) {
+String tableName = segment.getStorageLocationIdentifier();
+if (allTableNames.contains(tableName)) {
+result.add(tableName);
+}
+}
+}
+return result;
+}
+
+private static List filterByTables(List allTableNames, 
List

svn commit: r1725680 [3/3] - in /kylin/site: ./ cn/docs/ cn/docs/install/ cn/docs/tutorial/ docs/ docs/gettingstarted/ docs/howto/ docs/install/ docs/tutorial/

2016-01-20 Thread shaofengshi 
Modified: kylin/site/docs/release_notes.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs/release_notes.html?rev=1725680=1725679=1725680=diff
==
--- kylin/site/docs/release_notes.html (original)
+++ kylin/site/docs/release_notes.html Wed Jan 20 08:34:15 2016
@@ -224,6 +224,8 @@
 
   
 
+  
+
   Overview  
   
 
@@ -286,6 +288,8 @@
 
   
 
+  
+
   Apache Kylin™ Release Notes  
   
 
@@ -430,6 +434,8 @@
 
   
 
+  
+
   Installation Guide  
   
 
@@ -478,6 +484,8 @@
 
   
 
+  
+
   Hadoop Environment  
   
 
@@ -538,6 +546,8 @@
 
   
 
+  
+
   Manual Installation 
Guide  
   
 
@@ -592,6 +602,8 @@
 
   
 
+  
+
   Multiple Kylin REST 
servers  
   
 
@@ -638,6 +650,8 @@
 
   
 
+  
+
   Advance Settings of Kylin 
Environment  
   
 
@@ -694,6 +708,8 @@
 
   
 
+  
+
   On Hadoop Kylin installation using Docker  
   
 
@@ -788,6 +804,8 @@
 
   
 
+  
+
   Quick Start with Sample 
Cube  
   
 
@@ -858,6 +876,8 @@
 
   
 
+  
+
   Kylin Cube Creation Tutorial  
   
 
@@ -932,6 +952,8 @@
 
   
 
+  
+
   Kylin Cube Build and Job 
Monitoring Tutorial  
   
 
@@ -998,6 +1020,8 @@
 
   
 
+  
+
   Kylin Cube Permission Grant Tutorial  
   
 
@@ -1094,6 +1118,8 @@
 
   
 
+  
+
   Kylin Web Tutorial  
   
 
@@ -1182,6 +1208,8 @@
 
   
 
+  
+
   Tableau Tutorial  
   
 
@@ -1274,6 +1302,8 @@
 
   
 
+  
+
   Tableau 9 Tutorial  
   
 
@@ -1358,6 +1388,8 @@
 
   
 
+  
+
   MS Excel and Power BI Tutorial  
   
 
@@ -1438,6 +1470,8 @@
 
   
 
+  
+
   Kylin ODBC Driver Tutorial  
   
 
@@ -1524,6 +1558,8 @@
 
   
 
+  
+
   How to Use Restful API in 
Javascript  
   
 
@@ -1562,6 +1598,8 @@
 
   
 
+  
+
   How to Use Restful 
API  
   
 
@@ -1596,6 +1634,8 @@
 
   
 
+  
+
   How to Optimize 
Cubes  
   
 
@@ -1720,10 +1760,46 @@
 
   
 
+  
+
   How to Upgrade  
   
 
 
+
+  
+  
+
+  
+
+  
+
+
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  
+
+  How to Enable Security with 
LDAP and SSO  
+  
+
+
 
 
 

Modified: kylin/site/docs/tutorial/acl.html
URL: 
http://svn.apache.org/viewvc/kylin/site/docs/tutorial/acl.html?rev=1725680=1725679=1725680=diff
==
--- kylin/site/docs/tutorial/acl.html (original)
+++ kylin/site/docs/tutorial/acl.html Wed Jan 20 08:34:15 2016
@@ -224,6 +224,8 @@
 
   
 
+  
+
   Overview  
   
 
@@ -286,6 +288,8 @@
 
   
 
+  
+
   Apache Kylin™ Release Notes  
   
 
@@ -430,6 +434,8 @@
 
   
 
+  
+
   Installation Guide  
   
 
@@ -478,6 +484,8 @@
 
   
 
+  
+
   Hadoop Environment  
   
 
@@ -538,6 +546,8 @@
 
   
 
+  
+
   Manual Installation 
Guide  
   
 
@@ -592,6 +602,8 @@
 
   
 
+  
+
   Multiple Kylin REST 
servers  
   
 
@@ -638,6 +650,8 @@
 
   
 
+  
+
   Advance Settings of Kylin 
Environment  
   
 
@@ -694,6 +708,8 @@
 
   
 
+  
+
   On Hadoop Kylin installation using Docker  
   
 
@@ -788,6 +804,8 @@
 
   
 
+  
+
   Quick Start with Sample 
Cube  
   
 
@@ -858,6 +876,8 @@
 
   
 
+  
+
   Kylin Cube Creation Tutorial  
   
 
@@ -932,6 +952,8 @@
 
   
 
+  
+
   Kylin Cube Build and Job 
Monitoring Tutorial  
   
 
@@ -998,6 +1020,8 @@
 
   
 
+  
+
   Kylin Cube Permission Grant Tutorial  
   
 
@@ -1094,6 +1118,8 @@
 
   
 
+  
+
   Kylin Web Tutorial  
   
 
@@ -1182,6 +1208,8 @@
 
   
 
+  
+
   Tableau Tutorial  
   
 
@@ -1274,6 +1302,8 @@
 
   
 
+  
+
   Tableau 9 Tutorial  
   
 
@@ -1358,6 +1388,8 @@
 
   
 
+  
+
   MS Excel and Power BI Tutorial  
   
 
@@ -1438,6 +1470,8 @@
 
   
 
+  
+
   Kylin ODBC Driver Tutorial  
   
 
@@ -1524,6 +1558,8 @@
 
   
 
+  
+
   How to Use Restful API in 
Javascript  
   
 
@@ -1562,6 +1598,8 @@
 
   
 
+  
+
   How to Use Restful 
API  
   
 
@@ -1596,6 +1634,8 @@
 
   
 
+  
+
   How to Optimize 
Cubes  
   
 
@@ -1720,10 +1760,46 @@
 
   
 
+  
+
   How to Upgrade  
   
 
 
+
+  
+  
+
+  
+
+

kylin git commit: KYLIN-1280 convert other hdfs path to master name node

2016-01-20 Thread fengyu 
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 5f7f3a750 -> 62b00f9ea


KYLIN-1280 convert other hdfs path to master name node


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/62b00f9e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/62b00f9e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/62b00f9e

Branch: refs/heads/2.0-rc
Commit: 62b00f9eaac63d5c18158d9c1e98ebbd857b5266
Parents: 5f7f3a7
Author: fengyu 
Authored: Wed Jan 20 10:22:38 2016 +0800
Committer: fengyu 
Committed: Wed Jan 20 16:43:07 2016 +0800

--
 .../apache/kylin/common/KylinConfigBase.java|  4 +
 .../org/apache/kylin/engine/mr/HadoopUtil.java  | 82 
 .../apache/kylin/engine/spark/SparkCubing.java  |  4 +
 .../kylin/storage/hbase/steps/CubeHFileJob.java |  4 +
 4 files changed, 94 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/62b00f9e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
--
diff --git 
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java 
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4c640f3..5f91d53 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -482,6 +482,10 @@ public class KylinConfigBase implements Serializable {
 public boolean isGetJobStatusWithKerberos() {
 return 
Boolean.valueOf(this.getOptional("kylin.job.status.with.kerberos", "false"));
 }
+
+public boolean isTransformPathToMasterNN() {
+return 
Boolean.valueOf(this.getOptional("kylin.transform.hdfs.path.enable", "false"));
+}
 
 public String getKylinOwner() {
 return this.getOptional("kylin.owner", "");

http://git-wip-us.apache.org/repos/asf/kylin/blob/62b00f9e/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
--
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java 
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 9ce2bab..f8d469e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -21,18 +21,25 @@ package org.apache.kylin.engine.mr;
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Map;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.io.Writable;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.log4j.Logger;
 
 public class HadoopUtil {
 private static final ThreadLocal hadoopConfig = new 
ThreadLocal<>();
+private static final Logger logger = Logger.getLogger(HadoopUtil.class);
 
 public static void setCurrentConfiguration(Configuration conf) {
 hadoopConfig.set(conf);
@@ -115,5 +122,80 @@ public class HadoopUtil {
 throw new RuntimeException(e);
 }
 }
+
+public static Configuration convertCurrentConfig(String path) {
+Configuration currentConfig = getCurrentConfiguration();
+if(path == null)
+return currentConfig;
+String nameService = currentConfig.get(FileSystem.FS_DEFAULT_NAME_KEY);
+logger.debug("Current convert path " + path);
+logger.debug("Current default name service " + nameService);
+try {
+URI pathUri = new URI(path);
+String host = pathUri.getHost();
+//do not transform path with default name service.
+if(nameService != null) {
+URI defaultUri = new URI(nameService);
+
if(pathUri.getScheme().equalsIgnoreCase(defaultUri.getScheme()) && 
host.equalsIgnoreCase(defaultUri.getHost())) {
+return currentConfig;
+}
+}
+//get namespace to real name node map..
+Map> map = 
DFSUtil.getHaNnRpcAddresses(currentConfig);
+Map addressesInNN = map.get(host);
+//if do not exist this namespace, such as we use real name node
+if(addressesInNN == null)
+return currentConfig;
+

[1/2] kylin git commit: KYLIN-1327 Tool for batch updating host information of htables

2016-01-20 Thread lidong 
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 8eb289d52 -> f86314869


KYLIN-1327 Tool for batch updating host information of htables


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e90763dc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e90763dc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e90763dc

Branch: refs/heads/1.x-staging
Commit: e90763dc8473225941d01b91fa141f5de2fa33c6
Parents: 8eb289d
Author: lidongsjtu 
Authored: Wed Jan 20 16:13:13 2016 +0800
Committer: lidongsjtu 
Committed: Wed Jan 20 16:13:13 2016 +0800

--
 .../kylin/job/tools/UpdateHTableHostCLI.java| 199 +++
 1 file changed, 199 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kylin/blob/e90763dc/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java
--
diff --git 
a/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java 
b/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java
new file mode 100644
index 000..30ffb46
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/tools/UpdateHTableHostCLI.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.job.tools;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+
+import com.google.common.collect.Lists;
+
+public class UpdateHTableHostCLI {
+private static final Log logger = 
LogFactory.getLog(UpdateHTableHostCLI.class);
+private List updatedResources = Lists.newArrayList();
+private List errorMsgs = Lists.newArrayList();
+
+private List htables;
+private HBaseAdmin hbaseAdmin;
+private KylinConfig kylinConfig;
+private String oldHostValue;
+
+public UpdateHTableHostCLI(List htables, String oldHostValue) 
throws IOException {
+this.htables = htables;
+this.oldHostValue = oldHostValue;
+this.hbaseAdmin = new 
HBaseAdmin(HadoopUtil.getCurrentHBaseConfiguration());
+this.kylinConfig = KylinConfig.getInstanceFromEnv();
+}
+
+public static void main(String args[]) throws Exception {
+if (args.length < 1) {
+printUsageAndExit();
+}
+
+List tableNames = 
getHTableNames(KylinConfig.getInstanceFromEnv());
+if (!args[0].toLowerCase().equals("-from")) {
+printUsageAndExit();
+}
+String oldHostValue = args[1].toLowerCase();
+String filterType = args[2].toLowerCase();
+if (filterType.equals("-table")) {
+tableNames = filterByTables(tableNames, 
Arrays.asList(args).subList(3, args.length));
+} else if (filterType.equals("-cube")) {
+tableNames = filterByCubes(tableNames, 
Arrays.asList(args).subList(3, args.length));
+} else if (!filterType.equals("-all")) {
+printUsageAndExit();
+}
+logger.info("These htables are needed to be updated: " + 
StringUtils.join(tableNames, ","));
+
+UpdateHTableHostCLI updateHTableHostCLI = new