[kylin] 27/41: EBAY-KYLIN-1204 Check and fix Kylin Yarn job tracking url
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c5ef6a5ed2f97c1bdb8b1e064613f65f3cb3c830
Author: Wang Ken
AuthorDate: Fri Jun 19 18:14:22 2020 +0800
EBAY-KYLIN-1204 Check and fix Kylin Yarn job tracking url
---
.../kylin/engine/mr/common/HadoopCmdOutput.java | 19 ++-
1 file changed, 18 insertions(+), 1 deletion(-)
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index df89ed8..53282cc 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -47,6 +47,8 @@ public class HadoopCmdOutput {
private final StringBuilder output;
private final Job job;
+private String trackingURL;
+
public HadoopCmdOutput(Job job, StringBuilder output) {
super();
this.job = job;
@@ -64,7 +66,22 @@ public class HadoopCmdOutput {
status.put(ExecutableConstants.MR_JOB_ID,
job.getJobID().toString());
}
if (null != job.getTrackingURL()) {
-status.put(ExecutableConstants.YARN_APP_URL,
job.getTrackingURL().toString());
+String newTrackingURL = job.getTrackingURL();
+logger.info(ExecutableConstants.YARN_APP_URL + ":" +
newTrackingURL);
+if (trackingURL == null) {
+trackingURL = newTrackingURL;
+} else if (!newTrackingURL.equals(trackingURL)) {
+// fix for the bug that getTrackingURL() return url
without http or https header even https is enabled,
+// just a workaround, tobe deleted when this issue resolved
+if (trackingURL.startsWith("https://;) &&
(!newTrackingURL.startsWith("https://;))) {
+trackingURL = "https://; + newTrackingURL;
+} else if (trackingURL.startsWith("http://;) &&
(!newTrackingURL.startsWith("http://;))) {
+trackingURL = "http://; + newTrackingURL;
+} else {
+trackingURL = newTrackingURL;
+}
+}
+status.put(ExecutableConstants.YARN_APP_URL, trackingURL);
}
return status;
} else {
[kylin] 13/41: KYLIN-4564 Refine query metrics update for hybrid realization
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d435a6490d70df84ea1952393549b1f2bd94db8b
Author: Zhong, Yanghong
AuthorDate: Mon Jun 15 12:20:07 2020 +0800
KYLIN-4564 Refine query metrics update for hybrid realization
---
.../kylin/rest/metrics/QueryMetricsFacade.java | 29 +-
1 file changed, 28 insertions(+), 1 deletion(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
index f51bf08..ab06093 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.metrics;
import java.nio.charset.Charset;
+import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -30,6 +31,7 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryContextFacade;
+import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.metrics.lib.impl.RecordEvent;
import org.apache.kylin.metrics.lib.impl.TimedRecordEvent;
@@ -121,6 +123,7 @@ public class QueryMetricsFacade {
sqlResponse.getThrowable());
long totalStorageReturnCount = 0L;
+Map totalStorageReturnCountMap = new HashMap<>();
if
(contextEntry.getQueryType().equalsIgnoreCase(OLAPQuery.EnumeratorTypeEnum.OLAP.name()))
{
for (Map cubeEntry
: contextEntry.getCubeSegmentStatisticsMap()
.values()) {
@@ -140,6 +143,12 @@ public class QueryMetricsFacade {
segmentEntry.isIfSuccess(), 1.0 /
cubeEntry.size());
totalStorageReturnCount +=
segmentEntry.getStorageReturnedRows();
+Long storageReturnCount =
totalStorageReturnCountMap.get(segmentEntry.getCubeName());
+if (storageReturnCount == null) {
+storageReturnCount = 0L;
+}
+storageReturnCount +=
segmentEntry.getStorageReturnedRows();
+
totalStorageReturnCountMap.put(segmentEntry.getCubeName(), storageReturnCount);
//For update cube segment level related query metrics
MetricsManager.getInstance().update(cubeSegmentMetricsEvent);
}
@@ -152,8 +161,26 @@ public class QueryMetricsFacade {
setQueryStats(queryMetricsEvent, //
sqlResponse.getDuration(), sqlResponse.getResults() ==
null ? 0 : sqlResponse.getResults().size(),
totalStorageReturnCount);
-//For update query level metrics
+// For update query level metrics
MetricsManager.getInstance().update(queryMetricsEvent);
+
+// If hit a hybrid, update query level metrics for each inner cubes
+if (contextEntry.getRealizationType() == IStorageAware.ID_HYBRID) {
+for (Map.Entry entry :
totalStorageReturnCountMap.entrySet()) {
+RecordEvent queryMetricsEventInner = new TimedRecordEvent(
+
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQuery());
+setQueryWrapper(queryMetricsEventInner, //
+user, sqlRequest.getSql(),
+sqlResponse.isStorageCacheUsed() ? "CACHE" :
contextEntry.getQueryType(),
+norm(sqlRequest.getProject()), entry.getKey(),
IStorageAware.ID_SHARDED_HBASE,
+sqlResponse.getThrowable());
+// Use the total result size for each cube currently
+setQueryStats(queryMetricsEventInner, //
+sqlResponse.getDuration(),
+sqlResponse.getResults() == null ? 0 :
sqlResponse.getResults().size(), entry.getValue());
+
MetricsManager.getInstance().update(queryMetricsEventInner);
+}
+}
}
}
[kylin] 19/41: EBAY-KYLIN-1919 Add prometheus metrics reporter
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2b3b6df124dc93c7efd4dbdd32a92d87c2b8dcc1
Author: Wang gang
AuthorDate: Fri Jun 19 10:31:14 2020 +0800
EBAY-KYLIN-1919 Add prometheus metrics reporter
---
metrics-reporter-prometheus/pom.xml| 79
.../impl/prometheus/PrometheusEventProducer.java | 489 +
.../prometheus/PrometheusReservoirReporter.java| 140 ++
.../prometheus/PrometheusEventProducerTest.java| 207 +
.../PrometheusReservoirReporterTest.java | 86
pom.xml| 9 +
server/src/main/resources/kylinMetrics.xml | 12 +-
7 files changed, 1020 insertions(+), 2 deletions(-)
diff --git a/metrics-reporter-prometheus/pom.xml
b/metrics-reporter-prometheus/pom.xml
new file mode 100644
index 000..f1ac16c
--- /dev/null
+++ b/metrics-reporter-prometheus/pom.xml
@@ -0,0 +1,79 @@
+
+
+
+http://maven.apache.org/POM/4.0.0;
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+kylin-metrics-reporter-prometheus
+jar
+Apache Kylin - Metrics Reporter Prometheus
+Apache Kylin - Metrics Reporter Prometheus
+
+
+kylin
+org.apache.kylin
+3.1.0
+
+
+
+
+org.apache.kylin
+kylin-core-metadata
+
+
+org.apache.kylin
+kylin-storage-hbase
+
+
+io.prometheus
+simpleclient_servlet
+${prometheus.version}
+
+
+org.eclipse.jetty
+jetty-servlet
+
+
+org.eclipse.jetty
+jetty-server
+
+
+
+
+junit
+junit
+test
+
+
+org.powermock
+powermock-api-mockito
+${powermock.version}
+test
+
+
+org.powermock
+powermock-module-junit4-rule-agent
+${powermock.version}
+test
+
+
+
+
\ No newline at end of file
diff --git
a/metrics-reporter-prometheus/src/main/java/org/apache/kylin/metrics/lib/impl/prometheus/PrometheusEventProducer.java
b/metrics-reporter-prometheus/src/main/java/org/apache/kylin/metrics/lib/impl/prometheus/PrometheusEventProducer.java
new file mode 100644
index 000..8784c18
--- /dev/null
+++
b/metrics-reporter-prometheus/src/main/java/org/apache/kylin/metrics/lib/impl/prometheus/PrometheusEventProducer.java
@@ -0,0 +1,489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.metrics.lib.impl.prometheus;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exceptions.ResourceLimitExceededException;
+import org.apache.kylin.metrics.lib.Record;
+import org.apache.kylin.metrics.property.JobPropertyEnum;
+import org.apache.kylin.metrics.property.QueryPropertyEnum;
+import org.apache.kylin.metrics.property.QueryRPCPropertyEnum;
+import org.apache.kylin.stream.core.util.NamedThreadFactory;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.prometheus.client.Counter;
+import io.prometheus.client.Gauge;
+import io.prometheus.client.exporter.MetricsServlet;
+
+public class PrometheusEventProducer {
+private static final Logger logger =
LoggerFactory.getLogger(PrometheusEventProducer.c
[kylin] 18/41: KYLIN-4579 Add fat API to create cube & delete cube
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6c5c4551609a354c99cbec670dc398061ce111f7
Author: Zhong, Yanghong
AuthorDate: Thu Jun 18 17:28:41 2020 +0800
KYLIN-4579 Add fat API to create cube & delete cube
---
.../kylin/rest/controller/CubeController.java | 142 -
.../kylin/rest/request/CubeWithAllRequest.java | 97 ++
.../org/apache/kylin/rest/service/CubeService.java | 12 +-
3 files changed, 245 insertions(+), 6 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index af3e9b1..baedcd9 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -29,6 +29,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.UUID;
import javax.servlet.http.HttpServletResponse;
@@ -36,6 +37,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -55,11 +57,13 @@ import
org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.job.exception.JobException;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -70,6 +74,7 @@ import
org.apache.kylin.rest.exception.TooManyRequestException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.CubeRequest;
+import org.apache.kylin.rest.request.CubeWithAllRequest;
import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
@@ -84,8 +89,10 @@ import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
+import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.QueryService;
+import org.apache.kylin.rest.service.TableService;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
import org.apache.kylin.shaded.com.google.common.collect.Lists;
@@ -135,6 +142,14 @@ public class CubeController extends BasicController {
private QueryService queryService;
@Autowired
+@Qualifier("modelMgmtService")
+private ModelService modelService;
+
+@Autowired
+@Qualifier("tableService")
+private TableService tableService;
+
+@Autowired
private AclEvaluate aclEvaluate;
@RequestMapping(value = "{cubeName}/validate", method = RequestMethod.GET,
produces = { "application/json" })
@@ -593,7 +608,7 @@ public class CubeController extends BasicController {
//drop Cube
try {
-cubeService.deleteCube(cube);
+cubeService.deleteCube(cube, true);
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException("Failed to delete cube. " + "
Caused by: " + e.getMessage(), e);
@@ -601,6 +616,131 @@ public class CubeController extends BasicController {
}
+@RequestMapping(value = "/{cubeName}/withAll", method = {
RequestMethod.DELETE }, produces = { "application/json" })
+@ResponseBody
+public GeneralResponse deleteCubeWithAll(@PathVariable String cubeName) {
+CubeInstance cube = getCube(cubeName);
+
+DataModelDesc model = cube.getModel();
+ProjectInstance project =
cubeService.getProjectManager().getProjectOfModel(model.getName());
+Set tables = model.getAllTables();
+
+//drop Cube
+t
[kylin] 29/41: KYLIN-4592 Make KylinConfig used by hbase coprocessor be cube segment level
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b7f0bd00a3fe4d5cab6ef718303ecd5637c3ae10
Author: Zhong, Yanghong
AuthorDate: Mon Jun 22 09:58:09 2020 +0800
KYLIN-4592 Make KylinConfig used by hbase coprocessor be cube segment level
---
.../org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index c694c0d..29dd141 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -221,9 +221,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
logger.debug("Submitting rpc to {} shards starting from shard {}, scan
range count {}", shardNum,
cuboidBaseShard, rawScans.size());
-// KylinConfig: use env instance instead of CubeSegment, because
KylinConfig will share among queries
-// for different cubes until redeployment of coprocessor jar.
-final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+final KylinConfig kylinConfig = cubeSeg.getConfig();
final boolean compressionResult = kylinConfig.getCompressionResult();
final boolean querySegmentCacheEnabled = isSegmentLevelCacheEnabled();
[kylin] 39/41: KYLIN-4550 Provide advanced refresh interface inside the refresh panel
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 75f76a633f224005c67bb86a6c2542f23640143c
Author: Julian Pan
AuthorDate: Fri Jun 26 16:13:09 2020 +0800
KYLIN-4550 Provide advanced refresh interface inside the refresh panel
---
webapp/app/js/controllers/cubes.js| 46
webapp/app/less/app.less | 15 +++
webapp/app/partials/jobs/job_refresh.html | 182 --
3 files changed, 185 insertions(+), 58 deletions(-)
diff --git a/webapp/app/js/controllers/cubes.js
b/webapp/app/js/controllers/cubes.js
index ddc037e..af529fd 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -850,6 +850,7 @@ var jobSubmitCtrl = function ($scope, $modalInstance,
CubeService, MessageServic
endTime: 0
};
$scope.message = "";
+ $scope.refreshType = 'normal';
var startTime;
if(cube.segments.length == 0){
startTime =
(!!cube.detail.partition_date_start)?cube.detail.partition_date_start:0;
@@ -950,6 +951,51 @@ var jobSubmitCtrl = function ($scope, $modalInstance,
CubeService, MessageServic
$scope.cancel = function () {
$modalInstance.dismiss('cancel');
};
+
+ $scope.getAdvRefreshTimeOptions = function(status) {
+if ('start' === status) {
+ var startTimeOptions = [];
+ var lastInd = $scope.cube.segments.length - 1;
+ angular.forEach($scope.cube.segments, function(segment, ind) {
+startTimeOptions.push(segment.date_range_start);
+if (lastInd == ind) {
+ startTimeOptions.push(segment.date_range_end);
+}
+ });
+ return startTimeOptions;
+} else if ('end' === status) {
+ var endTimeOptions = [];
+ angular.forEach($scope.cube.segments, function(segment, ind) {
+endTimeOptions.push(segment.date_range_end);
+ });
+ return endTimeOptions;
+}
+ };
+ $scope.advRefreshStartTimeOptions = $scope.getAdvRefreshTimeOptions('start');
+ $scope.advRefreshEndTimeOptions = $scope.getAdvRefreshTimeOptions('end');
+ $scope.endTimeTypeCustomize = false;
+
+ $scope.changeEndTimeDisplay = function() {
+$scope.endTimeTypeCustomize = !$scope.endTimeTypeCustomize;
+ };
+
+ $scope.setDateRange = function($view, $dates) {
+var minDate =
$scope.cube.segments[$scope.cube.segments.length-1].date_range_end;
+// var maxDate = moment().startOf($view).valueOf(); // Now
+angular.forEach($dates, function(date) {
+ var utcDateValue = date.utcDateValue;
+ date.selectable = utcDateValue >= minDate; // && utcDateValue <= maxDate;
+});
+ };
+
+ $scope.changeRefreshType = function (type) {
+$scope.refreshType = type;
+if (type ==='normal') {
+ $scope.jobBuildRequest.buildType = 'REFRESH';
+} else if (type === 'advance'){
+ $scope.jobBuildRequest.buildType = 'BUILD';
+}
+ }
};
diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less
index 708d430..b34fa67 100644
--- a/webapp/app/less/app.less
+++ b/webapp/app/less/app.less
@@ -1171,3 +1171,18 @@ tags-input .tags .tag-item {
.pagination{
cursor: pointer;
}
+/* adv refresh for cube action*/
+.adv-refresh-customized-on {
+ position: relative;
+ bottom: 10px;
+ color:#438eb9;
+ cursor:pointer;
+ font-size:18px;
+}
+.adv-refresh-customized-off {
+ position: relative;
+ top: 2px;
+ color:#438eb9;
+ cursor:pointer;
+ font-size:18px;
+}
diff --git a/webapp/app/partials/jobs/job_refresh.html
b/webapp/app/partials/jobs/job_refresh.html
index cf73353..9bfaf08 100644
--- a/webapp/app/partials/jobs/job_refresh.html
+++ b/webapp/app/partials/jobs/job_refresh.html
@@ -20,70 +20,136 @@
CUBE REFRESH CONFIRM
-
-
-
-
-
-No Segment to refresh.
-
-
-
-
-
-PARTITION DATE COLUMN
-
{{metaModel.model.partition_desc.partition_date_column}}
-
-
-REFRESH SEGMENT
-
-
-
-
-
-
-SEGMENT DETAIL
-
-
-
-Start Date (Include)
-{{selectedSegment.date_range_start
| reverseToGMT0}}
-
-
-End Date (Exclude)
-{{selectedSegment.date_ran
[kylin] 30/41: EBAY-KYLIN-1997 Add eBay specified migration tool cross DC
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit cb10d1891d067ee080a695e39a990bf8008dee42
Author: Zhong, Yanghong
AuthorDate: Mon Jun 22 10:28:20 2020 +0800
EBAY-KYLIN-1997 Add eBay specified migration tool cross DC
---
.../migration/ECubeMigrationCrossClusterCLI.java | 154 +++
.../kylin/tool/migration/EDistCpRestClient.java| 299 +
2 files changed, 453 insertions(+)
diff --git
a/tool/src/main/java/org/apache/kylin/tool/migration/ECubeMigrationCrossClusterCLI.java
b/tool/src/main/java/org/apache/kylin/tool/migration/ECubeMigrationCrossClusterCLI.java
new file mode 100644
index 000..9696867
--- /dev/null
+++
b/tool/src/main/java/org/apache/kylin/tool/migration/ECubeMigrationCrossClusterCLI.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool.migration;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Strings;
+
+public class ECubeMigrationCrossClusterCLI extends
CubeMigrationCrossClusterCLI {
+
+private static final Logger logger =
LoggerFactory.getLogger(ECubeMigrationCrossClusterCLI.class);
+
+@SuppressWarnings("static-access")
+public static final Option OPTION_DISTCP_DO_AS_NAME =
OptionBuilder.withArgName("distCpDoAsName").hasArg()
+.isRequired(true).withDescription("Specify do as name for
distCp").create("distCpDoAsName");
+
+@SuppressWarnings("static-access")
+public static final Option OPTION_DISTCP_REST_URL =
OptionBuilder.withArgName("distCpRestUrl").hasArg()
+.isRequired(true).withDescription("Specify rest url for
distCp").create("distCpRestUrl");
+
+@SuppressWarnings("static-access")
+public static final Option OPTION_DISTCP_TOKEN_URL =
OptionBuilder.withArgName("distCpTokenUrl").hasArg()
+.isRequired(true).withDescription("Specify rest url for asking
token for distCp").create("distCpTokenUrl");
+
+@SuppressWarnings("static-access")
+public static final Option OPTION_DISTCP_TOKEN_KEY =
OptionBuilder.withArgName("distCpTokenKey").hasArg()
+.isRequired(true).withDescription("Specify api key for asking
token").create("distCpTokenKey");
+
+@SuppressWarnings("static-access")
+public static final Option OPTION_DISTCP_TOKEN_SECRET =
OptionBuilder.withArgName("distCpTokenSecret").hasArg()
+.isRequired(true).withDescription("Specify api secret for asking
token").create("distCpTokenSecret");
+
+@SuppressWarnings("static-access")
+public static final Option OPTION_DISTCP_FILE_ATTR_KEPT =
OptionBuilder.withArgName("distCpFileAttrKept").hasArg()
+.isRequired(true).withDescription("Specify the distCp fileAttrKept
code").create("distCpFileAttrKept");
+
+@SuppressWarnings("static-access")
+public static final Option OPTION_DISTCP_JOB_MAX_DURATION =
OptionBuilder.withArgName("distCpJobMaxDuration")
+.hasArg().isRequired(false).withDescription("Specify distCp job
max duration")
+.create("distCpJobMaxDuration");
+@SuppressWarnings("static-access")
+public static final Option OPTION_DISTCP_JOB_STATUS_CHECK_INTERVAL =
OptionBuilder
+.withArgName("distCpJobCheckInterval").hasArg().isRequired(false)
+.withDescription("Specify distCp job status check interval(minute
unit)").create("distCpJobCheckInterval");
+
+@SuppressWarnings("static-access")
+publi
[kylin] 15/41: KYLIN-4535 Frontend support for query & storage trend
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 637d83f5d75dab7eb1ec72868290f8bb14a618d0
Author: Zhong, Yanghong
AuthorDate: Thu Jun 11 18:16:14 2020 +0800
KYLIN-4535 Frontend support for query & storage trend
---
.../kylin/rest/service/DashboardService.java | 1 +
.../apache/kylin/rest/service/QueryService.java| 1 -
.../apache/kylin/rest/util/SqlCreationUtil.java| 6 ++-
webapp/app/js/controllers/cube.js | 57 +++---
webapp/app/js/controllers/dashboard.js | 12 +
webapp/app/js/model/cubeConfig.js | 45 -
webapp/app/js/model/dashboardConfig.js | 5 +-
webapp/app/js/services/cubes.js| 3 +-
webapp/app/js/services/dashboard.js| 2 +-
webapp/app/less/app.less | 6 +--
webapp/app/partials/cubes/cube_detail.html | 18 +++
webapp/app/partials/dashboard/dashboard.html | 26 --
12 files changed, 158 insertions(+), 24 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
index 17d51aa..4792b1b 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
@@ -115,6 +115,7 @@ public class DashboardService extends BasicService {
jobMetrics.increase("avgJobBuildTime", getMetricValue(row.get(1)));
jobMetrics.increase("maxJobBuildTime", getMetricValue(row.get(2)));
jobMetrics.increase("minJobBuildTime", getMetricValue(row.get(3)));
+jobMetrics.increase("avgJobExpansionRate",
getMetricValue(row.get(4)));
}
return jobMetrics;
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 625b688..7d9b481 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -95,7 +95,6 @@ import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.querymeta.TableMetaWithType;
import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.query.QueryConnection;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.PushDownUtil;
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
b/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
index 4b6acf9..021bb8b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
@@ -95,7 +95,8 @@ public class SqlCreationUtil {
public static PrepareSqlRequest
createPrepareSqlRequestOfTotalJobMetrics(String projectName, String cubeName,
String startTime, String endTime) {
String[] measures = new String[] { JobMeasureEnum.JOB_COUNT.toSQL(),
JobMeasureEnum.AVG_JOB_BUILD_TIME.toSQL(),
-JobMeasureEnum.MAX_JOB_BUILD_TIME.toSQL(),
JobMeasureEnum.MIN_JOB_BUILD_TIME.toSQL() };
+JobMeasureEnum.MAX_JOB_BUILD_TIME.toSQL(),
JobMeasureEnum.MIN_JOB_BUILD_TIME.toSQL(),
+JobMeasureEnum.EXPANSION_RATE.toSQL() };
return createPrepareSqlRequestOfJobMetrics(projectName, cubeName,
startTime, endTime, null, measures);
}
@@ -373,7 +374,8 @@ public class SqlCreationUtil {
JOB_COUNT("count(*)"), //
AVG_JOB_BUILD_TIME("avg(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
MAX_JOB_BUILD_TIME("max(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
-MIN_JOB_BUILD_TIME("min(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")");
+MIN_JOB_BUILD_TIME("min(" +
JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
+EXPANSION_RATE(getExpansionRateMetric());
private final String sql;
diff --git a/webapp/app/js/controllers/cube.js
b/webapp/app/js/controllers/cube.js
index 5664690..89e0ea2 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -125,7 +125,7 @@ KylinApp.controller('CubeCtrl', function ($scope,
$rootScope, AccessService, Mes
if (!cube.currentCuboids) {
CubeService.getCurrentCuboids({cube
[kylin] 05/41: KYLIN-4530 Reduce duplication SQL log
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit bad7f0607f5ff436fbc6eb2b043306aa993ad59d
Author: Zhong, Yanghong
AuthorDate: Wed May 27 15:17:33 2020 +0800
KYLIN-4530 Reduce duplication SQL log
---
.../src/main/java/org/apache/kylin/rest/service/QueryService.java| 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index e731852..6c38f65 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -631,6 +631,7 @@ public class QueryService extends BasicService {
}
// Check whether duplicate query exists
+int lazyCount = 0;
while (response.isRunning()) {
// Wait at most one minute
if (System.currentTimeMillis() - response.getLazyQueryStartTime()
>= getConfig()
@@ -638,7 +639,9 @@ public class QueryService extends BasicService {
cache.evict(sqlRequest.getCacheKey());
return null;
}
-logger.info("Duplicated SQL request is running, waiting...");
+if (lazyCount++ % 100 == 0) {
+logger.info("Duplicated SQL request is running, waiting...");
+}
try {
Thread.sleep(100L);
} catch (InterruptedException e) {
[kylin] 04/41: KYLIN-4529 Add user behavior audit log
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1709e64fe1cfdb64a3f32bd0f78c05f60dff308d
Author: sanjulian
AuthorDate: Wed May 27 14:55:30 2020 +0800
KYLIN-4529 Add user behavior audit log
---
build/conf/kylin-server-log4j.properties | 11 +-
.../java/org/apache/kylin/job/JobInstance.java | 4 +
.../kylin/rest/util/UserBehaviorAuditLog.java | 94
.../kylin/rest/util/UserBehaviorAuditLogTest.java | 266 +
server/src/main/resources/kylinAuditLog.xml| 106
server/src/main/resources/kylinMetrics.xml | 2 +-
server/src/main/webapp/WEB-INF/web.xml | 1 +
7 files changed, 482 insertions(+), 2 deletions(-)
diff --git a/build/conf/kylin-server-log4j.properties
b/build/conf/kylin-server-log4j.properties
index bcaea65..7031b13 100644
--- a/build/conf/kylin-server-log4j.properties
+++ b/build/conf/kylin-server-log4j.properties
@@ -33,6 +33,12 @@ log4j.appender.realtime.Append=true
log4j.appender.realtime.MaxFileSize=268435456
log4j.appender.realtime.MaxBackupIndex=10
+log4j.appender.audit=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.audit.layout=org.apache.log4j.PatternLayout
+log4j.appender.audit.File=${catalina.home}/../logs/user-audit.log
+log4j.appender.audit.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}:%L :
%m%n
+log4j.appender.audit.Append=true
+
#overall config
log4j.rootLogger=INFO
log4j.logger.org.apache.kylin=DEBUG,file
@@ -45,4 +51,7 @@ log4j.logger.org.apache.kylin.job=DEBUG,realtime
log4j.logger.org.apache.kylin.rest.service.StreamingCoordinatorService=DEBUG,realtime
log4j.logger.org.apache.kylin.rest.service.StreamingV2Service=DEBUG,realtime
log4j.logger.org.apache.kylin.rest.controller.StreamingCoordinatorController=DEBUG,realtime
-log4j.logger.org.apache.kylin.rest.controller.StreamingV2Controller=DEBUG,realtime
\ No newline at end of file
+log4j.logger.org.apache.kylin.rest.controller.StreamingV2Controller=DEBUG,realtime
+
+#audit config
+log4j.logger.org.apache.kylin.rest.util.UserBehaviorAuditLog=INFO, audit
\ No newline at end of file
diff --git a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
index 1d78a25..a9b7759 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
@@ -533,4 +533,8 @@ public class JobInstance extends RootPersistentEntity
implements Comparable
this.lastModified ? 1 : 0;
}
+@Override
+public String toString() {
+return "Job[name=" + getName() + "]";
+}
}
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/util/UserBehaviorAuditLog.java
b/server-base/src/main/java/org/apache/kylin/rest/util/UserBehaviorAuditLog.java
new file mode 100644
index 000..874b62f
--- /dev/null
+++
b/server-base/src/main/java/org/apache/kylin/rest/util/UserBehaviorAuditLog.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.util;
+
+import java.util.Arrays;
+
+import org.apache.kylin.common.persistence.AclEntity;
+import org.aspectj.lang.JoinPoint;
+import org.aspectj.lang.annotation.Aspect;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.acls.model.Permission;
+import org.springframework.security.acls.model.Sid;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Component;
+
+@Component("auditLog")
+@Aspect
+public class UserBehaviorAuditLog {
+private static final Logger logger =
LoggerFactory.getLogger(UserBehaviorAuditLog.class);
+
+/* base audit */
+public void auditAllParamWithSignature(JoinPoint joinPoint) {
+logger.info("User: {} trigger {}, arguments: {}.",
+
SecurityContextHolder.getContext().getAuthentication().getName(),
joinPoint.getSignature().getName(),
+Arrays.toSt
[kylin] 02/41: KYLIN-4513 Introduce query call trace
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit eb70b5d52b7a149c8e6cee9cd0a7fdaf881504d1
Author: Zhong, Yanghong
AuthorDate: Mon May 25 19:23:19 2020 +0800
KYLIN-4513 Introduce query call trace
---
core-common/pom.xml| 10 ++
.../org/apache/kylin/common/KylinConfigBase.java | 7 ++
.../java/org/apache/kylin/common/QueryContext.java | 76 +-
.../kylin/common/tracer/JaegerTracerWrapper.java | 48 +
.../kylin/common/tracer/TracerConstants.java | 111 +
.../apache/kylin/common/tracer/TracerManager.java | 59 +++
.../apache/kylin/common/tracer/TracerWrapper.java | 58 +++
.../kylin/common/tracer/TracerManagerTest.java | 46 +
.../storage/gtrecord/GTCubeStorageQueryBase.java | 17 +++-
pom.xml| 6 ++
.../query/relnode/OLAPToEnumerableConverter.java | 9 +-
.../apache/kylin/rest/service/QueryService.java| 29 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java| 56 +--
.../stream/rpc/HttpStreamDataSearchClient.java | 19 ++--
14 files changed, 526 insertions(+), 25 deletions(-)
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 0549898..ca5cefa 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -63,6 +63,16 @@
org.apache.commons
commons-compress
+
+io.jaegertracing
+jaeger-client
+
+
+org.mockito
+mockito-core
+
+
+
com.google.code.findbugs
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a5af5e3..f2d423d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2670,4 +2670,11 @@ public abstract class KylinConfigBase implements
Serializable {
public int getDefaultTimeFilter() {
return Integer.parseInt(getOptional("kylin.web.default-time-filter",
"2"));
}
+
+//
+// Trace
+//
+public String getTracerCollectorEndpoint() {
+return getOptional("kylin.trace.collector-endpoint", "");
+}
}
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 0f6534f..df2fa4d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -29,12 +29,18 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.kylin.common.exceptions.KylinTimeoutException;
+import org.apache.kylin.common.tracer.TracerConstants.OperationEum;
+import org.apache.kylin.common.tracer.TracerConstants.TagEnum;
+import org.apache.kylin.common.tracer.TracerManager;
+import org.apache.kylin.common.tracer.TracerWrapper;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.shaded.com.google.common.collect.Lists;
import org.apache.kylin.shaded.com.google.common.collect.Maps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import io.opentracing.Span;
+
/**
* Holds per query information and statistics.
*/
@@ -48,14 +54,19 @@ public class QueryContext {
public interface QueryStopListener {
void stop(QueryContext query);
}
-
+
+private static final String SPAN_QUERY_SERVER =
KylinConfig.getInstanceFromEnv().getDeployEnv() + "-Query";
+
private final String queryId;
private final String project;
private final String sql;
private final String username;
private final int maxConnThreads;
private final long queryStartMillis;
-
+
+private final TracerWrapper tracer;
+private final Span rootSpan;
+
private Set groups;
private AtomicLong scannedRows = new AtomicLong();
private AtomicLong returnedRows = new AtomicLong();
@@ -84,6 +95,13 @@ public class QueryContext {
this.username = user;
this.maxConnThreads = maxConnThreads;
this.queryStartMillis = startMills;
+
+this.tracer = TracerManager.getTracerWrapper(SPAN_QUERY_SERVER);
+this.rootSpan = tracer.startSpan(OperationEum.MAIN, null);
+rootSpan.setTag(TagEnum.PROJECT.toString(), projectName);
+rootSpan.setTag(TagEnum.SQL.toString(), sql);
[kylin] 40/41: KYLIN-4551 Provide interfaces to transfer cube/model/project ownership
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c27577f98f3cd57a7d1af4ff09d38a88352566de
Author: Guangxu Cheng
AuthorDate: Wed Jun 24 12:48:18 2020 +0800
KYLIN-4551 Provide interfaces to transfer cube/model/project ownership
---
.../java/org/apache/kylin/cube/CubeManager.java| 10
.../kylin/metadata/project/ProjectManager.java | 12 +
.../kylin/rest/controller/CubeController.java | 32
.../kylin/rest/controller/ModelController.java | 50 +++
.../kylin/rest/controller/ProjectController.java | 57 ++
.../org/apache/kylin/rest/service/CubeService.java | 12 +
.../apache/kylin/rest/service/ProjectService.java | 9
.../kylin/rest/controller/CubeControllerTest.java | 25 +++---
.../rest/controller/ProjectControllerTest.java | 42
webapp/app/js/controllers/cube.js | 18 +++
webapp/app/js/controllers/projects.js | 23 -
webapp/app/js/services/cubes.js| 1 +
webapp/app/js/services/projects.js | 3 +-
webapp/app/partials/cubes/cube_detail.html | 19 +++-
webapp/app/partials/projects/project_detail.html | 21
15 files changed, 324 insertions(+), 10 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b255dd8..32b9692 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -334,6 +334,16 @@ public class CubeManager implements IRealizationProvider {
}
}
+public CubeInstance updateCubeOwner(CubeInstance cube, String owner)
throws IOException {
+try (AutoLock lock = cubeMapLock.lockForWrite()) {
+cube = cube.latestCopyForWrite(); // get a latest copy
+CubeUpdate update = new CubeUpdate(cube);
+update.setOwner(owner);
+ProjectManager.getInstance(config).touchProject(cube.getProject());
+return updateCube(update);
+}
+}
+
public CubeInstance updateCubeDropSegments(CubeInstance cube,
Collection segsToDrop)
throws IOException {
CubeSegment[] arr = segsToDrop.toArray(new
CubeSegment[segsToDrop.size()]);
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index ebcd45c..b3bcc0e 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -242,6 +242,18 @@ public class ProjectManager {
}
}
+// update project itself
+public ProjectInstance updateProjectOwner(ProjectInstance project, String
newOwner) throws IOException {
+try (AutoLock lock = prjMapLock.lockForWrite()) {
+project.setOwner(newOwner);
+
+if (project.getUuid() == null)
+project.updateRandomUuid();
+
+return save(project);
+}
+}
+
public void removeProjectLocal(String proj) {
try (AutoLock lock = prjMapLock.lockForWrite()) {
projectMap.removeLocal(proj);
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 52b7ae5..ba5a0b4 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -149,6 +149,10 @@ public class CubeController extends BasicController {
private TableService tableService;
@Autowired
+@Qualifier("validateUtil")
+private ValidateUtil validateUtil;
+
+@Autowired
private AclEvaluate aclEvaluate;
@RequestMapping(value = "{cubeName}/validate", method = RequestMethod.GET,
produces = { "application/json" })
@@ -281,6 +285,30 @@ public class CubeController extends BasicController {
}
}
+/**
+ * Update cube owner
+ *
+ * @param cubeName
+ * @param owner
+ * @throws IOException
+ */
+@RequestMapping(value = "/{cubeName}/owner", method = { RequestMethod.PUT
}, produces = {
+"application/json" })
+@ResponseBody
+public CubeInstance updateCubeOwner(@PathVariable String cubeName,
@RequestBody String owner) {
+checkCubeExists(cubeName);
+try {
+validateUtil.checkIdentifiersExists(owner, true);
+CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+
[kylin] 28/41: KYLIN-4589 Display ready segments only in the 'REFRESH SEGMENT' drop-down box
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701 in repository https://gitbox.apache.org/repos/asf/kylin.git commit ab09889d10bab85766c4ebb9409340fef3d174ee Author: sanjulian AuthorDate: Fri Jun 19 18:20:08 2020 +0800 KYLIN-4589 Display ready segments only in the 'REFRESH SEGMENT' drop-down box --- webapp/app/partials/jobs/job_refresh.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/app/partials/jobs/job_refresh.html b/webapp/app/partials/jobs/job_refresh.html index 83242ae..cf73353 100644 --- a/webapp/app/partials/jobs/job_refresh.html +++ b/webapp/app/partials/jobs/job_refresh.html @@ -39,7 +39,7 @@
[kylin] 26/41: EBAY-KYLIN-1442 Json ignore new added fields in SQLResponse
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6c7f4e0dd3672c41c7275f05d4146ade34e80855
Author: Zhong, Yanghong
AuthorDate: Fri Jun 19 18:00:28 2020 +0800
EBAY-KYLIN-1442 Json ignore new added fields in SQLResponse
---
.../src/main/java/org/apache/kylin/rest/response/SQLResponse.java | 3 +++
.../src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java | 3 +--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index aa9698d..85b7ab5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -169,6 +169,7 @@ public class SQLResponse implements Serializable {
return isPartial;
}
+@JsonIgnore
public boolean isPushDown() {
return queryPushDown;
}
@@ -181,6 +182,7 @@ public class SQLResponse implements Serializable {
this.totalScanCount = totalScanCount;
}
+@JsonIgnore
public long getTotalScanBytes() {
return totalScanBytes;
}
@@ -205,6 +207,7 @@ public class SQLResponse implements Serializable {
this.storageCacheUsed = storageCacheUsed;
}
+@JsonIgnore
public String getTraceUrl() {
return traceUrl;
}
diff --git
a/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
b/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
index 4055a27..6ca279f 100644
---
a/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
+++
b/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
@@ -33,8 +33,7 @@ public class SQLResponseTest {
@Test
public void testInterfaceConsistency() throws IOException {
String[] attrArray = new String[] { "columnMetas", "results", "cube",
"affectedRowCount", "isException",
-"exceptionMessage", "duration", "partial", "totalScanCount",
"hitExceptionCache", "storageCacheUsed",
-"pushDown", "traceUrl", "totalScanBytes" };
+"exceptionMessage", "duration", "partial", "totalScanCount",
"hitExceptionCache", "storageCacheUsed"};
SQLResponse sqlResponse = new SQLResponse(null, null, "learn_cube",
100, false, null, false, false);
String jsonStr = JsonUtil.writeValueAsString(sqlResponse);
[kylin] 23/41: KYLIN-4586 Add a configuration of maximum partition returned bytes
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8ba2aa76eda7dc7f60f17ffe85cfed84e96c6411
Author: Zhong, Yanghong
AuthorDate: Fri Jun 19 15:51:18 2020 +0800
KYLIN-4586 Add a configuration of maximum partition returned bytes
---
.../org/apache/kylin/common/KylinConfigBase.java | 6 +
.../hbase/cube/v2/CubeHBaseEndpointRPC.java| 1 +
.../v2/coprocessor/endpoint/CubeVisitService.java | 6 +
.../endpoint/generated/CubeVisitProtos.java| 212 -
.../coprocessor/endpoint/protobuf/CubeVisit.proto | 4 +-
5 files changed, 185 insertions(+), 44 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 416800b..2ac703d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1416,6 +1416,12 @@ public abstract class KylinConfigBase implements
Serializable {
return value > 0 ? value : Long.MAX_VALUE;
}
+public long getPartitionMaxReturnBytes() {
+long value = Long.parseLong(
+this.getOptional("kylin.storage.partition.max-return-bytes",
String.valueOf(3L * 1024 * 1024 * 1024)));
+return value > 0 ? value : Long.MAX_VALUE;
+}
+
public int getQueryCoprocessorTimeoutSeconds() {
return
Integer.parseInt(this.getOptional("kylin.storage.hbase.coprocessor-timeout-seconds",
"0"));
}
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 175bc37..c694c0d 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -284,6 +284,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
builder.setSpillEnabled(cubeSeg.getConfig().getQueryCoprocessorSpillEnabled());
builder.setMaxScanBytes(cubeSeg.getConfig().getPartitionMaxScanBytes());
+
builder.setMaxReturnBytes(cubeSeg.getConfig().getPartitionMaxReturnBytes());
builder.setIsExactAggregate(storageContext.isExactAggregation());
final String logHeader = String.format(Locale.ROOT, "",
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index e39e352..450f6a9 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -334,7 +334,13 @@ public class CubeVisitService extends
CubeVisitProtos.CubeVisitService implement
long finalRowCount = 0L;
try {
+long maxReturnBytes = request.hasMaxReturnBytes() ?
request.getMaxReturnBytes() : Long.MAX_VALUE;
for (GTRecord oneRecord : finalScanner) {
+int outputSize = outputStream.size();
+if (outputSize > maxReturnBytes) {
+throw new ResourceLimitExceededException("return row
size exceeds threshold " + outputSize);
+}
+
buffer.clear();
try {
oneRecord.exportColumns(scanReq.getColumns(), buffer);
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index 2da6020..1b96a75 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -136,13 +136,31 @@ public final class CubeVisitProtos {
*/
long getMaxScanBytes();
-// optional bool isExactAggregate = 9 [default = false];
+// optional int64 maxReturnBytes = 9;
/**
- * optional bool isExactAggregate = 9 [default = false];
+ * optional int64 maxReturnBytes = 9;
+ *
+ *
+ * max return bytes
+ *
+ */
+boolean hasMaxReturnBytes();
+/**
+ * optional int64 maxReturnBytes = 9;
+ *
+ *
+ * max return bytes
+ *
+ */
+long getMaxR
[kylin] 03/41: KYLIN-4528 Bad hbase region & irregular query detection
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f6e26d11b623b93a6de7b2ccb35f90b0e66f91e1
Author: Zhong, Yanghong
AuthorDate: Wed May 27 14:18:58 2020 +0800
KYLIN-4528 Bad hbase region & irregular query detection
---
.../org/apache/kylin/common/KylinConfigBase.java | 44 --
.../java/org/apache/kylin/common/QueryContext.java | 24 +++-
.../kylin/common/util/MailTemplateProvider.java| 5 +
.../kylin/rest/service/MigrationService.java | 4 +-
.../kylin/rest/util/MailNotificationUtil.java | 6 -
.../hbase/cube/v2/CubeHBaseEndpointRPC.java| 67 +
.../storage/hbase/cube/v2/HBaseRPCHealthCheck.java | 152 +
.../hbase/cube/v2/HBaseRPCHealthCheckTest.java | 63 +
8 files changed, 342 insertions(+), 23 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f2d423d..93cdcf9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -18,20 +18,6 @@
package org.apache.kylin.common;
-import org.apache.kylin.shaded.com.google.common.collect.Lists;
-import org.apache.kylin.shaded.com.google.common.collect.Maps;
-import org.apache.kylin.shaded.com.google.common.collect.Sets;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.text.StrSubstitutor;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.lock.DistributedLockFactory;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
@@ -49,6 +35,20 @@ import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.text.StrSubstitutor;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.lock.DistributedLockFactory;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.shaded.com.google.common.collect.Lists;
+import org.apache.kylin.shaded.com.google.common.collect.Maps;
+import org.apache.kylin.shaded.com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* An abstract class to encapsulate access to a set of 'properties'.
* Subclass can override methods in this class to extend the content of the
'properties',
@@ -1497,6 +1497,22 @@ public abstract class KylinConfigBase implements
Serializable {
return
Integer.parseInt(getOptional("kylin.storage.hbase.max-hconnection-threads-per-project",
"800"));
}
+public String[] getHBaseAdminDls() {
+return
getOptionalStringArray("kylin.storage.hbase.notification-admin-emails", null);
+}
+
+public boolean isHBaseBadRegionDetectEnabled() {
+return
Boolean.parseBoolean(getOptional("kylin.storage.hbase.bad-region-detect-enabled",
"true"));
+}
+
+public int getHBaseBadRegionDetectThreshold() {
+return
Integer.parseInt(getOptional("kylin.storage.hbase.bad-region-detect-threshold",
"15"));
+}
+
+public int getHBaseBadRegionDetectMultiplier() {
+return
Integer.parseInt(getOptional("kylin.storage.hbase.bad-region-detect-multiplier",
"10"));
+}
+
//
// ENGINE.MR
//
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index df2fa4d..8fed1f5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -82,6 +82,8 @@ public class QueryContext {
private List rpcStatisticsList =
Lists.newCopyOnWriteArrayList();
private Map
cubeSegmentStatisticsResultMap = Maps.newConcurrentMap();
+private ConcurrentMap isAlreadyAlert =
Maps.newConcurrentMap();
+
private ExecutorService connPool;
QueryContext(String projectName, String sql, String user, int
maxConnThreads) {
@@ -127,10 +129,18 @@ public class QueryContext {
}
}
+public boolean isAlreadyAlert() {
+re
[kylin] 22/41: KYLIN-4584 Add one option backup-hdfs for metastore.sh to backup metadata to hdfs
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a47ce3b49be01fb0879ca4e896b4ff460af43a0f
Author: Zhong, Yanghong
AuthorDate: Fri Jun 19 15:03:35 2020 +0800
KYLIN-4584 Add one option backup-hdfs for metastore.sh to backup metadata
to hdfs
---
build/bin/metastore.sh | 58 ++
1 file changed, 58 insertions(+)
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index d86aac0..ba0ab75 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -40,6 +40,64 @@ then
echo "metadata store backed up to ${_file}"
+elif [ "$1" == "backup-hdfs" ]
+then
+
+BACKUP_HOME="$2/meta_backups"
+
+if [ "$#" -eq 3 ]
+then
+HDFS_META_HOME=$3
+else
+HDFS_META_HOME=$(grep kylin.env.hdfs-working-dir
${KYLIN_HOME}/conf/kylin.properties | awk -F= '{print $2}')
+fi
+
+if [ -z "$HDFS_META_HOME" ]
+then
+quit "HDFS_META_HOME should be defined!!!"
+else
+echo "HDFS_META_HOME is $HDFS_META_HOME"
+fi
+
+_now=$(date +"%Y_%m_%d_%H_%M_%S")
+_file="meta_${_now}"
+_tar="${_file}.tar.gz"
+_cur_meta_dir="${BACKUP_HOME}/${_file}"
+mkdir -p ${_cur_meta_dir}
+
+echo "Starting backup to ${_cur_meta_dir}"
+${KYLIN_HOME}/bin/kylin.sh
org.apache.kylin.common.persistence.ResourceTool download ${_cur_meta_dir}
+echo "metadata store backed up to ${_cur_meta_dir}"
+
+_cur_meta_tar="${BACKUP_HOME}/${_tar}"
+tar -czvf ${_cur_meta_tar} ${_cur_meta_dir}
+
+HDFS_BACKUP_HOME="${HDFS_META_HOME}/meta_backups"
+hadoop fs -mkdir -p ${HDFS_BACKUP_HOME}
+
+echo "Upload metadata ${_cur_meta_tar} to hdfs ${HDFS_BACKUP_HOME}"
+hadoop fs -copyFromLocal ${_cur_meta_tar} ${HDFS_BACKUP_HOME}
+echo "metadata store backed up to hdfs ${HDFS_BACKUP_HOME}/${_tar}"
+
+rm -f ${_cur_meta_tar}
+rm -rf ${_cur_meta_dir}
+echo "remove local meta ${_cur_meta_dir}"
+
+# keep latest n tars
+_n_keep=5
+
+_n_exists=`hadoop fs -ls ${HDFS_BACKUP_HOME} | wc -l`
+
+if ((${_n_exists} > ${_n_keep}))
+then
+_n_delete=$((${_n_exists} - ${_n_keep}))
+for _hdfs_tar in `hadoop fs -ls ${HDFS_BACKUP_HOME} | sort -k6,7 -r |
awk '{print $8}' | tail -${_n_delete}`
+do
+ echo "Going to delete hdfs file ${_hdfs_tar}"
+ hadoop fs -rm ${_hdfs_tar}
+done
+fi
+
elif [ "$1" == "fetch" ]
then
[kylin] 35/41: KYLIN-3140 Auto merge jobs should not be counted in kylin.cube.max-building-segments
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 23bc5ac8949e45a088a1891f51e22476801f1e44
Author: Wang gang
AuthorDate: Wed Jun 24 11:18:21 2020 +0800
KYLIN-3140 Auto merge jobs should not be counted in
kylin.cube.max-building-segments
---
.../java/org/apache/kylin/cube/CubeManager.java| 9 +++
.../kylin/rest/controller/CubeController.java | 13 -
.../org/apache/kylin/rest/service/CubeService.java | 3 +
.../org/apache/kylin/rest/service/JobService.java | 39 +
.../apache/kylin/rest/service/JobServiceTest.java | 66 --
.../org/apache/kylin/tool/job/CubeBuildingCLI.java | 13 +++--
6 files changed, 119 insertions(+), 24 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index a61a409..b255dd8 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -349,6 +349,15 @@ public class CubeManager implements IRealizationProvider {
}
}
+public CubeInstance updateCubeSegments(CubeInstance cube, CubeSegment...
segsToUpdate) throws IOException {
+try (AutoLock lock = cubeMapLock.lockForWrite()) {
+cube = cube.latestCopyForWrite(); // get a latest copy
+CubeUpdate update = new CubeUpdate(cube);
+update.setToUpdateSegs(segsToUpdate);
+return updateCube(update);
+}
+}
+
public CubeInstance dropOptmizingSegments(CubeInstance cube,
CubeSegment... segsToDrop) throws IOException {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
cube = cube.latestCopyForWrite(); // get a latest copy
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index baedcd9..52b7ae5 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -70,7 +70,6 @@ import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.exception.NotFoundException;
-import org.apache.kylin.rest.exception.TooManyRequestException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.CubeRequest;
@@ -419,7 +418,6 @@ public class CubeController extends BasicController {
String submitter =
SecurityContextHolder.getContext().getAuthentication().getName();
CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
-checkBuildingSegment(cube);
return jobService.submitJob(cube, tsRange, segRange,
sourcePartitionOffsetStart, sourcePartitionOffsetEnd,
CubeBuildTypeEnum.valueOf(buildType), force, submitter,
priorityOffset);
} catch (Throwable e) {
@@ -1244,17 +1242,6 @@ public class CubeController extends BasicController {
}
}
-private void checkBuildingSegment(CubeInstance cube) {
-checkBuildingSegment(cube, cube.getConfig().getMaxBuildingSegments());
-}
-
-private void checkBuildingSegment(CubeInstance cube, int maxBuildingSeg) {
-if (cube.getBuildingSegments().size() >= maxBuildingSeg) {
-throw new TooManyRequestException(
-"There is already " + cube.getBuildingSegments().size() +
" building segment; ");
-}
-}
-
public void setCubeService(CubeService cubeService) {
this.cubeService = cubeService;
}
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 4793510..1a4b468 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -782,6 +782,9 @@ public class CubeService extends BasicService implements
InitializingBean {
logger.info("Will submit merge job on " + newSeg);
DefaultChainedExecutable job =
EngineFactory.createBatchMergeJob(newSeg, "SYSTEM");
getExecutableManager().addJob(job);
+
+newSeg.setLastBuildJobID(job.getId());
+getCubeManager().updateCubeSegments(cube, newSeg);
return job.getId();
} else {
logger.info("Not ready for merge on cube " + cubeName);
diff --git
a/se
[kylin] 31/41: KYLIN-4594 Send out notification email when triggering cube building job and finding out table schema changes
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit db2a22c56705ade98f6b05d3851ca2b9db61ed72
Author: Wang gang
AuthorDate: Mon Jun 22 12:20:26 2020 +0800
KYLIN-4594 Send out notification email when triggering cube building job
and finding out table schema changes
---
.../org/apache/kylin/rest/service/JobService.java | 49 +++-
.../rest/service/TableSchemaUpdateChecker.java | 139 +--
.../mail_templates/HIVE_SCHEMA_CHANGED.ftl | 264 +
3 files changed, 417 insertions(+), 35 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index a214c12..f9880bd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -33,7 +33,6 @@ import java.util.TimeZone;
import javax.annotation.Nullable;
-import com.google.common.collect.Maps;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.lock.DistributedLock;
@@ -51,8 +50,8 @@ import
org.apache.kylin.engine.mr.BatchOptimizeJobCheckpointBuilder;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.LookupSnapshotBuildJob;
import org.apache.kylin.engine.mr.LookupSnapshotJobBuilder;
-import org.apache.kylin.engine.mr.common.CubeJobLockUtil;
import org.apache.kylin.engine.mr.StreamingCubingEngine;
+import org.apache.kylin.engine.mr.common.CubeJobLockUtil;
import org.apache.kylin.engine.mr.common.JobInfoConverter;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.JobInstance;
@@ -70,21 +69,31 @@ import org.apache.kylin.job.execution.CheckpointExecutable;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.Output;
+import org.apache.kylin.job.lock.zookeeper.ZookeeperJobLock;
import org.apache.kylin.job.util.MailNotificationUtil;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
+import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.util.AclEvaluate;
+import org.apache.kylin.shaded.com.google.common.base.Function;
+import org.apache.kylin.shaded.com.google.common.base.Predicate;
+import org.apache.kylin.shaded.com.google.common.base.Predicates;
+import org.apache.kylin.shaded.com.google.common.collect.FluentIterable;
+import org.apache.kylin.shaded.com.google.common.collect.Lists;
+import org.apache.kylin.shaded.com.google.common.collect.Maps;
+import org.apache.kylin.shaded.com.google.common.collect.Sets;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceManager;
import org.apache.kylin.source.SourcePartition;
-import org.apache.kylin.job.lock.zookeeper.ZookeeperJobLock;
import org.apache.kylin.source.hive.MRHiveDictUtil;
import org.apache.kylin.stream.coordinator.Coordinator;
import org.apache.kylin.stream.core.model.SegmentBuildState;
@@ -92,17 +101,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
-import org.apache.kylin.shaded.com.google.common.base.Function;
-import org.apache.kylin.shaded.com.google.common.base.Predicate;
-import org.apache.kylin.shaded.com.google.common.base.Predicates;
-import org.apache.kylin.shaded.com.google.common.collect.FluentIterable;
-import org.apache.kylin.shaded.com.google.common.collect.Lists;
-import org.apache.kylin.shaded.com.google.common.collect.Sets;
-
/**
* @author ysong1
*/
@@ -116,6 +119,10 @@ public class JobService extends BasicService implements
InitializingBean {
@Autowired
private AclEvaluate aclEvaluate;
+@Autowired
+@Qualifier("tableService")
+private TableService tableService;
+
/*
* (non-Javadoc)
*
@@ -24
[kylin] 37/41: KYLIN-4602 Fix query display format not correct in insight table
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 69fd35b4e121fd27bb3089b064e238510ba300eb
Author: sanjulian
AuthorDate: Sun Jun 28 13:43:44 2020 +0800
KYLIN-4602 Fix query display format not correct in insight table
---
webapp/app/js/controllers/query.js | 2 +-
webapp/app/js/utils/utils.js | 6 +-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/webapp/app/js/controllers/query.js
b/webapp/app/js/controllers/query.js
index 6601666..d75f3a7 100644
--- a/webapp/app/js/controllers/query.js
+++ b/webapp/app/js/controllers/query.js
@@ -218,7 +218,7 @@ KylinApp
}
angular.forEach(oneQuery.result.data,function(row,index){
angular.forEach(row,function(column,value){
-var float =VdmUtil.SCToFloat(column);
+var float =VdmUtil.SCToFloat(column,
result.columnMetas.find(function(columnMeta){ return columnMeta.label ===
value}).columnTypeName);
if (float!=""){
oneQuery.result.data[index][value]=parseFloat(float);
}
diff --git a/webapp/app/js/utils/utils.js b/webapp/app/js/utils/utils.js
index 9c92a02..31e4739 100644
--- a/webapp/app/js/utils/utils.js
+++ b/webapp/app/js/utils/utils.js
@@ -65,8 +65,12 @@ KylinApp.factory('VdmUtil', function ($modal, $timeout,
$location, $anchorScroll
return fmt;
},
-SCToFloat:function(data){
+SCToFloat:function(data, type){
var resultValue = "";
+ var columnType = ['TINYINT', 'SMALLINT', 'INT', 'BIGINT', 'INTEGER',
'DECIMAL', 'DOUBLE', 'FLOAT'];
+ if (!!type && !columnType.includes(type)) {
+return resultValue;
+ }
if (data&('E') != -1){
var regExp = new RegExp('^((\\d+.?\\d+)[Ee]{1}(\\d+))$', 'ig');
var result = regExp.exec(data);
[kylin] 32/41: KYLIN-4596 Consider CubeInstance reinitialization time when querying system cubes
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 574f5c0f918266bc8ae42f7ea065c458602d26ab
Author: Zhong, Yanghong
AuthorDate: Mon Jun 22 14:01:12 2020 +0800
KYLIN-4596 Consider CubeInstance reinitialization time when querying system
cubes
---
.../apache/kylin/rest/util/SqlCreationUtil.java| 84 ++
1 file changed, 69 insertions(+), 15 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
b/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
index 021bb8b..505cf84 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.util;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -27,6 +28,7 @@ import java.util.stream.Collectors;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.metrics.lib.impl.RecordEventTimeDetail;
import org.apache.kylin.metrics.lib.impl.TimePropertyEnum;
@@ -73,7 +75,7 @@ public class SqlCreationUtil {
String table = MetricsManager
.getSystemTableFromSubject(KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQuery());
-Map filterMap = Maps.newHashMap();
+Map> filterMap = Maps.newHashMap();
if (StringUtil.isEmpty(projectName)) {
addFilter(filterMap, QueryPropertyEnum.PROJECT.toString(), "<>",
MetricsManager.SYSTEM_PROJECT,
String.class.getName());
@@ -83,6 +85,7 @@ public class SqlCreationUtil {
}
if (!StringUtil.isEmpty(cubeName)) {
addFilter(filterMap, QueryPropertyEnum.REALIZATION.toString(),
"=", cubeName, String.class.getName());
+addCubeCreationUTCFilter(filterMap, cubeName);
}
addFilter(filterMap, QueryPropertyEnum.EXCEPTION.toString(), "=",
"NULL", String.class.getName());
@@ -106,7 +109,7 @@ public class SqlCreationUtil {
String table = MetricsManager
.getSystemTableFromSubject(KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectJob());
-Map filterMap = Maps.newHashMap();
+Map> filterMap = Maps.newHashMap();
if (StringUtil.isEmpty(projectName)) {
addFilter(filterMap, JobPropertyEnum.PROJECT.toString(), "<>",
MetricsManager.SYSTEM_PROJECT,
String.class.getName());
@@ -116,6 +119,7 @@ public class SqlCreationUtil {
}
if (!StringUtil.isEmpty(cubeName)) {
addFilter(filterMap, JobPropertyEnum.CUBE.toString(), "IN",
cubeName, String.class.getName());
+addCubeCreationUTCFilter(filterMap, cubeName);
}
addFilter(filterMap, TimePropertyEnum.DAY_DATE.toString(), ">=",
startTime, String.class.getName());
@@ -136,8 +140,9 @@ public class SqlCreationUtil {
String[] measures = new String[1];
measures[0] = "sum(" + QueryCubePropertyEnum.WEIGHT_PER_HIT.toString()
+ ")";
-Map filterMap = Maps.newHashMap();
+Map> filterMap = Maps.newHashMap();
addFilter(filterMap, QueryCubePropertyEnum.CUBE.toString(), "=",
cubeName, String.class.getName());
+addCubeCreationUTCFilter(filterMap, cubeName);
return createPrepareSqlRequest(table, dimensions, measures, filterMap);
}
@@ -154,8 +159,9 @@ public class SqlCreationUtil {
measures[0] = "avg(" + QueryCubePropertyEnum.AGGR_COUNT.toString() +
")";
measures[1] = "avg(" + QueryCubePropertyEnum.RETURN_COUNT.toString() +
")";
-Map filterMap = Maps.newHashMap();
+Map> filterMap = Maps.newHashMap();
addFilter(filterMap, QueryCubePropertyEnum.CUBE.toString(), "=",
cubeName, String.class.getName());
+addCubeCreationUTCFilter(filterMap, cubeName);
return createPrepareSqlRequest(table, dimensions, measures, filterMap);
}
@@ -170,8 +176,9 @@ public class SqlCreationUtil {
String[] measures = new String[1];
measures[0] = "sum(" + QueryCubePropertyEnum.WEIGHT_PER_HIT.toString()
+ ")";
-Map filterMap = Maps.newHashMap();
+Map> filterMap = Maps.newHashMap();
addFilter(filterMap, QueryCubePropertyEnum.CUBE.toString(), "=",
cubeName, String.class.getName());
+addCubeCreationUTCF
[kylin] 16/41: KYLIN-4565 Add cli tool to export base cuboid data
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 415a43a4c9aa9cf583d7ffa943ab9abec57cc2af
Author: Zhong, Yanghong
AuthorDate: Mon Jun 15 14:52:53 2020 +0800
KYLIN-4565 Add cli tool to export base cuboid data
---
.../kylin/engine/mr/common/AbstractHadoopJob.java | 5 +-
.../kylin/engine/mr/common/BatchConstants.java | 3 +
.../kylin/engine/mr/steps/ExportBaseCuboidJob.java | 115
.../engine/mr/steps/ExportBaseCuboidMapper.java| 123 +
.../apache/kylin/tool/job/BaseCuboidExportCLI.java | 153 +
5 files changed, 398 insertions(+), 1 deletion(-)
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index c463014..c7976df 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -139,7 +139,10 @@ public abstract class AbstractHadoopJob extends Configured
implements Tool {
.hasArg().isRequired(true).withDescription("HDFS metadata
url").create(BatchConstants.ARG_META_URL);
public static final Option OPTION_HBASE_CONF_PATH =
OptionBuilder.withArgName(BatchConstants.ARG_HBASE_CONF_PATH).hasArg()
.isRequired(true).withDescription("HBase config file
path").create(BatchConstants.ARG_HBASE_CONF_PATH);
-
+protected static final Option OPTION_DATA_EXPORT_DELIMITER = OptionBuilder
+
.withArgName(BatchConstants.ARG_DATA_EXPORT_DELIMITER).hasArg().isRequired(false)
+.withDescription("Data export delimiter for
columns").create(BatchConstants.ARG_DATA_EXPORT_DELIMITER);
+
private static final String MAP_REDUCE_CLASSPATH =
"mapreduce.application.classpath";
private static final Map kylinConfigCache =
Maps.newConcurrentMap();
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index d0e2936..2f153c1 100644
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -81,6 +81,8 @@ public interface BatchConstants {
String CFG_CONVERGE_CUBOID_PARTITION_PARAM =
"converge.cuboid.partition.param";
+String CFG_DATA_EXPORT_DELIMITER = "data.export.delimiter";
+
/**
* command line ARGuments
*/
@@ -116,6 +118,7 @@ public interface BatchConstants {
String ARG_BASE64_ENCODED_SQL = "base64EncodedSql";
String ARG_GLOBAL_DIC_PART_REDUCE_STATS = "global_dict_part_reduce_stats";
String ARG_GLOBAL_DIC_MAX_DISTINCT_COUNT =
"global_dict_max_distinct_count";
+String ARG_DATA_EXPORT_DELIMITER = "dataExportDelimiter";
/**
* logger and counter
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ExportBaseCuboidJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ExportBaseCuboidJob.java
new file mode 100644
index 000..1a5ee2d
--- /dev/null
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/ExportBaseCuboidJob.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.steps;
+
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.
[kylin] 34/41: EBAY-KYLIN-2346 Deal with tls for Windows ODBC
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b0ef8997219e3a363aec863ed1595709b60371a4
Author: Zhong, Yanghong
AuthorDate: Mon Jun 22 15:34:32 2020 +0800
EBAY-KYLIN-2346 Deal with tls for Windows ODBC
---
odbc/Common/Common.vcxproj| 34 +--
odbc/Common/JsonConverter.cpp | 12 ++--
odbc/Common/QueryCache.cpp| 1 -
odbc/Common/REST.cpp | 131 +-
odbc/Common/StringUtils.cpp | 6 ++
odbc/Common/StringUtils.h | 1 +
odbc/Driver/KO_INFO.CPP | 2 +-
odbc/Driver/driver.vcxproj| 45 ---
odbc/README.md| 5 +-
odbc/TestDLL/TestDLL.vcxproj | 7 +--
10 files changed, 140 insertions(+), 104 deletions(-)
diff --git a/odbc/Common/Common.vcxproj b/odbc/Common/Common.vcxproj
index a9f5cc6b..6087c87 100644
--- a/odbc/Common/Common.vcxproj
+++ b/odbc/Common/Common.vcxproj
@@ -16,7 +16,6 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-
http://schemas.microsoft.com/developer/msbuild/2003;>
@@ -40,31 +39,32 @@
{6EE1C577-1826-4EA2-86AC-FE8047D396A9}
Win32Proj
Common
+10.0
StaticLibrary
true
-v110
+v142
Unicode
StaticLibrary
true
-v110
+v142
Unicode
StaticLibrary
false
-v110
+v142
true
Unicode
StaticLibrary
false
-v110
+v142
true
Unicode
@@ -85,25 +85,25 @@
-$(CPPREST_HOME)\Release\include;C:\Program Files (x86)\Visual
Leak Detector\include;$(IncludePath)
-
$(CPPREST_HOME)\Binaries\$(Platform)\$(Configuration);C:\Program
Files (x86)\Visual Leak
Detector\lib\Win$(PlatformArchitecture);$(LibraryPath)
+
$(CPPREST_HOME)\Release\include;$(vcpkgRoot)\$(Platform)-windows-static\include;C:\Program
Files (x86)\Visual Leak Detector\include;$(IncludePath)
+
$(CPPREST_HOME)\Binaries\$(Platform)\$(Configuration);$(vcpkgRoot)\$(Platform)-windows-static\lib;C:\Program
Files (x86)\Visual Leak
Detector\lib\Win$(PlatformArchitecture);$(LibraryPath)
$(Platform)\$(Configuration)\
$(Platform)\$(Configuration)\
-$(CPPREST_HOME)\Release\include;C:\Program Files (x86)\Visual
Leak Detector\include;$(IncludePath)
-
$(CPPREST_HOME)\Binaries\$(Platform)\$(Configuration);C:\Program
Files (x86)\Visual Leak
Detector\lib\Win$(PlatformArchitecture);$(LibraryPath)
+
$(CPPREST_HOME)\Release\include;$(vcpkgRoot)\$(Platform)-windows-static\include;C:\Program
Files (x86)\Visual Leak Detector\include;$(IncludePath)
+
$(CPPREST_HOME)\Binaries\$(Platform)\$(Configuration);$(vcpkgRoot)\$(Platform)-windows-static\lib;C:\Program
Files (x86)\Visual Leak
Detector\lib\Win$(PlatformArchitecture);$(LibraryPath)
$(Platform)\$(Configuration)\
-$(CPPREST_HOME)\Release\include;C:\Program Files (x86)\Visual
Leak Detector\include;$(IncludePath)
-
$(CPPREST_HOME)\Binaries\$(Platform)\$(Configuration);C:\Program
Files (x86)\Visual Leak
Detector\lib\Win$(PlatformArchitecture);$(LibraryPath)
+
$(CPPREST_HOME)\Release\include;$(vcpkgRoot)\$(Platform)-windows-static\include;C:\Program
Files (x86)\Visual Leak Detector\include;$(IncludePath)
+
$(CPPREST_HOME)\Binaries\$(Platform)\$(Configuration);$(vcpkgRoot)\$(Platform)-windows-static\lib;C:\Program
Files (x86)\Visual Leak
Detector\lib\Win$(PlatformArchitecture);$(LibraryPath)
$(Platform)\$(Configuration)\
$(Platform)\$(Configuration)\
-$(CPPREST_HOME)\Release\include;C:\Program Files (x86)\Visual
Leak Detector\include;$(IncludePath)
-
$(CPPREST_HOME)\Binaries\$(Platform)\$(Configuration);C:\Program
Files (x86)\Visual Leak
Detector\lib\Win$(PlatformArchitecture);$(LibraryPath)
+
$(CPPREST_HOME)\Release\include;$(vcpkgRoot)\$(Platform)-windows-static\include;C:\Program
Files (x86)\Visual Leak Detector\include;$(IncludePath)
+
$(CPPREST_HOME)\Binaries\$(Platform)\$(Configuration);$(vcpkgRoot)\$(Platform)-windows-static\lib;C:\Program
Files (x86)\Visual Leak
Detector\lib\Win$(PlatformArchitecture);$(LibraryPath)
$(Platform)\$(Configuration)\
@@ -112,7 +112,7 @@
Level3
Disabled
-
_CRT_SECURE_NO_WARNINGS;ASMV;ASMINF;ZLIB_WINAPI;WIN32;_DEBUG;_LIB;_NO_ASYNCRTIMP;%(PreprocessorDefinitions)
+
CPPREST_FORCE_HTTP_CLIENT_ASIO;_CRT_SECURE_NO_WARNINGS;ASMV;ASMINF;ZLIB_WINAPI;WIN32;_DEBUG;_LIB;_NO_ASYNCRTIMP;%(PreprocessorDefinitions)
MultiThreadedDebug
$(ZLIB_HOME);%(AdditionalIncludeDirectories)
@@ -133,7 +133,7 @@
Level3
Disabled
-
_CRT_SECURE_NO_WARNINGS;ASMV;ASMINF;ZLIB_WINAPI;WIN32;NDEBUG;_DEBUG;_LIB;_NO_ASYNCRTIMP;%(PreprocessorDefinitions)
+
CPPREST_FORCE_HTTP_CLIEN
[kylin] 33/41: EBAY-KYLIN-2371 Don't use DictionaryEnumerator for foreign key only query for old cubes whose dictionaries were built from lookup tables
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d0e352bb729cfac4ed1371eb50e84dbea34b19f1
Author: Zhong, Yanghong
AuthorDate: Mon Jun 22 14:16:48 2020 +0800
EBAY-KYLIN-2371 Don't use DictionaryEnumerator for foreign key only query
for old cubes whose dictionaries were built from lookup tables
---
.../java/org/apache/kylin/common/KylinConfigBase.java | 4
.../org/apache/kylin/metadata/model/DataModelDesc.java| 15 +++
.../kylin/query/enumerator/DictionaryEnumerator.java | 8
3 files changed, 27 insertions(+)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2ac703d..f9bafee 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1838,6 +1838,10 @@ public abstract class KylinConfigBase implements
Serializable {
return Lists.newArrayList(rules.split(","));
}
+public boolean isDictionaryEnumeratorEnabledForForeignKey() {
+return
Boolean.valueOf(getOptional("kylin.query.enable-dict-enumerator-for-foreign-key",
"false"));
+}
+
// check KYLIN-3358, need deploy coprocessor if enabled
// finally should be deprecated
public boolean isDynamicColumnEnabled() {
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index d126d3d..9a3e0e9 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -225,6 +225,21 @@ public class DataModelDesc extends RootPersistentEntity {
return result;
}
+// check whether a col in fact table is part of FK
+public boolean isBelongToFK(TblColRef col) {
+if (!isFactTable(col.getTableRef())) {
+return false;
+}
+for (TableRef lookup : getLookupTables()) {
+JoinDesc lookupJoin = getJoinByPKSide(lookup);
+int find = ArrayUtils.indexOf(lookupJoin.getForeignKeyColumns(),
col);
+if (find < 0)
+continue;
+return true;
+}
+return false;
+}
+
public boolean isLookupTable(TableRef t) {
if (t == null)
return false;
diff --git
a/query/src/main/java/org/apache/kylin/query/enumerator/DictionaryEnumerator.java
b/query/src/main/java/org/apache/kylin/query/enumerator/DictionaryEnumerator.java
index 6af65ee..4725ec7 100644
---
a/query/src/main/java/org/apache/kylin/query/enumerator/DictionaryEnumerator.java
+++
b/query/src/main/java/org/apache/kylin/query/enumerator/DictionaryEnumerator.java
@@ -70,8 +70,15 @@ public class DictionaryEnumerator implements
Enumerator {
return false;
}
+IRealization realization = olapContext.realization;
TblColRef dictCol = olapContext.allColumns.iterator().next();
+if
(!realization.getConfig().isDictionaryEnumeratorEnabledForForeignKey()
+&& realization.getModel().isBelongToFK(dictCol)) {
+logger.info("DictionaryEnumerator not enabled due to col {}
belongs to FK", dictCol);
+return false;
+}
if (!ifColumnHaveDictionary(dictCol, olapContext.realization, true)) {
+logger.info("DictionaryEnumerator not enabled due to col {} is not
dictionary encoded", dictCol);
return false;
}
return true;
@@ -138,5 +145,6 @@ public class DictionaryEnumerator implements
Enumerator {
@Override
public void close() {
+logger.info("DictionaryEnumerator closed");
}
}
\ No newline at end of file
[kylin] 07/41: KYLIN-4536 Lower the bar to trigger cube planner phase one
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c0e4cbbf318b18021065d37805ed1519d3d50e50
Author: Zhong, Yanghong
AuthorDate: Wed May 27 16:30:21 2020 +0800
KYLIN-4536 Lower the bar to trigger cube planner phase one
---
.../org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
index 1f3eaaf..dfea903 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
@@ -118,8 +118,8 @@ public class CuboidRecommender {
*/
public Map getRecommendCuboidList(CuboidStats cuboidStats,
KylinConfig kylinConf,
boolean ifForceRecommend) {
-long threshold1 = 1L <<
kylinConf.getCubePlannerAgreedyAlgorithmAutoThreshold();
-long threshold2 = 1L <<
kylinConf.getCubePlannerGeneticAlgorithmAutoThreshold();
+long threshold1 = 1L <<
kylinConf.getCubePlannerAgreedyAlgorithmAutoThreshold() - 1;
+long threshold2 = 1L <<
kylinConf.getCubePlannerGeneticAlgorithmAutoThreshold() - 1;
if (threshold1 >= threshold2) {
logger.error("Invalid Cube Planner Algorithm configuration");
return null;
@@ -128,7 +128,7 @@ public class CuboidRecommender {
int allCuboidCount = cuboidStats.getAllCuboidsForMandatory().size()
+ cuboidStats.getAllCuboidsForSelection().size();
-if (!ifForceRecommend && allCuboidCount <= threshold1) {
+if (!ifForceRecommend && allCuboidCount < threshold1) {
return null;
}
[kylin] 36/41: KYLIN-4600 Forbidden users except system admin to do metadata addition & update in Production
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 45647aaa492abf7fc5184f3de72f538d0408bbe2
Author: sanjulian
AuthorDate: Sun Jun 28 11:28:06 2020 +0800
KYLIN-4600 Forbidden users except system admin to do metadata addition &
update in Production
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 +++
.../org/apache/kylin/rest/service/CubeService.java | 29 +-
.../apache/kylin/rest/service/HybridService.java | 8 --
.../apache/kylin/rest/service/ModelService.java| 13 +++---
.../apache/kylin/rest/service/TableService.java| 12 ++---
.../org/apache/kylin/rest/util/AclEvaluate.java| 11
webapp/app/js/services/kylinProperties.js | 8 ++
webapp/app/partials/cubes/cubes.html | 6 ++---
webapp/app/partials/models/models_tree.html| 14 +--
webapp/app/partials/tables/source_table_tree.html | 8 +++---
webapp/app/partials/tables/table_detail.html | 2 +-
11 files changed, 79 insertions(+), 36 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f9bafee..01d0e10 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -262,6 +262,10 @@ public abstract class KylinConfigBase implements
Serializable {
return "DEV".equals(getOptional("kylin.env", "DEV"));
}
+public boolean isProdEnv() {
+return Boolean.parseBoolean(getOptional("kylin.env.is-prod", "false"));
+}
+
public String getDeployEnv() {
return getOptional("kylin.env", "DEV");
}
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 1a4b468..59781c5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -106,7 +106,6 @@ import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.access.AccessDeniedException;
-import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
@@ -213,10 +212,11 @@ public class CubeService extends BasicService implements
InitializingBean {
CubeUpdate update = new
CubeUpdate(cube.latestCopyForWrite()).setOwner(owner).setCost(cost);
return getCubeManager().updateCube(update);
}
-
-@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-+ " or hasPermission(#project, 'ADMINISTRATION') or
hasPermission(#project, 'MANAGEMENT')")
+
public CubeInstance createCubeAndDesc(ProjectInstance project, CubeDesc
desc) throws IOException {
+if (!aclEvaluate.checkProjectWritePermissionInProd()) {
+aclEvaluate.checkProjectWritePermission(project.getName());
+}
Message msg = MsgPicker.getMsg();
String cubeName = desc.getName();
@@ -288,7 +288,9 @@ public class CubeService extends BasicService implements
InitializingBean {
public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String
newProjectName, boolean forceUpdate)
throws IOException {
-aclEvaluate.checkProjectWritePermission(cube);
+if (!aclEvaluate.checkProjectWritePermissionInProd()) {
+aclEvaluate.checkProjectWritePermission(newProjectName);
+}
Message msg = MsgPicker.getMsg();
final List cubingJobs =
jobService.listJobsByRealizationName(cube.getName(), null,
@@ -844,10 +846,11 @@ public class CubeService extends BasicService implements
InitializingBean {
}
}
}
-
-@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-+ " or hasPermission(#project, 'ADMINISTRATION') or
hasPermission(#project, 'MANAGEMENT')")
+
public CubeDesc saveCube(CubeDesc desc, ProjectInstance project) throws
IOException {
+if (!aclEvaluate.checkProjectWritePermissionInProd()) {
+aclEvaluate.checkProjectWritePermission(project.getName());
+}
Message msg = MsgPicker.getMsg();
desc.setDraft(false);
@@ -867,10 +870,11 @@ public class CubeService extends BasicService implements
InitializingBean {
return desc;
}
-@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-+ " or has
[kylin] 41/41: EBAY-KYLIN-1381 Add google doc for cube migration request
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7e00a6a0ac9dc04698394e8f4cd3f874b346425c
Author: Zhong, Yanghong
AuthorDate: Mon Jun 29 13:31:25 2020 +0800
EBAY-KYLIN-1381 Add google doc for cube migration request
---
webapp/app/js/controllers/cubes.js | 2 ++
webapp/app/partials/cubes/cube_migrate.html | 8
2 files changed, 10 insertions(+)
diff --git a/webapp/app/js/controllers/cubes.js
b/webapp/app/js/controllers/cubes.js
index af529fd..4beb66d 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -1164,6 +1164,7 @@ var cubeMigrateCtrl = function ($scope, $modalInstance,
CubeService, cube, Proje
$scope.migrate={
targetProject: ProjectModel.selectedProject,
cubeValidate: true,
+googleDoc: false,
lockProjectName: false
}
@@ -1179,6 +1180,7 @@ var cubeMigrateCtrl = function ($scope, $modalInstance,
CubeService, cube, Proje
CubeService.ruleCheck({cubeId: cube.name, projectName:
$scope.migrate.targetProject}, function(result) {
loadingRequest.hide();
$scope.migrate.cubeValidate = false;
+ $scope.migrate.googleDoc = true;
$scope.migrate.lockProjectName = true;
if (result.length > 0) {
SweetAlert.swal('Attention', result, 'warning');
diff --git a/webapp/app/partials/cubes/cube_migrate.html
b/webapp/app/partials/cubes/cube_migrate.html
index 51b0900..d182c5f 100644
--- a/webapp/app/partials/cubes/cube_migrate.html
+++ b/webapp/app/partials/cubes/cube_migrate.html
@@ -39,6 +39,14 @@
Validate
+
+Business Justification:
+
+https://docs.google.com/a/ebay.com/forms/d/e/1FAIpQLSegYgCFpsbFsIZi5TxLs2rJtkNXP-92bTZQIzD-aHPjiKsKYQ/viewform;
target="_blank">
+
+ I acknowledge I have
submitted google form for onboard approval.
+
+
[kylin] 21/41: KYLIN-4583 Collect cpu time for coprocessor thread
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 40c61c29395707de5b5dcbffdff2892abfc359b8
Author: Zhong, Yanghong
AuthorDate: Fri Jun 19 14:45:02 2020 +0800
KYLIN-4583 Collect cpu time for coprocessor thread
---
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 9 +
1 file changed, 9 insertions(+)
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 93d8de4..e39e352 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -21,6 +21,7 @@ package
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadMXBean;
import java.net.InetAddress;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
@@ -242,6 +243,8 @@ public class CubeVisitService extends
CubeVisitProtos.CubeVisitService implement
CubeVisitProtos.CubeVisitResponse.ErrorInfo errorInfo = null;
+ThreadMXBean tdMXBean = ManagementFactory.getThreadMXBean();
+
// if user change kylin.properties on kylin server, need to manually
redeploy coprocessor jar to update KylinConfig of Env.
KylinConfig kylinConfig =
KylinConfig.createKylinConfig(request.getKylinProperties());
@@ -274,6 +277,7 @@ public class CubeVisitService extends
CubeVisitProtos.CubeVisitService implement
appendProfileInfo(sb, "start latency: " + (serviceStartTime -
scanReq.getStartTime()), serviceStartTime);
+final long serviceStartCpuTime =
tdMXBean.getCurrentThreadCpuTime();
final List cellListsForeachRawScan =
Lists.newArrayList();
for (RawScan hbaseRawScan : hbaseRawScans) {
@@ -386,6 +390,11 @@ public class CubeVisitService extends
CubeVisitProtos.CubeVisitService implement
logger.info("Total scanned {} rows and {} bytes",
cellListIterator.getTotalScannedRowCount(),
cellListIterator.getTotalScannedRowBytes());
+long cpuTime = (tdMXBean.getCurrentThreadCpuTime() -
serviceStartCpuTime) / 100L;
+sb.append("cpuTime@");
+sb.append(cpuTime);
+sb.append(",");
+
//outputStream.close() is not necessary
byte[] compressedAllRows;
if (errorInfo == null) {
[kylin] 10/41: KYLIN-4536 Update cube planner label and info
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 56b8cd01978efc76d63f8f6e9cc21428991303e6
Author: sanjulian
AuthorDate: Thu May 28 10:45:28 2020 +0800
KYLIN-4536 Update cube planner label and info
---
webapp/app/js/controllers/cube.js | 22 +-
webapp/app/js/model/cubeConfig.js | 10 +++---
2 files changed, 12 insertions(+), 20 deletions(-)
diff --git a/webapp/app/js/controllers/cube.js
b/webapp/app/js/controllers/cube.js
index 0d48dbc..5664690 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -227,14 +227,14 @@ KylinApp.controller('CubeCtrl', function ($scope,
$rootScope, AccessService, Mes
$scope.currentOptions.caption =
angular.copy(cubeConfig.currentCaption);
if ($scope.cube.recommendCuboids){
$scope.currentOptions.caption.css['text-align'] = 'right';
-$scope.currentOptions.caption.css['right'] = '-12px';
+$scope.currentOptions.caption.css['right'] = '-120px';
}
$scope.currentOptions.chart.color = function(d) {
var cuboid = _.find(data.nodeInfos, function(o) { return
o.name == d; });
if (cuboid.deleted) {
return d3.scale.category20c().range()[17];
} else {
-return getColorByQuery(0, 1/data.nodeInfos.length,
cuboid.query_rate);
+return getColorByQuery(1/data.nodeInfos.length,
cuboid.query_rate);
}
};
$scope.currentOptions.chart.sunburst = getSunburstDispatch();
@@ -246,14 +246,10 @@ KylinApp.controller('CubeCtrl', function ($scope,
$rootScope, AccessService, Mes
$scope.recommendOptions.caption =
angular.copy(cubeConfig.recommendCaption);
$scope.recommendOptions.chart.color = function(d) {
var cuboid = _.find(data.nodeInfos, function(o) { return
o.name == d; });
-if (cuboid.row_count < 0) {
+if (!cuboid.existed) {
return d3.scale.category20c().range()[5];
} else {
-var colorIndex = 0;
-if (!cuboid.existed) {
-colorIndex = 8;
-}
-return getColorByQuery(colorIndex,
1/data.nodeInfos.length, cuboid.query_rate);
+return getColorByQuery(1/data.nodeInfos.length,
cuboid.query_rate);
}
};
$scope.recommendOptions.chart.sunburst = getSunburstDispatch();
@@ -282,15 +278,15 @@ KylinApp.controller('CubeCtrl', function ($scope,
$rootScope, AccessService, Mes
};
// Different color for chart element by query count
-function getColorByQuery(colorIndex, baseRate, queryRate) {
+function getColorByQuery(baseRate, queryRate) {
if (queryRate > (3 * baseRate)) {
-return d3.scale.category20c().range()[colorIndex];
+return d3.scale.category20c().range()[0];
} else if (queryRate > (2 * baseRate)) {
-return d3.scale.category20c().range()[colorIndex+1];
+return d3.scale.category20c().range()[1];
} else if (queryRate > baseRate) {
-return d3.scale.category20c().range()[colorIndex+2];
+return d3.scale.category20c().range()[2];
} else {
-return d3.scale.category20c().range()[colorIndex+3];
+return d3.scale.category20c().range()[3];
}
}
diff --git a/webapp/app/js/model/cubeConfig.js
b/webapp/app/js/model/cubeConfig.js
index 77285f9..9dc9a68 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -176,7 +176,7 @@ KylinApp.constant('cubeConfig', {
},
currentCaption: {
enable: true,
-html: 'Existed:
Hottest '
+html: ' Hottest '
+ ' Hot '
+ ' Warm '
+ ' Cold '
@@ -189,17 +189,13 @@ KylinApp.constant('cubeConfig', {
},
recommendCaption: {
enable: true,
-html: 'New: Hottest '
- + ' Hot '
- + ' Warm '
- + ' Cold '
- + ' Mandatory',
+html: ' New',
css: {
position: 'relative',
top: '-35px',
height: 0,
'text-align': 'left',
- 'left': '-12px'
+ 'left': '93px'
}
}
});
[kylin] 24/41: KYLIN-4587 Send out notification email when triggering cube optimization
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 344edfc30ca3593cf63a5de2b00a032754447357
Author: sanjulian
AuthorDate: Fri Jun 19 16:58:27 2020 +0800
KYLIN-4587 Send out notification email when triggering cube optimization
---
.../kylin/job/util/MailNotificationUtil.java | 1 +
.../mail_templates/CUBE_OPTIMIZE_TRIGGER.ftl | 206 +
.../org/apache/kylin/rest/service/JobService.java | 35
3 files changed, 242 insertions(+)
diff --git
a/core-job/src/main/java/org/apache/kylin/job/util/MailNotificationUtil.java
b/core-job/src/main/java/org/apache/kylin/job/util/MailNotificationUtil.java
index 2335965..234a225 100644
--- a/core-job/src/main/java/org/apache/kylin/job/util/MailNotificationUtil.java
+++ b/core-job/src/main/java/org/apache/kylin/job/util/MailNotificationUtil.java
@@ -32,6 +32,7 @@ public class MailNotificationUtil {
public static final String JOB_DISCARD = "JOB_DISCARD";
public static final String JOB_SUCCEED = "JOB_SUCCEED";
public static final String METADATA_PERSIST_FAIL = "METADATA_PERSIST_FAIL";
+public static final String CUBE_OPTIMIZE_TRIGGER = "CUBE_OPTIMIZE_TRIGGER";
public static final String NA = "NA";
diff --git
a/core-job/src/main/resources/mail_templates/CUBE_OPTIMIZE_TRIGGER.ftl
b/core-job/src/main/resources/mail_templates/CUBE_OPTIMIZE_TRIGGER.ftl
new file mode 100644
index 000..1687b55
--- /dev/null
+++ b/core-job/src/main/resources/mail_templates/CUBE_OPTIMIZE_TRIGGER.ftl
@@ -0,0 +1,206 @@
+http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd;>
+http://www.w3.org/1999/xhtml;>
+
+
+
+
+
+
+html {
+font-size: 10px;
+}
+
+* {
+box-sizing: border-box;
+}
+
+a:hover,
+a:focus {
+color: #23527c;
+text-decoration: underline;
+}
+
+a:focus {
+outline: 5px auto -webkit-focus-ring-color;
+outline-offset: -2px;
+}
+
+
+
+
+
+Dear Kylin user,
+Congratulations! You trigger the optimize cube job.Thank you for using cube
planner function in Kylin.
+
+
+Info
+
+
+
+
+
+
+
+
+${job_name}
+
+
+
+
+
+
+
+${env_name}
+
+
+
+
+
+
+
+
+
+Submitter
+
+
+
+${submitter}
+
+
+
+
+Job Engine
+
+
+
+${job_engine}
+
+
+
+
+Project
+
+
+
+${project_name}
+
+
+
+
+Cube Name
+
+
+
+${cube_name}
+
+
+
+
+
+
+
+
+Best Wishes!
+mailto:dl-ebay-kylin-c...@ebay.com "
+style="color: #337ab7;text-decoration:
none;">eBay ADI Kylin Team
+
+
+
+
+
\ No newline at end of file
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 2f42410..a214c12 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -33,10 +33,14 @@ import java.util.TimeZone;
import javax.annotation.Nullable;
+import com.google.common.collect.Maps;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.lock.DistributedLock;
+import org.apache.kylin.common.util.MailService;
+import org.apache.kylin.common.util.MailTemplateProvider;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.common.zookeeper.KylinServerDiscovery;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -66,6 +70,7 @@ import org.apache.kylin.job.execution.CheckpointExecutable
[kylin] 06/41: KYLIN-4530 Adjust log level for CubeHBaseEndpointRPC
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 26dbfa6c889a957f73a5cc0d1ac969637a21cc6a
Author: Zhong, Yanghong
AuthorDate: Wed May 27 15:30:11 2020 +0800
KYLIN-4530 Adjust log level for CubeHBaseEndpointRPC
---
.../apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 8
.../java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index a810351..175bc37 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -335,7 +335,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
HRegionLocation regionLocation =
getStartRegionLocation(rowsService);
String regionServerName = regionLocation == null ?
"UNKNOWN" : regionLocation.getHostname();
-logger.info("Query-{}: send request to the init
region server {} on table {} ", queryId,
+logger.debug("Query-{}: send request to the init
region server {} on table {} ", queryId,
regionServerName, table.getName());
queryContext.addQueryStopListener(new
QueryContext.QueryStopListener() {
@@ -427,7 +427,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
return;
}
-logger.info(logHeader + getStatsString(region,
result));
+logger.debug(logHeader + getStatsString(region,
result));
Stats stats = result.getStats();
queryContext.addAndGetScannedRows(stats.getScannedRowCount());
@@ -490,13 +490,13 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
cubeSeg.getCubeInstance().getName(), cubeSeg.getName());
if (cubeSegmentStatistics != null) {
segmentQueryResultBuilder.setCubeSegmentStatistics(cubeSegmentStatistics);
-logger.info(
+logger.debug(
"Query-{}: try to put
segment query result to cache for segment:{}",
queryContext.getQueryId(),
cubeSeg);
SegmentQueryResult
segmentQueryResult = segmentQueryResultBuilder.build();
SegmentQueryCache.getInstance().put(segmentQueryCacheKey,
segmentQueryResult);
-logger.info(
+logger.debug(
"Query-{}: successfully
put segment query result to cache for segment:{}",
queryContext.getQueryId(),
cubeSeg);
}
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index c14d5d4..ec581aa 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -285,7 +285,7 @@ public abstract class CubeHBaseRPC implements IGTStorage {
} else {
info.append(", No Fuzzy Key");
}
-logger.info(info.toString());
+logger.debug(info.toString());
}
protected long getCoprocessorTimeoutMillis() {
[kylin] 12/41: KYLIN-4533 Create a tool to check cube optimization benefit
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1b5badb0ecb09f9d82dc1296c714e452f830150b
Author: Zhong, Yanghong
AuthorDate: Mon Jun 8 18:49:43 2020 +0800
KYLIN-4533 Create a tool to check cube optimization benefit
---
.../apache/kylin/common/restclient/RestClient.java | 12 +
.../org/apache/kylin/tool/CubeOptimizationCLI.java | 363 +
.../mail_templates/OPTIMIZATION_BENEFITS.ftl | 123 +++
3 files changed, 498 insertions(+)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 21b08f8..e23c3a9 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -382,6 +382,18 @@ public class RestClient {
return content;
}
+public String getCuboidRecommendResponse(String cubeName) throws
IOException {
+String url = baseUrl + "/cubes/" + cubeName + "/cuboids/recommend";
+HttpGet get = newGet(url);
+HttpResponse response = client.execute(get);
+String content = getContent(response);
+if (response.getStatusLine().getStatusCode() != 200) {
+throw new IOException(
+"Invalid response " +
response.getStatusLine().getStatusCode() + " with url " + url + "\n");
+}
+return content;
+}
+
public void checkCompatibility(String jsonRequest) throws IOException {
checkCompatibility(jsonRequest, false);
}
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeOptimizationCLI.java
b/tool/src/main/java/org/apache/kylin/tool/CubeOptimizationCLI.java
new file mode 100644
index 000..af4cfb1
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeOptimizationCLI.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.restclient.RestClient;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.MailService;
+import org.apache.kylin.common.util.MailTemplateProvider;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.ToolUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.cuboid.algorithm.OptimizationBenefit;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.shaded.com.google.common.base.Joiner;
+import org.apache.kylin.shaded.com.google.common.collect.Lists;
+import org.apache.kylin.shaded.com.google.common.collect.Maps;
+import org.apache.kylin.shaded.com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+public class CubeOptimizationCLI extends AbstractApplication {
+private static final Logger logger =
LoggerFactory.getLogger(CubeOptimizationCLI.class);
+
+@SuppressWarnings("static-access")
+private static final Option OPTION_KYLIN_CONFIG_URI =
OptionBuilder.withArgName("kylinConfigUri").hasArg()
+.isRequired(true).withDescription("Kylin configuration
[kylin] 08/41: KYLIN-4536 Make the stats for current cuboids consistent with the ones for recommend cuboids
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8a0567e47528b63db4de01308a56efa801e66c4e
Author: Zhong, Yanghong
AuthorDate: Wed May 27 17:15:10 2020 +0800
KYLIN-4536 Make the stats for current cuboids consistent with the ones for
recommend cuboids
---
.../java/org/apache/kylin/rest/controller/CubeController.java | 8 +++-
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 1b5f1e4..d237002 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -902,12 +902,11 @@ public class CubeController extends BasicController {
public CuboidTreeResponse getCurrentCuboids(@PathVariable String cubeName)
{
checkCubeExists(cubeName);
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+
// The cuboid tree displayed should be consistent with the current one
CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
-Map cuboidStatsMap = cube.getCuboids();
-if (cuboidStatsMap == null) {
-cuboidStatsMap =
CuboidStatsReaderUtil.readCuboidStatsFromCube(cuboidScheduler.getAllCuboidIds(),
cube);
-}
+Set currentCuboidSet = cuboidScheduler.getAllCuboidIds();
+Map cuboidStatsMap =
CuboidStatsReaderUtil.readCuboidStatsFromCube(currentCuboidSet, cube);
Map hitFrequencyMap = null;
Map queryMatchMap = null;
@@ -918,7 +917,6 @@ public class CubeController extends BasicController {
logger.warn("Fail to query on system cube due to " + e);
}
-Set currentCuboidSet =
cube.getCuboidScheduler().getAllCuboidIds();
return cubeService.getCuboidTreeResponse(cuboidScheduler,
cuboidStatsMap, hitFrequencyMap, queryMatchMap,
currentCuboidSet);
}
[kylin] 09/41: KYLIN-4536 Add more logs
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9167c1bf2263437039d6a50ecc339dfdfbdf2b47
Author: Zhong, Yanghong
AuthorDate: Wed May 27 18:17:25 2020 +0800
KYLIN-4536 Add more logs
---
.../algorithm/AbstractRecommendAlgorithm.java | 1 +
.../generic/CombinedStoppingCondition.java | 9 -
.../cuboid/algorithm/greedy/GreedyAlgorithm.java | 46 --
3 files changed, 33 insertions(+), 23 deletions(-)
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
index 094b960..1635264 100755
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
@@ -48,6 +48,7 @@ public abstract class AbstractRecommendAlgorithm implements
CuboidRecommendAlgor
@Override
public List recommend(double expansionRate) {
double spaceLimit = cuboidStats.getBaseCuboidSize() * expansionRate;
+logger.info("space limit for the algorithm is {} with expansion rate
{}", spaceLimit, expansionRate);
return start(spaceLimit);
}
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
index a12c44d..cad3839 100755
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
@@ -18,12 +18,16 @@
package org.apache.kylin.cube.cuboid.algorithm.generic;
+import java.util.List;
+
import org.apache.commons.math3.genetics.Population;
import org.apache.commons.math3.genetics.StoppingCondition;
-
-import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CombinedStoppingCondition implements StoppingCondition {
+private static final Logger logger =
LoggerFactory.getLogger(GeneticAlgorithm.class);
+
private List conditions;
public CombinedStoppingCondition(List conditions) {
@@ -34,6 +38,7 @@ public class CombinedStoppingCondition implements
StoppingCondition {
public boolean isSatisfied(Population population) {
for (StoppingCondition condition : conditions) {
if (condition.isSatisfied(population)) {
+logger.info("Stopping condition {} is satisfied", condition);
return true;
}
}
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
index 0a48eea..268223c 100755
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
@@ -19,6 +19,7 @@
package org.apache.kylin.cube.cuboid.algorithm.greedy;
import java.util.List;
+import java.util.Locale;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@@ -74,32 +75,35 @@ public class GreedyAlgorithm extends
AbstractRecommendAlgorithm {
remaining.addAll(cuboidStats.getAllCuboidsForSelection());
long round = 0;
-boolean doesRemainSpace = true;
-while (!shouldCancel() && doesRemainSpace) {
-// Choose one cuboId having the maximum benefit per unit space in
all available list
+while (!shouldCancel()) {
+// Choose one cuboid having the maximum benefit per unit space in
all available list
CuboidBenefitModel best = recommendBestOne();
-
// If return null, then we should finish the process and return
+if (best == null) {
+logger.info("Greedy algorithm ends due to cannot find next
best one");
+break;
+}
// If we finally find the cuboid selected does not meet a minimum
threshold of benefit (for
// example, a cuboid with 0.99M roll up from a parent cuboid with
1M
// rows), then we should finish the process and return
-if (best != null && benefitPolicy.ifEfficient(best)) {
-remainingSpace -=
cuboidStats.getCuboidSize(best.getCuboidId());
-
-// If we finally find there is no remaining space, then we
should finish the process and return
-if (remainingSpace > 0) {
-selected.add(best.getC
[kylin] 01/41: KYLIN-4513 Refactor QueryContext constructor
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a692adf8c7064861206e3f1d282f9e06b72f68d1
Author: Zhong, Yanghong
AuthorDate: Mon May 25 19:21:58 2020 +0800
KYLIN-4513 Refactor QueryContext constructor
---
.../java/org/apache/kylin/common/QueryContext.java | 40 +-
.../apache/kylin/common/QueryContextFacade.java| 36 +++
.../java/org/apache/kylin/query/KylinTestBase.java | 6 ++--
.../apache/kylin/storage/hbase/ITStorageTest.java | 3 ++
.../kylin/query/security/QueryACLTestUtil.java | 9 +++--
.../apache/kylin/rest/service/QueryService.java| 23 -
.../kylin/rest/metrics/QueryMetricsTest.java | 9 +++--
.../kylin/rest/service/QueryInfoCollectorTest.java | 16 ++---
.../kylin/rest/service/QueryServiceTest.java | 7 ++--
.../server/rest/controller/QueryController.java| 3 ++
10 files changed, 89 insertions(+), 63 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 680a694..0f6534f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -30,11 +30,10 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.kylin.common.exceptions.KylinTimeoutException;
import org.apache.kylin.common.util.RandomUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.shaded.com.google.common.collect.Lists;
import org.apache.kylin.shaded.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Holds per query information and statistics.
@@ -49,12 +48,14 @@ public class QueryContext {
public interface QueryStopListener {
void stop(QueryContext query);
}
-
-private long queryStartMillis;
-
+
private final String queryId;
-private String username;
-private String project;
+private final String project;
+private final String sql;
+private final String username;
+private final int maxConnThreads;
+private final long queryStartMillis;
+
private Set groups;
private AtomicLong scannedRows = new AtomicLong();
private AtomicLong returnedRows = new AtomicLong();
@@ -70,18 +71,19 @@ public class QueryContext {
private List rpcStatisticsList =
Lists.newCopyOnWriteArrayList();
private Map
cubeSegmentStatisticsResultMap = Maps.newConcurrentMap();
-final int maxConnThreads;
-
private ExecutorService connPool;
-QueryContext(int maxConnThreads) {
-this(maxConnThreads, System.currentTimeMillis());
+QueryContext(String projectName, String sql, String user, int
maxConnThreads) {
+this(projectName, sql, user, maxConnThreads,
System.currentTimeMillis());
}
-QueryContext(int maxConnThreads, long startMills) {
-queryId = RandomUtil.randomUUID().toString();
-queryStartMillis = startMills;
+QueryContext(String projectName, String sql, String user, int
maxConnThreads, long startMills) {
+this.queryId = RandomUtil.randomUUID().toString();
+this.project = projectName;
+this.sql = sql;
+this.username = user;
this.maxConnThreads = maxConnThreads;
+this.queryStartMillis = startMills;
}
public ExecutorService getConnectionPool(ExecutorService sharedConnPool) {
@@ -119,18 +121,10 @@ public class QueryContext {
return username;
}
-public void setUsername(String username) {
-this.username = username;
-}
-
public String getProject() {
return project;
}
-public void setProject(String project) {
-this.project = project;
-}
-
public Set getGroups() {
return groups;
}
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContextFacade.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContextFacade.java
index e1cc1ec..293232f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContextFacade.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContextFacade.java
@@ -24,29 +24,24 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentMap;
import org.apache.kylin.common.threadlocal.InternalThreadLocal;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.shaded.com.google.common.collect.Maps;
import org.apache.kylin.shaded.com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class QueryContextFacade {
private static final Logger logger =
LoggerFactory.getLogger(QueryContextFacade.class);
private static final ConcurrentMap RUNNING_CTX_MAP
[kylin] 38/41: KYLIN-4549 Show column cardinality in rowkeys area of advanced settings
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0a4b628d461cbb532414f1f32c0de73b00c79254
Author: Julian Pan
AuthorDate: Fri Jun 26 11:56:23 2020 +0800
KYLIN-4549 Show column cardinality in rowkeys area of advanced settings
---
webapp/app/js/controllers/cubeAdvanceSetting.js| 73 +-
.../partials/cubeDesigner/advanced_settings.html | 9 +++
webapp/app/partials/tables/table_detail.html | 2 +-
3 files changed, 55 insertions(+), 29 deletions(-)
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js
b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 1048839..360642d 100755
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -18,39 +18,54 @@
'use strict';
-KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope,
$modal,cubeConfig,MetaModel,cubesManager,CubeDescModel,SweetAlert,VdmUtil,modelsManager)
{
+KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope,
$modal,cubeConfig,MetaModel,TableService,cubesManager,CubeDescModel,SweetAlert,VdmUtil,modelsManager)
{
$scope.cubesManager = cubesManager;
var needLengthKeyList=cubeConfig.needSetLengthEncodingList;
$scope.convertedRowkeys = [];
$scope.dim_cap = $scope.cubeMetaFrame.aggregation_groups.length > 0 &&
$scope.cubeMetaFrame.aggregation_groups[0].select_rule.dim_cap ?
$scope.cubeMetaFrame.aggregation_groups[0].select_rule.dim_cap : 0;
- angular.forEach($scope.cubeMetaFrame.rowkey.rowkey_columns,function(item){
-item.encoding=$scope.removeVersion(item.encoding);
-var _valueLength;
-var tableName=VdmUtil.getNameSpaceTopName(item.column);
-var databaseName=modelsManager.getDatabaseByColumnName(item.column);
-var baseKey=item.encoding.replace(/:\d+/,'');
-if(needLengthKeyList.indexOf(baseKey)>-1){
- var result=/:(\d+)/.exec(item.encoding);
- _valueLength=result?result[1]:0;
-}
-var _encoding=baseKey;
-var rowkeyObj = {
- column:item.column,
-
encoding:_encoding+(item.encoding_version?"[v"+item.encoding_version+"]":"[v1]"),
- encodingName:_encoding,
- valueLength:_valueLength,
- isShardBy:item.isShardBy,
- encoding_version:item.encoding_version||1,
- table:tableName,
- database:databaseName
-}
-if(item.index){
- rowkeyObj.index=item.index;
-}
-$scope.convertedRowkeys.push(rowkeyObj);
-
- })
+
+ TableService.list({ext: true, project:$scope.projectModel.selectedProject},
function(tables) {
+$scope.initRowKey(tables);
+ }, function (error) {
+$scope.initRowKey([]);
+ });
+
+ $scope.initRowKey = function(tables) {
+angular.forEach($scope.cubeMetaFrame.rowkey.rowkey_columns,function(item){
+ item.encoding=$scope.removeVersion(item.encoding);
+ var _valueLength;
+ var tableName=VdmUtil.getNameSpaceTopName(item.column);
+ var databaseName=modelsManager.getDatabaseByColumnName(item.column);
+ var baseKey=item.encoding.replace(/:\d+/,'');
+ if(needLengthKeyList.indexOf(baseKey)>-1){
+var result=/:(\d+)/.exec(item.encoding);
+_valueLength=result?result[1]:0;
+ }$scope.cubeMetaFrame
+ var _encoding=baseKey;
+ var rowkeyTable = _.find(tables, function(table) {
+var modelDesc =
modelsManager.getModel($scope.cubeMetaFrame.model_name);
+var lookupTable = modelDesc ? _.find(modelDesc.lookups,
function(lookup){ return lookup.alias === tableName; }) : undefined;
+return ((modelDesc && modelDesc.fact_table === (table.database + '.' +
table.name) && table.name === tableName) || (lookupTable && lookupTable.table
=== (table.name + table.database)));
+ });
+ var cardinality = rowkeyTable ?
rowkeyTable.cardinality[VdmUtil.removeNameSpace(item.column)] : undefined;
+ var rowkeyObj = {
+column:item.column,
+
encoding:_encoding+(item.encoding_version?"[v"+item.encoding_version+"]":"[v1]"),
+encodingName:_encoding,
+valueLength:_valueLength,
+isShardBy:item.isShardBy,
+encoding_version:item.encoding_version||1,
+table:tableName,
+database:databaseName,
+cardinality: cardinality || 'N/A'
+ }
+ if(item.index){
+rowkeyObj.index=item.index;
+ }
+ $scope.convertedRowkeys.push(rowkeyObj);
+})
+ }
$scope.rule={
@@ -66,6 +81,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function
($scope, $modal,cubeConfi
var isShardBy = item.isShardBy;
var version=$scope.getTypeVersion(item.encoding);
var encodingType=$scope.removeVersion(item.encoding);
+var cardinality = item.cardinality;
if(needLengthKeyList.indexOf(encodingType)!=-1){
encodi
[kylin] 20/41: KYLIN-4508 Add unit test for core-metrics module & reporters
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 948368fd3210add6e1cb71964c785fb2f2ea3bae
Author: Zhong, Yanghong
AuthorDate: Fri Jun 19 13:41:09 2020 +0800
KYLIN-4508 Add unit test for core-metrics module & reporters
---
.../kylin/metrics/lib/impl/TimedRecordEvent.java | 32
metrics-reporter-hive/pom.xml | 25
.../kylin/metrics/lib/impl/hive/HiveProducer.java | 7 +-
.../metrics/lib/impl/hive/HiveProducerRecord.java | 141 +++---
.../lib/impl/hive/HiveReservoirReporter.java | 39 +++--
.../tool/metrics/systemcube/HiveTableCreator.java | 3 +-
.../lib/impl/hive/HiveProducerRecordTest.java | 81 +++
.../metrics/lib/impl/hive/HiveProducerTest.java| 161 +
.../lib/impl/hive/HiveReservoirReporterTest.java | 88 +++
metrics-reporter-kafka/pom.xml | 19 +++
.../impl/kafka/KafkaActiveReserviorListener.java | 8 +
.../lib/impl/kafka/KafkaReservoirReporter.java | 6 +-
.../lib/impl/kafka/KafkaReservoirReporterTest.java | 79 ++
13 files changed, 585 insertions(+), 104 deletions(-)
diff --git
a/core-metrics/src/main/java/org/apache/kylin/metrics/lib/impl/TimedRecordEvent.java
b/core-metrics/src/main/java/org/apache/kylin/metrics/lib/impl/TimedRecordEvent.java
index a866163..984d5f5 100644
---
a/core-metrics/src/main/java/org/apache/kylin/metrics/lib/impl/TimedRecordEvent.java
+++
b/core-metrics/src/main/java/org/apache/kylin/metrics/lib/impl/TimedRecordEvent.java
@@ -44,4 +44,36 @@ public class TimedRecordEvent extends RecordEvent {
super.resetTime();
addTimeDetails();
}
+
+public String getYear() {
+return (String) get(TimePropertyEnum.YEAR.toString());
+}
+
+public String getMonth() {
+return (String) get(TimePropertyEnum.MONTH.toString());
+}
+
+public String getWeekBeginDate() {
+return (String) get(TimePropertyEnum.WEEK_BEGIN_DATE.toString());
+}
+
+public String getDayDate() {
+return (String) get(TimePropertyEnum.DAY_DATE.toString());
+}
+
+public String getDayTime() {
+return (String) get(TimePropertyEnum.DAY_TIME.toString());
+}
+
+public int getTimeHour() {
+return (int) get(TimePropertyEnum.TIME_HOUR.toString());
+}
+
+public int getTimeMinute() {
+return (int) get(TimePropertyEnum.TIME_MINUTE.toString());
+}
+
+public int getTimeSecond() {
+return (int) get(TimePropertyEnum.TIME_SECOND.toString());
+}
}
diff --git a/metrics-reporter-hive/pom.xml b/metrics-reporter-hive/pom.xml
index 0437030..9fefa51 100644
--- a/metrics-reporter-hive/pom.xml
+++ b/metrics-reporter-hive/pom.xml
@@ -56,5 +56,30 @@
hadoop-hdfs
provided
+
+
+
+junit
+junit
+test
+
+
+org.apache.hadoop
+hadoop-mapreduce-client-core
+test
+
+
+
+org.powermock
+powermock-api-mockito
+${powermock.version}
+test
+
+
+org.powermock
+powermock-module-junit4-rule-agent
+${powermock.version}
+test
+
\ No newline at end of file
diff --git
a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
index 8bc7a43..e79010c 100644
---
a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
+++
b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
@@ -309,7 +309,7 @@ public class HiveProducer {
fout = null;
}
-private HiveProducerRecord convertTo(Record record) throws Exception {
+HiveProducerRecord convertTo(Record record) throws Exception {
Map rawValue = record.getValueRaw();
//Set partition values for hive table
@@ -330,7 +330,8 @@ public class HiveProducer {
columnValues.add(rawValue.get(fieldSchema.getName().toUpperCase(Locale.ROOT)));
}
-return new HiveProducerRecord(tableNameSplits.getFirst(),
tableNameSplits.getSecond(), partitionKVs,
-columnValues);
+HiveProducerRecord.RecordKey key = new
HiveProducerRecord.KeyBuilder(tableNameSplits.getSecond())
+
.setDbName(tableNameSplits.getFirst()).setPartitionKVs(partitionKVs).build();
+return new HiveProducerRecord(key, columnValues);
}
}
diff --git
a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducerRecord.java
b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/
[kylin] 17/41: KYLIN-4566 Add cli tool to get hbase storage usage statistics
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f6912b46825d8c90f02ea3ca2ac8102edf318193
Author: Zhong, Yanghong
AuthorDate: Mon Jun 15 15:02:21 2020 +0800
KYLIN-4566 Add cli tool to get hbase storage usage statistics
---
.../org/apache/kylin/tool/HTableMonitorCLI.java| 308 +++
.../resources/mail_templates/HBASE_USAGE_CHECK.ftl | 341 +
2 files changed, 649 insertions(+)
diff --git a/tool/src/main/java/org/apache/kylin/tool/HTableMonitorCLI.java
b/tool/src/main/java/org/apache/kylin/tool/HTableMonitorCLI.java
new file mode 100644
index 000..969fa36
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/HTableMonitorCLI.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.MailService;
+import org.apache.kylin.common.util.MailTemplateProvider;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.shaded.com.google.common.collect.Lists;
+import org.apache.kylin.shaded.com.google.common.collect.Maps;
+import org.apache.kylin.shaded.com.google.common.collect.Sets;
+import
org.apache.kylin.shaded.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HTableMonitorCLI {
+private static final Logger logger =
LoggerFactory.getLogger(HTableMonitorCLI.class);
+
+private static final int THREAD_NUM = 10;
+
+public static void main(String[] args) throws IOException {
+
+if (args.length != 2) {
+logger.warn("Usage: HTableMonitorCLI receivers kylinConfigUris");
+return;
+}
+
+StringBuilder htableStatsInfo = new StringBuilder();
+Map>>>
hTableStats = Maps.newHashMap();
+for (String kylinConfigUri : args[1].split(",")) {
+try {
+hTableStats.put(kylinConfigUri,
getHTableStatsByUri(kylinConfigUri));
+} catch (Exception e) {
+String errMsg = "Environment " + kylinConfigUri + ": fail to
get htable stats due to " + e;
+logger.warn(errMsg);
+htableStatsInfo.append(errMsg + "\n");
+}
+}
+
+if (!hTableStats.isEmpty()) {
+htableStatsInfo.append(printHTableStats(hTableStats));
+}
+
+Set receiverSet = Sets.newHashSet(args[0].split(","));
+List receivers = receiverSet.stream().filter(r ->
r.contains("@")).collect(Collectors.toList());
+
+String title = MailTemplateProvider.getMailTitle("HBASE",
"USAGE_CHECK");
+emailHTableStatsInfo(receivers, title, htableStatsInfo.toString());
+}
+
+private static Map>>
getHTableStatsByUri(String kylinConfigUri)
+throws IOException {
+
+Map>> result =
Maps.newHashMap();
+
+KylinConfig kylinConfig =
KylinConfig.createInstanceFromUri(kylinConfigUri);
+final Connection conn =
HBaseConnection.get
[kylin] 14/41: KYLIN-4534 Backend support for query & storage trend
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4e6884b0a982e6fdb4045ba33fb14bbe33a7dc35
Author: Zhong, Yanghong
AuthorDate: Wed Jun 10 14:21:16 2020 +0800
KYLIN-4534 Backend support for query & storage trend
---
.../java/org/apache/kylin/cube/CubeInstance.java | 14 +
.../kylin/rest/controller/CubeController.java | 58 ++-
.../kylin/rest/controller/DashboardController.java | 44 +--
.../response/CubeOptimizationTrendResponse.java| 46 +++
.../org/apache/kylin/rest/service/CubeService.java | 96 +++--
.../kylin/rest/service/DashboardService.java | 288 +++
.../apache/kylin/rest/service/QueryService.java| 7 -
.../apache/kylin/rest/util/SqlCreationUtil.java| 388 +
.../kylin/rest/util/SqlCreationUtilTest.java | 240 +
9 files changed, 837 insertions(+), 344 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 0d3f39d..7a8c13b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -25,6 +25,7 @@ import static
org.apache.kylin.cube.cuboid.CuboidModeEnum.RECOMMEND;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -69,6 +70,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.kylin.shaded.com.google.common.base.Objects;
import org.apache.kylin.shaded.com.google.common.collect.Lists;
+import org.apache.kylin.shaded.com.google.common.collect.Sets;
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility =
Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility =
Visibility.NONE)
@@ -127,6 +129,9 @@ public class CubeInstance extends RootPersistentEntity
implements IRealization,
@JsonProperty("cuboid_last_optimized")
private long cuboidLastOptimized;
+@JsonProperty("cuboid_optimized_timestamp_serial")
+private Set cuboidOptimizedTimestamps = Sets.newHashSet();
+
@JsonProperty("snapshots")
private Map snapshots = Maps.newHashMap();
@@ -457,6 +462,7 @@ public class CubeInstance extends RootPersistentEntity
implements IRealization,
cuboidBytes = null;
cuboidBytesRecommend = null;
cuboidLastOptimized = 0L;
+cuboidOptimizedTimestamps.clear();
}
public Set getCuboidsByMode(String cuboidModeName) {
@@ -559,6 +565,14 @@ public class CubeInstance extends RootPersistentEntity
implements IRealization,
public void setCuboidLastOptimized(long lastOptimized) {
this.cuboidLastOptimized = lastOptimized;
+this.cuboidOptimizedTimestamps.add(lastOptimized);
+}
+
+public List getCuboidOptimizedTimestamps() {
+this.cuboidOptimizedTimestamps.add(cuboidLastOptimized);
+List ret = Lists.newArrayList(this.cuboidOptimizedTimestamps);
+Collections.sort(ret);
+return ret;
}
/**
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index e937920..af3e9b1 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
import javax.servlet.http.HttpServletResponse;
@@ -73,6 +75,7 @@ import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
import org.apache.kylin.rest.request.LookupSnapshotBuildRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
+import org.apache.kylin.rest.response.CubeOptimizationTrendResponse;
import org.apache.kylin.rest.response.CuboidRecommendResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
@@ -85,6 +88,9 @@ import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
+import org.apache.kylin.shaded.com.google.common.collect.Lists;
+import org.apache.kylin.shaded.com.google.common.collect.Maps;
+import org.apache.kylin.shaded.com.google.common.collect.Sets;
import org.apache.kylin.source.kafka.util.Ka
[kylin] 25/41: KYLIN-4588 Add a rest API to get project list for a user able to access
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2a7cb7b1024452b87a2d9b000e3a93c711c2949d
Author: Zhong, Yanghong
AuthorDate: Fri Jun 19 17:47:50 2020 +0800
KYLIN-4588 Add a rest API to get project list for a user able to access
---
.../kylin/rest/controller/ProjectController.java | 30 ++
.../apache/kylin/rest/service/AccessService.java | 12 +
2 files changed, 42 insertions(+)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
index 410ff11..7edca34 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
@@ -36,10 +36,14 @@ import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
+import org.apache.kylin.shaded.com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
@@ -118,6 +122,32 @@ public class ProjectController extends BasicController {
return readableProjects.subList(projectOffset, projectOffset +
projectLimit);
}
+@RequestMapping(value = "/readable/user/{userName}", method = {
RequestMethod.GET }, produces = {
+"application/json" })
+@ResponseBody
+public List getReadableProjects(@PathVariable String
userName,
+@RequestParam(value = "limit", required = false) Integer limit,
+@RequestParam(value = "offset", required = false) Integer offset) {
+Authentication origAuth =
SecurityContextHolder.getContext().getAuthentication();
+
+logger.info("Going to impersonate to user {} from {}", userName,
origAuth.getName());
+
+try {
+accessService.switchToUser(userName);
+return getReadableProjects(limit, offset);
+} catch (UsernameNotFoundException e) {
+logger.warn("User {} does not exist in system", userName);
+return Lists.newArrayList();
+} catch (InternalErrorException e) {
+throw e;
+} catch (Exception e) {
+throw new InternalErrorException(e);
+} finally {
+SecurityContextHolder.getContext().setAuthentication(origAuth);
+logger.info("Recover to the origin user {} from impersonated one
{}", origAuth.getName(), userName);
+}
+}
+
@RequestMapping(value = "", method = { RequestMethod.POST }, produces = {
"application/json" })
@ResponseBody
public ProjectInstance saveProject(@RequestBody ProjectRequest
projectRequest) {
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 5b7d562..62c7b71 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -58,9 +58,11 @@ import
org.springframework.security.acls.model.NotFoundException;
import org.springframework.security.acls.model.ObjectIdentity;
import org.springframework.security.acls.model.Permission;
import org.springframework.security.acls.model.Sid;
+import
org.springframework.security.authentication.AnonymousAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
@@ -75,6 +77,10 @@ public class AccessService {
@Qualifier("aclService")
private AclService aclService;
+@Autowired
+@Qualifier("userService")
+UserService userService;
+
// ~ Methods to manage acl life circle of domain objects ~
@Transactional
@@ -481,4 +487,10 @@ public class AccessService {
[kylin] tag ebay-3.1.0-release-20200701 created (now 7e00a6a)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to tag ebay-3.1.0-release-20200701 in repository https://gitbox.apache.org/repos/asf/kylin.git. at 7e00a6a (commit) This tag includes the following new commits: new a692adf KYLIN-4513 Refactor QueryContext constructor new eb70b5d KYLIN-4513 Introduce query call trace new f6e26d1 KYLIN-4528 Bad hbase region & irregular query detection new 1709e64 KYLIN-4529 Add user behavior audit log new bad7f06 KYLIN-4530 Reduce duplication SQL log new 26dbfa6 KYLIN-4530 Adjust log level for CubeHBaseEndpointRPC new c0e4cbb KYLIN-4536 Lower the bar to trigger cube planner phase one new 8a0567e KYLIN-4536 Make the stats for current cuboids consistent with the ones for recommend cuboids new 9167c1b KYLIN-4536 Add more logs new 56b8cd0 KYLIN-4536 Update cube planner label and info new 99d3c9b KYLIN-4532 Introduce cube planner optimization benefit score new 1b5badb KYLIN-4533 Create a tool to check cube optimization benefit new d435a64 KYLIN-4564 Refine query metrics update for hybrid realization new 4e6884b KYLIN-4534 Backend support for query & storage trend new 637d83f KYLIN-4535 Frontend support for query & storage trend new 415a43a KYLIN-4565 Add cli tool to export base cuboid data new f6912b4 KYLIN-4566 Add cli tool to get hbase storage usage statistics new 6c5c455 KYLIN-4579 Add fat API to create cube & delete cube new 2b3b6df EBAY-KYLIN-1919 Add prometheus metrics reporter new 948368f KYLIN-4508 Add unit test for core-metrics module & reporters new 40c61c2 KYLIN-4583 Collect cpu time for coprocessor thread new a47ce3b KYLIN-4584 Add one option backup-hdfs for metastore.sh to backup metadata to hdfs new 8ba2aa7 KYLIN-4586 Add a configuration of maximum partition returned bytes new 344edfc KYLIN-4587 Send out notification email when triggering cube optimization new 2a7cb7b KYLIN-4588 Add a rest API to get project list for a user able to access new 6c7f4e0 EBAY-KYLIN-1442 Json ignore new added fields in SQLResponse new c5ef6a5 EBAY-KYLIN-1204 Check and fix Kylin Yarn job tracking url new ab09889 KYLIN-4589 Display ready segments only in the 'REFRESH SEGMENT' drop-down box new b7f0bd0 KYLIN-4592 Make KylinConfig used by hbase coprocessor be cube segment level new cb10d18 EBAY-KYLIN-1997 Add eBay specified migration tool cross DC new db2a22c KYLIN-4594 Send out notification email when triggering cube building job and finding out table schema changes new 574f5c0 KYLIN-4596 Consider CubeInstance reinitialization time when querying system cubes new d0e352b EBAY-KYLIN-2371 Don't use DictionaryEnumerator for foreign key only query for old cubes whose dictionaries were built from lookup tables new b0ef899 EBAY-KYLIN-2346 Deal with tls for Windows ODBC new 23bc5ac KYLIN-3140 Auto merge jobs should not be counted in kylin.cube.max-building-segments new 45647aa KYLIN-4600 Forbidden users except system admin to do metadata addition & update in Production new 69fd35b KYLIN-4602 Fix query display format not correct in insight table new 0a4b628 KYLIN-4549 Show column cardinality in rowkeys area of advanced settings new 75f76a6 KYLIN-4550 Provide advanced refresh interface inside the refresh panel new c27577f KYLIN-4551 Provide interfaces to transfer cube/model/project ownership new 7e00a6a EBAY-KYLIN-1381 Add google doc for cube migration request The 41 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[kylin] branch master updated: KYLIN-3794 fix mergeToInClause not working properly in some edge cases
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new b1f1b7b KYLIN-3794 fix mergeToInClause not working properly in some
edge cases
b1f1b7b is described below
commit b1f1b7bdf04a00fd9fd96e35e700f4ee7aeb455e
Author: kyotoYaho
AuthorDate: Wed Jan 30 10:07:35 2019 +0800
KYLIN-3794 fix mergeToInClause not working properly in some edge cases
---
.../query/relnode/visitor/TupleFilterVisitor.java | 78
.../relnode/visitor/TupleFilterVisitorTest.java| 86 ++
2 files changed, 132 insertions(+), 32 deletions(-)
diff --git
a/query/src/main/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitor.java
b/query/src/main/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitor.java
index faa9988..7634023 100644
---
a/query/src/main/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitor.java
+++
b/query/src/main/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitor.java
@@ -18,13 +18,12 @@
package org.apache.kylin.query.relnode.visitor;
-import java.math.BigDecimal;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.calcite.avatica.util.TimeUnitRange;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexCall;
@@ -39,6 +38,7 @@ import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.util.NlsString;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.filter.CaseTupleFilter;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -52,9 +52,11 @@ import org.apache.kylin.metadata.filter.function.Functions;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.relnode.ColumnRowType;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
+import java.math.BigDecimal;
+import java.util.GregorianCalendar;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
public class TupleFilterVisitor extends RexVisitorImpl {
@@ -146,10 +148,7 @@ public class TupleFilterVisitor extends
RexVisitorImpl {
}
if (op.getKind() == SqlKind.OR) {
-CompareTupleFilter inFilter = mergeToInClause(filter);
-if (inFilter != null) {
-filter = inFilter;
-}
+filter = mergeToInClause(filter);
} else if (op.getKind() == SqlKind.NOT) {
assert (filter.getChildren().size() == 1);
filter = filter.getChildren().get(0).reverse();
@@ -221,36 +220,51 @@ public class TupleFilterVisitor extends
RexVisitorImpl {
return constFilter;
}
-private CompareTupleFilter mergeToInClause(TupleFilter filter) {
+@VisibleForTesting
+static TupleFilter mergeToInClause(TupleFilter filter) {
List children = filter.getChildren();
-TblColRef inColumn = null;
-List inValues = new LinkedList();
-Map dynamicVariables = new HashMap<>();
+if (children.isEmpty()) {
+return filter;
+}
+// key: inColumn
+// Value: first: inValues
+// Value: second: dynamicVariables
+Map, Map>> inColumnMap =
Maps.newHashMap();
+List extraFilters = Lists.newLinkedList();
for (TupleFilter child : children) {
if (child.getOperator() == TupleFilter.FilterOperatorEnum.EQ) {
CompareTupleFilter compFilter = (CompareTupleFilter) child;
TblColRef column = compFilter.getColumn();
-if (inColumn == null) {
-inColumn = column;
-}
+if (column != null) {
+Pair, Map> tmpValue =
inColumnMap.get(column);
+if (tmpValue == null) {
+Set inValues = Sets.newHashSet();
+Map dynamicVariables =
Maps.newHashMap();
+tmpValue = new Pair<>(inValues, dynamicVariables);
+inColumnMap.put(column, tmpValue);
+}
-if (column == null || !column.equals(inColumn)) {
-return null;
+tmpValue.getFirst().addAll
[kylin] branch master updated: KYLIN-2725 fix cube desc signature issue
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 7437af8 KYLIN-2725 fix cube desc signature issue
7437af8 is described below
commit 7437af838c5fe1fc9eb2055cc5e9f81e5dc4aab5
Author: kyotoYaho
AuthorDate: Mon Jan 28 12:06:09 2019 +0800
KYLIN-2725 fix cube desc signature issue
---
.../java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java | 1 -
1 file changed, 1 deletion(-)
diff --git
a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
index 9eb5ab2..f525e44 100644
---
a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
+++
b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/CubeDescCreator.java
@@ -455,7 +455,6 @@ public class CubeDescCreator {
desc.setStorageType(storageType);
desc.setAggregationGroups(Lists.newArrayList(aggGroup));
desc.getOverrideKylinProps().putAll(overrideProperties);
-desc.setSignature(desc.calculateSignature());
desc.updateRandomUuid();
return desc;
}
[kylin] branch document updated: Add blog for v2.6.0 release
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch document in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/document by this push: new dc5d2b5 Add blog for v2.6.0 release dc5d2b5 is described below commit dc5d2b50f1f8a56b7a87196f2746ce5888d8af2d Author: kyotoYaho AuthorDate: Mon Jan 21 11:23:56 2019 +0800 Add blog for v2.6.0 release --- .../_posts/blog/2019-01-18-release-v2.6.0.cn.md| 76 ++ website/_posts/blog/2019-01-18-release-v2.6.0.md | 74 + 2 files changed, 150 insertions(+) diff --git a/website/_posts/blog/2019-01-18-release-v2.6.0.cn.md b/website/_posts/blog/2019-01-18-release-v2.6.0.cn.md new file mode 100644 index 000..2c3f812 --- /dev/null +++ b/website/_posts/blog/2019-01-18-release-v2.6.0.cn.md @@ -0,0 +1,76 @@ +--- +layout: post-blog +title: Apache Kylin v2.6.0 正式发布 +date: 2019-01-18 20:00:00 +author: Yanghong Zhong +categories: blog +--- + +近日Apache Kylin 社区很高兴地宣布，Apache Kylin 2.6.0 正式发布。 + +Apache Kylin 是一个开源的分布式分析引擎，旨在为极大数据集提供 SQL 接口和多维分析（OLAP）的能力。 + +这是继2.5.0 后的一个新功能版本。该版本引入了很多有价值的改进，完整的改动列表请参见[release notes](https://kylin.apache.org/docs/release_notes.html)；这里挑一些主要改进做说明： + +### 针对以JDBC为数据源的SDK +Kylin目前已经支持通过JDBC连接包括Amazon Redshift, SQL Server在内的多种数据源。 +为了便于开发者更便利地处理各种SQL dialect的不同以更加简单地开发新的基于JDBC的数据源，Kylin提供了相应的SDK和统一的API入口： +* 同步元数据和数据 +* 构建cube +* 当找不到相应的cube来解答查询时，下推查询到数据源 + +更多内容参见 KYLIN-3552。 + +### Memcached作Kylin的分布式缓存 +在过去，Kylin对查询结果的缓存不是十分高效，主要有以下两个方面的原因。 +一个是当Kylin的metadata发生变化时，会主动盲目地去删除大量有效的缓存，使得缓存会被频繁刷新而导致利用率很低。 +另一点是由于只使用本地缓存而导致Kylin server之间不能共享彼此的缓存，这样查询的缓存命中率就会降低。 +本地缓存还有一个缺点就是大小受到限制，不能像分布式缓存那样水平扩展。这样导致能缓存的查询结果量受到了限制。 + +针对这些缺陷，我们改变了缓存失效的机制，不再主动去清理缓存，而是采取如下的方案： +1. 在将查询结果放入缓存之前，根据当前的元数据信息计算一个数字签名，并与查询结果一同放入缓存中 +2. 从缓存中获取查询结果之后，根据当前的元数据信息计算一个数字签名，对比两者的数字签名是否一致。如果一致，那么缓存有效；反之，该缓存失效并删除 + +我们还引入了Memcached作为Kylin的分布式缓存。这样Kylin server之间可以共享查询结果的缓存，而且由于Memcached server之间的独立性，非常易于水平拓展，更加有利于缓存更多的数据。 +相关开发任务是KYLIN-2895, KYLIN-2894, KYLIN-2896, KYLIN-2897, KYLIN-2898, KYLIN-2899。 + +### ForkJoinPool简化fast cubing的线程模型 +在过去进行fast cubing时，Kylin使用自己定义的一系列线程，如split线程，task线程，main线程等等进行并发的cube构建。 +在这个线程模型中，线程之间的关系十分的复杂，而且对异常处理也十分容易出错。 + +现在我们引入了ForkJoinPool，在主线程中处理split逻辑，构建cuboid的任务以及子任务都在fork join pool中执行，cuboid构建的结果可以被异步的收集并且可以更早地输出给下游的merge操作。更多内容参见 KYLIN-2932。 + +### 改进HLLCounter的性能 +对于HLLCounter， 我们从两方面进行了改进：构建HLLCounter和计算调和平均的方式。 +1. 关于HLLCounter的构建，我们不再使用merge的方式，而是直接copy别的HLLCounter里面的registers +2. 关于计算HLLCSnapshot里面的调和平均，做了以下三个方面的改进： +* 缓存所有的1/2^r +* 使用整型相加代替浮点型相加 +* 删除条件分支，例如无需检查registers[i]是不是为0 + +更多内容参见 KYLIN-3656。 + +### 改进Cube Planner算法 +在过去，cube planner的phase two增加未被预计算的cuboid的方式只能通过mandatory cuboid的方式。而一个cuboid是否为mandatory，又有两种方式： +手动设置，查询时rollup的行数足够大。这里通过判断查询时rollup的行数是否足够大来判断是否为mandatory cuboid的方式有两大缺陷： +* 一个是估算rollup的行数的算法不是很好 +* 一个是很难设立一个静态的阈值来做判定 + +现在我们不再从rollup行数的角度看问题了。一切都是从cuboid行数的角度看问题，这样就和cost based的cube planner算法做了统一。 +为此我们通过使用rollup比率来改进了未被预先构建的cuboid的行数的估算，然后让cost based的cube planner算法来判定哪些未被构建的cuboid该被构建，哪些该被遗弃。 +通过这样的改进，无需通过设定静态的阈值来推荐mandatory cuboid了，而mandatory cuboid只能被手动设置，不能被推荐了。更多内容参见 KYLIN-3540。 + +__下载__ + +要下载Apache Kylin v2.6.0源代码或二进制包，请访问[下载页面](http://kylin.apache.org/download) . + +__升级__ + +参考[升级指南](/docs/howto/howto_upgrade.html). + +__反馈__ + +如果您遇到问题或疑问，请发送邮件至 Apache Kylin dev 或 user 邮件列表：d...@kylin.apache.org，u...@kylin.apache.org; 在发送之前，请确保您已通过发送电子邮件至 dev-subscr...@kylin.apache.org 或 user-subscr...@kylin.apache.org订阅了邮件列表。 + + +_非常感谢所有贡献Apache Kylin的朋友!_ \ No newline at end of file diff --git a/website/_posts/blog/2019-01-18-release-v2.6.0.md b/website/_posts/blog/2019-01-18-release-v2.6.0.md new file mode 100644 index 000..d01292b --- /dev/null +++ b/website/_posts/blog/2019-01-18-release-v2.6.0.md @@ -0,0 +1,74 @@ +--- +layout: post-blog +title: Apache Kylin v2.6.0 Release Announcement +date: 2019-01-18 20:00:00 +author: Yanghong Zhong +categories: blog +--- + +The Apache Kylin community is pleased to announce the release of Apache Kylin v2.6.0. + +Apache Kylin is an open source Distributed Analytics Engine designed to provide SQL interface and multi-dimensional analysis (OLAP) on Big Data supporting extremely large datasets. + +This is a major release after 2.5.0, including many enhancements. All of the changes can be found in the [release notes](https://kylin.apache.org/docs/release_notes.html). Here just highlight the major ones: + +### SDK for JDBC sources +Apache Kylin has already supported several data sources like Amazon Redshift, SQL Server through JDBC. +To help developers handle SQL dialect differences and easily implement a new data source through JDBC, Kylin provides a new data source SDK with APIs for: +* Synchronize metadata and data from JDBC source +* Build cube from JDBC source +* Query pushdown to JDBC source engine when cube is unmatched + +Check KYLIN-3552 for more. + +### Memcached as distributed cache
[kylin] branch document updated: Update release notes for v2.6.0
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch document in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/document by this push: new a740074 Update release notes for v2.6.0 a740074 is described below commit a7400749033bfbdb310b959ca61e8ea74abd4320 Author: kyotoYaho AuthorDate: Mon Jan 14 09:59:42 2019 +0800 Update release notes for v2.6.0 --- website/_docs/release_notes.md | 116 + 1 file changed, 116 insertions(+) diff --git a/website/_docs/release_notes.md b/website/_docs/release_notes.md index ee93382..f3ff179 100644 --- a/website/_docs/release_notes.md +++ b/website/_docs/release_notes.md @@ -15,6 +15,122 @@ or send to Apache Kylin mailing list: * User relative: [u...@kylin.apache.org](mailto:u...@kylin.apache.org) * Development relative: [d...@kylin.apache.org](mailto:d...@kylin.apache.org) +## v2.6.0 - 2019-01-12 +_Tag:_ [kylin-2.6.0](https://github.com/apache/kylin/tree/kylin-2.6.0) +This is a major release after 2.5, with more than 90 bug fixes and enhancements. + +__New Feature__ + +* [KYLIN-3552] - Data Source SDK to ingest data from different JDBC sources + +__Improvement__ + +* [KYLIN-] - Ignore unsupported hive column types when sync hive table +* [KYLIN-2861] - For dictionary building of lookup table columns, reduce the table scan chance +* [KYLIN-2895] - Refine query cache +* [KYLIN-2932] - Simplify the thread model for in-memory cubing +* [KYLIN-2972] - CacheKey from SQLRequest should ignore the case of project name +* [KYLIN-3005] - getAllDictColumnsOnFact in CubeManager may include fact table's foreign key +* [KYLIN-3021] - Check MapReduce job failed reason and include the diagnostics into email notification +* [KYLIN-3272] - Upgrade Spark dependency to 2.3.2 +* [KYLIN-3326] - Better way to update migration target cluster's metadata after cube migration +* [KYLIN-3430] - Global Dictionary Cleanup +* [KYLIN-3445] - Upgrade checkstyle version to 8.6 +* [KYLIN-3496] - Make calcite extras props available in JDBC Driver +* [KYLIN-3497] - Make JDBC Module more testable +* [KYLIN-3499] - Double check timestamp in HBase when got "RetriesExhaustedException" +* [KYLIN-3540] - Improve Mandatory Cuboid Recommendation Algorithm +* [KYLIN-3544] - Refine guava cache for the recommended cuboids of cube planner +* [KYLIN-3570] - Scripts to automatically build system cube +* [KYLIN-3592] - Synchronized should be placed after static in declaration +* [KYLIN-3597] - Fix sonar reported static code issues phase 1 +* [KYLIN-3600] - Utility classes should not have public constructors +* [KYLIN-3602] - Enable more checkstyle rules +* [KYLIN-3611] - Upgrade Tomcat to 7.0.91, 8.5.34 or later +* [KYLIN-3617] - Reduce number of visiting metastore for job scheduler +* [KYLIN-3628] - Query with lookup table always use latest snapshot +* [KYLIN-3630] - Remove unused fields in the implementations of MeasureType +* [KYLIN-3631] - Utilize Arrays#parallelSort for better performance +* [KYLIN-3640] - Cube with desc broken should be able to delete +* [KYLIN-3655] - Reinitialize CubeInstance when clear segments +* [KYLIN-3656] - Improve HLLCounter performance +* [KYLIN-3670] - Misspelled constant DEFAUL_JOB_CONF_SUFFIX +* [KYLIN-3671] - Improve ResourceTool and JDBCResourceStore's performance +* [KYLIN-3700] - Quote sql identities when creating flat table +* [KYLIN-3707] - Add configuration for setting isolation-level for sqoop +* [KYLIN-3720] - Add column family check when save/update cube desc +* [KYLIN-3729] - CLUSTER BY CAST(field AS STRING) will accelerate base cuboid build with UHC global dict +* [KYLIN-3737] - Refactor cache part for RDBMS +* [KYLIN-3749] - Add configuration to override xml for RDBMS + +__Bug Fix__ + +* [KYLIN-1819] - Exception swallowed when start DefaultScheduler fail +* [KYLIN-2841] - LIMIT pushdown should be applied to subquery +* [KYLIN-2973] - Potential issue of not atomically update cube instance map +* [KYLIN-3291] - 在构建好的cube上提交逻辑相同的sql查询结果不同 +* [KYLIN-3406] - When the /execute_output/ metadata file sinking to HDFS is deleted, the monitoring page does not display any tasks. +* [KYLIN-3416] - Kylin bitmap null pointer exception when "group by" is an expression +* [KYLIN-3439] - configuration "kylin.web.timezone" is Inconsistent between kylin-defaults.properties and KylinConfigBase.java +* [KYLIN-3515] - Cubing jobs may interfere with each other if use same hive view +* [KYLIN-3574] - Unclosed lookup table in DictionaryGeneratorCLI#processSegment +* [KYLIN-3575] - Unclosed Connection in DriverTest +* [KYLIN-3576] - IllegalArgumentException: No enum constant org.apache.kylin.rest.service.JobService.JobSearchMode. +* [KYLIN-3578] - Do not synchronize on the intrinsic locks of high-level concurrency objects +* [KYLIN-3579] - entrySet iterator should be used in
svn commit: r1851219 - in /kylin/site: docs/release_notes.html feed.xml
Author: nju_yaho Date: Mon Jan 14 02:03:05 2019 New Revision: 1851219 URL: http://svn.apache.org/viewvc?rev=1851219=rev Log: update release notes for v2.6.0 Modified: kylin/site/docs/release_notes.html kylin/site/feed.xml Modified: kylin/site/docs/release_notes.html URL: http://svn.apache.org/viewvc/kylin/site/docs/release_notes.html?rev=1851219=1851218=1851219=diff == --- kylin/site/docs/release_notes.html (original) +++ kylin/site/docs/release_notes.html Mon Jan 14 02:03:05 2019 @@ -6035,8 +6035,132 @@ there are source code package, binary pa or send to Apache Kylin mailing list: - User relative: - Development relative: + User relative: mailto:u...@kylin.apache.org;>u...@kylin.apache.org + Development relative: mailto:d...@kylin.apache.org;>d...@kylin.apache.org + + +v2.6.0 - 2019-01-12 +Tag: https://github.com/apache/kylin/tree/kylin-2.6.0;>kylin-2.6.0 +This is a major release after 2.5, with more than 90 bug fixes and enhancements. + +New Feature + + + [KYLIN-3552] - Data Source SDK to ingest data from different JDBC sources + + +Improvement + + + [KYLIN-] - Ignore unsupported hive column types when sync hive table + [KYLIN-2861] - For dictionary building of lookup table columns, reduce the table scan chance + [KYLIN-2895] - Refine query cache + [KYLIN-2932] - Simplify the thread model for in-memory cubing + [KYLIN-2972] - CacheKey from SQLRequest should ignore the case of project name + [KYLIN-3005] - getAllDictColumnsOnFact in CubeManager may include fact tableâs foreign key + [KYLIN-3021] - Check MapReduce job failed reason and include the diagnostics into email notification + [KYLIN-3272] - Upgrade Spark dependency to 2.3.2 + [KYLIN-3326] - Better way to update migration target clusterâs metadata after cube migration + [KYLIN-3430] - Global Dictionary Cleanup + [KYLIN-3445] - Upgrade checkstyle version to 8.6 + [KYLIN-3496] - Make calcite extras props available in JDBC Driver + [KYLIN-3497] - Make JDBC Module more testable + [KYLIN-3499] - Double check timestamp in HBase when got âRetriesExhaustedExceptionâ + [KYLIN-3540] - Improve Mandatory Cuboid Recommendation Algorithm + [KYLIN-3544] - Refine guava cache for the recommended cuboids of cube planner + [KYLIN-3570] - Scripts to automatically build system cube + [KYLIN-3592] - Synchronized should be placed after static in declaration + [KYLIN-3597] - Fix sonar reported static code issues phase 1 + [KYLIN-3600] - Utility classes should not have public constructors + [KYLIN-3602] - Enable more checkstyle rules + [KYLIN-3611] - Upgrade Tomcat to 7.0.91, 8.5.34 or later + [KYLIN-3617] - Reduce number of visiting metastore for job scheduler + [KYLIN-3628] - Query with lookup table always use latest snapshot + [KYLIN-3630] - Remove unused fields in the implementations of MeasureType + [KYLIN-3631] - Utilize Arrays#parallelSort for better performance + [KYLIN-3640] - Cube with desc broken should be able to delete + [KYLIN-3655] - Reinitialize CubeInstance when clear segments + [KYLIN-3656] - Improve HLLCounter performance + [KYLIN-3670] - Misspelled constant DEFAUL_JOB_CONF_SUFFIX + [KYLIN-3671] - Improve ResourceTool and JDBCResourceStoreâs performance + [KYLIN-3700] - Quote sql identities when creating flat table + [KYLIN-3707] - Add configuration for setting isolation-level for sqoop + [KYLIN-3720] - Add column family check when save/update cube desc + [KYLIN-3729] - CLUSTER BY CAST(field AS STRING) will accelerate base cuboid build with UHC global dict + [KYLIN-3737] - Refactor cache part for RDBMS + [KYLIN-3749] - Add configuration to override xml for RDBMS + + +Bug Fix + + + [KYLIN-1819] - Exception swallowed when start DefaultScheduler fail + [KYLIN-2841] - LIMIT pushdown should be applied to subquery + [KYLIN-2973] - Potential issue of not atomically update cube instance map + [KYLIN-3291] - å¨æ建好çcubeä¸æ交é»è¾ç¸åçsqlæ¥è¯¢ç»æä¸å + [KYLIN-3406] - When the /execute_output/ metadata file sinking to HDFS is deleted, the monitoring page does not display any tasks. + [KYLIN-3416] - Kylin bitmap null pointer exception when âgroup byâ is an expression + [KYLIN-3439] - configuration âkylin.web.timezoneâ is Inconsistent between kylin-defaults.properties and KylinConfigBase.java + [KYLIN-3515] - Cubing jobs may interfere with each other if use same hive view + [KYLIN-3574] - Unclosed lookup table in DictionaryGeneratorCLI#processSegment + [KYLIN-3575] - Unclosed Connection in DriverTest + [KYLIN-3576] - IllegalArgumentException: No enum constant org.apache.kylin.rest.service.JobService.JobSearchMode. + [KYLIN-3578] - Do not synchronize on the intrinsic locks of high-level concurrency objects + [KYLIN-3579] - entrySet iterator should be used in BPUSCalculator + [KYLIN-3580] - CuboidStatsUtil#complementRowCountForMandatoryCubo
svn commit: r1851193 - in /kylin/site: cn/download/index.html download/index.html feed.xml
Author: nju_yaho Date: Sun Jan 13 05:04:15 2019 New Revision: 1851193 URL: http://svn.apache.org/viewvc?rev=1851193=rev Log: update download documents Modified: kylin/site/cn/download/index.html kylin/site/download/index.html kylin/site/feed.xml Modified: kylin/site/cn/download/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn/download/index.html?rev=1851193=1851192=1851193=diff == --- kylin/site/cn/download/index.html (original) +++ kylin/site/cn/download/index.html Sun Jan 13 05:04:15 2019 @@ -174,6 +174,19 @@ var _hmt = _hmt || []; æ¨å¯ä»¥æç §è¿äºhttps://www.apache.org/info/verification.html;>æ¥éª¤ 并使ç¨è¿äºhttps://www.apache.org/dist/kylin/KEYS;>KEYSæ¥éªè¯ä¸è½½æ件çæææ§. +v2.6.0 + + è¿æ¯2.5 çæ¬åçä¸ä¸ªä¸»è¦åå¸çæ¬ï¼å å«äº94个é®é¢çä¿®å¤ä»¥ååç§æ¹è¿ãå ³äºå ·ä½å 容请æ¥çåå¸è¯´æ. + åå¸è¯´æ and å级æå + æºç ä¸è½½: https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip;>apache-kylin-2.6.0-source-release.zip [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.asc;>asc] [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256;>sha256] + äºè¿å¶å ä¸è½½: + + for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz;>apache-kylin-2.6.0-bin-hbase1x.tar.gz [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc;>asc] [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256;>sha256] + for CDH 5.7+ - https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz;>apache-kylin-2.6.0-bin-cdh57.tar.gz [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc;>asc] [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256;>sha256] + + + + v2.5.2 è¿æ¯2.5.1 çæ¬åçä¸ä¸ªbugfixåå¸çæ¬ï¼å å«äº12 个以ååç§æ¹è¿ãå ³äºå ·ä½å 容请æ¥çåå¸è¯´æ. @@ -193,18 +206,18 @@ var _hmt = _hmt || []; -JDBC 驱å¨ç¨åº +JDBC 驱å¨ç¨åº Kylin JDBC 驱å¨ç¨åº, kylin-jdbc-version.jar, å¨ Kylin äºè¿å¶å ç âlibâ ç®å½ä¸. -ODBC 驱å¨ç¨åº +ODBC 驱å¨ç¨åº http://kylin.apache.org/download/KylinODBCDriver-2.1.0.zip;>Kylin ODBC é©±å¨ v2.1.0 ï¼ä¸ææ Kylin çæ¬å ¼å®¹ï¼ 注: Kylin ODBC 驱å¨ä¾èµ http://www.microsoft.com/en-us/download/details.aspx?id=30679;>Microsoft Visual C++ 2012 Redistributableã -以åççæ¬ +以åççæ¬ Apache Kylinçæ§çæ¬å¯ä»¥ä» https://archive.apache.org/dist/kylin/;>https://archive.apache.org/dist/kylin/ ä¸è½½ã Modified: kylin/site/download/index.html URL: http://svn.apache.org/viewvc/kylin/site/download/index.html?rev=1851193=1851192=1851193=diff == --- kylin/site/download/index.html (original) +++ kylin/site/download/index.html Sun Jan 13 05:04:15 2019 @@ -6033,6 +6033,19 @@ var _hmt = _hmt || []; You can verify the download by following these https://www.apache.org/info/verification.html;>procedures and using these https://www.apache.org/dist/kylin/KEYS;>KEYS. +v2.6.0 + + This is a major release after 2.5, with 94 bug fixes and enhancement. Check the release notes. + Release notes and upgrade guide + Source download: https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip;>apache-kylin-2.6.0-source-release.zip [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.asc;>asc] [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256;>sha256] + Binary download: + + for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz;>apache-kylin-2.6.0-bin-hbase1x.tar.gz [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc;>asc] [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256;>sha256] + for CDH 5.7+ - https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz;>apache-kylin-2.6.0-bin-cdh57.tar.gz [https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc;>asc] [https://www.apache.org/dist/kyl
[kylin] 01/01: Update download document for v2.6
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch document in repository https://gitbox.apache.org/repos/asf/kylin.git commit e52160874ed5c7ca0e93fd7f46daa63976d1895f Author: kyotoYaho AuthorDate: Sun Jan 13 10:41:34 2019 +0800 Update download document for v2.6 --- website/download/index.cn.md | 7 +++ website/download/index.md| 8 2 files changed, 15 insertions(+) diff --git a/website/download/index.cn.md b/website/download/index.cn.md index f795a9d..98c73e5 100644 --- a/website/download/index.cn.md +++ b/website/download/index.cn.md @@ -5,6 +5,13 @@ title: 下载 您可以按照这些[步骤](https://www.apache.org/info/verification.html) 并使用这些[KEYS](https://www.apache.org/dist/kylin/KEYS)来验证下载文件的有效性. + v2.6.0 +- 这是2.5 版本后的一个主要发布版本，包含了94个问题的修复以及各种改进。关于具体内容请查看发布说明. +- [发布说明](/docs/release_notes.html) and [升级指南](/docs/howto/howto_upgrade.html) +- 源码下载: [apache-kylin-2.6.0-source-release.zip](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256)\] +- 二进制包下载: + - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - [apache-kylin-2.6.0-bin-hbase1x.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256)\] + - for CDH 5.7+ - [apache-kylin-2.6.0-bin-cdh57.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256)\] v2.5.2 - 这是2.5.1 版本后的一个bugfix发布版本，包含了12 个以及各种改进。关于具体内容请查看发布说明. diff --git a/website/download/index.md b/website/download/index.md index 53575c2..bbfa7c5 100644 --- a/website/download/index.md +++ b/website/download/index.md @@ -6,6 +6,14 @@ permalink: /download/index.html You can verify the download by following these [procedures](https://www.apache.org/info/verification.html) and using these [KEYS](https://www.apache.org/dist/kylin/KEYS). + v2.6.0 +- This is a major release after 2.5, with 94 bug fixes and enhancement. Check the release notes. +- [Release notes](/docs/release_notes.html) and [upgrade guide](/docs/howto/howto_upgrade.html) +- Source download: [apache-kylin-2.6.0-source-release.zip](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256)\] +- Binary download: + - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - [apache-kylin-2.6.0-bin-hbase1x.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256)\] + - for CDH 5.7+ - [apache-kylin-2.6.0-bin-cdh57.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256)\] + v2.5.2 - This is a bugfix release after 2.5.1, with 12 bug fixes and enhancement. Check the release notes. - [Release notes](/docs/release_notes.html) and [upgrade guide](/docs/howto/howto_upgrade.html)
[kylin] branch document updated (69193d6 -> e521608)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch document in repository https://gitbox.apache.org/repos/asf/kylin.git. discard 69193d6 Update download document for v2.6 new e521608 Update download document for v2.6 This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (69193d6) \ N -- N -- N refs/heads/document (e521608) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: website/download/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[kylin] branch document updated: Update download document for v2.6
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch document in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/document by this push: new 69193d6 Update download document for v2.6 69193d6 is described below commit 69193d62f2d4c89ac87c586fa76907c1deaf0f60 Author: kyotoYaho AuthorDate: Sun Jan 13 10:41:34 2019 +0800 Update download document for v2.6 --- website/download/index.cn.md | 7 +++ website/download/index.md| 8 2 files changed, 15 insertions(+) diff --git a/website/download/index.cn.md b/website/download/index.cn.md index f795a9d..98c73e5 100644 --- a/website/download/index.cn.md +++ b/website/download/index.cn.md @@ -5,6 +5,13 @@ title: 下载 您可以按照这些[步骤](https://www.apache.org/info/verification.html) 并使用这些[KEYS](https://www.apache.org/dist/kylin/KEYS)来验证下载文件的有效性. + v2.6.0 +- 这是2.5 版本后的一个主要发布版本，包含了94个问题的修复以及各种改进。关于具体内容请查看发布说明. +- [发布说明](/docs/release_notes.html) and [升级指南](/docs/howto/howto_upgrade.html) +- 源码下载: [apache-kylin-2.6.0-source-release.zip](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256)\] +- 二进制包下载: + - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - [apache-kylin-2.6.0-bin-hbase1x.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256)\] + - for CDH 5.7+ - [apache-kylin-2.6.0-bin-cdh57.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256)\] v2.5.2 - 这是2.5.1 版本后的一个bugfix发布版本，包含了12 个以及各种改进。关于具体内容请查看发布说明. diff --git a/website/download/index.md b/website/download/index.md index 53575c2..15222a2 100644 --- a/website/download/index.md +++ b/website/download/index.md @@ -6,6 +6,14 @@ permalink: /download/index.html You can verify the download by following these [procedures](https://www.apache.org/info/verification.html) and using these [KEYS](https://www.apache.org/dist/kylin/KEYS). + v2.6.0 +- This is a major release after 2.5, with 94 bug fixes and enhancement. Check the release notes. +- [Release notes](/docs/release_notes.html) and [upgrade guide](/docs/howto/howto_upgrade.html) +- Source download: [apache-kylin-2.6.0-source-release.zip] (https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256)\] +- Binary download: + - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6) - [apache-kylin-2.6.0-bin-hbase1x.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256)\] + - for CDH 5.7+ - [apache-kylin-2.6.0-bin-cdh57.tar.gz](https://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz) \[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc)\] \[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256)\] + v2.5.2 - This is a bugfix release after 2.5.1, with 12 bug fixes and enhancement. Check the release notes. - [Release notes](/docs/release_notes.html) and [upgrade guide](/docs/howto/howto_upgrade.html)
[kylin] 01/01: minor, update Kylin version to 2.6.1
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9c441baf96945e95993913181407b0371261f402
Author: kyotoYaho
AuthorDate: Sun Jan 13 03:27:09 2019 +0800
minor, update Kylin version to 2.6.1
---
core-common/src/main/java/org/apache/kylin/common/KylinVersion.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index bbdb3a8..b6432d4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -95,7 +95,7 @@ public class KylinVersion implements Comparable {
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
-private static final KylinVersion CURRENT_KYLIN_VERSION = new
KylinVersion("2.6.0.20500");
+private static final KylinVersion CURRENT_KYLIN_VERSION = new
KylinVersion("2.6.1");
private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");
[kylin] branch 2.6.x updated (601deae -> 9c441ba)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch 2.6.x in repository https://gitbox.apache.org/repos/asf/kylin.git. from 601deae KYLIN-3580 Avoid using entrySet to add elements add 8737bc1 [maven-release-plugin] prepare release kylin-2.6.0 add 179e7aa [maven-release-plugin] prepare for next development iteration new 9c441ba minor, update Kylin version to 2.6.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assembly/pom.xml | 2 +- cache/pom.xml| 5 ++--- core-common/pom.xml | 2 +- .../src/main/java/org/apache/kylin/common/KylinVersion.java | 2 +- core-cube/pom.xml| 2 +- core-dictionary/pom.xml | 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml| 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 6 ++ engine-mr/pom.xml| 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml| 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml | 9 - query/pom.xml| 2 +- server-base/pom.xml | 2 +- server/pom.xml | 2 +- source-hive/pom.xml | 2 +- source-jdbc/pom.xml | 5 ++--- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml| 8 +++- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml| 2 +- tool/pom.xml | 2 +- 28 files changed, 36 insertions(+), 43 deletions(-)
svn commit: r31925 - /release/kylin/apache-kylin-2.6.0/
Author: nju_yaho Date: Sat Jan 12 19:11:55 2019 New Revision: 31925 Log: add binary package Added: release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz - copied unchanged from r31924, dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc - copied unchanged from r31924, dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256 - copied unchanged from r31924, dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256 release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz - copied unchanged from r31924, dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc - copied unchanged from r31924, dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256 - copied unchanged from r31924, dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256
svn commit: r31924 - /dev/kylin/apache-kylin-2.6.0-rc1/
Author: nju_yaho Date: Sat Jan 12 18:46:40 2019 New Revision: 31924 Log: add binary package Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz (with props) dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc (with props) dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256 dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz (with props) dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc (with props) dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256 Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz == Binary file - no diff available. Propchange: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz -- svn:mime-type = application/x-gzip Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc == Binary file - no diff available. Propchange: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz.asc -- svn:mime-type = application/pgp-signature Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256 == --- dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256 (added) +++ dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-cdh57.tar.gz.sha256 Sat Jan 12 18:46:40 2019 @@ -0,0 +1 @@ +0b2c927a80026860931464489973c0d9cb71bec5788b142c9702ee1904808b9b apache-kylin-2.6.0-bin-cdh57.tar.gz Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz == Binary file - no diff available. Propchange: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz -- svn:mime-type = application/x-gzip Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc == Binary file - no diff available. Propchange: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz.asc -- svn:mime-type = application/pgp-signature Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256 == --- dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256 (added) +++ dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-bin-hbase1x.tar.gz.sha256 Sat Jan 12 18:46:40 2019 @@ -0,0 +1 @@ +71bcf6eb5db8646007464a63c979b093d70af9ed1821f799f8347a11a5332403 apache-kylin-2.6.0-bin-hbase1x.tar.gz
svn commit: r31922 - /release/kylin/apache-kylin-2.5.1/
Author: nju_yaho Date: Sat Jan 12 16:13:47 2019 New Revision: 31922 Log: Remove old release Removed: release/kylin/apache-kylin-2.5.1/
svn commit: r31921 - in /release/kylin/apache-kylin-2.6.0: ./ apache-kylin-2.6.0-source-release.zip apache-kylin-2.6.0-source-release.zip.asc apache-kylin-2.6.0-source-release.zip.sha256
Author: nju_yaho Date: Sat Jan 12 16:12:33 2019 New Revision: 31921 Log: checkin release artifacts Added: release/kylin/apache-kylin-2.6.0/ release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip (with props) release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.asc (with props) release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256 Added: release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip == Binary file - no diff available. Propchange: release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip -- svn:mime-type = application/zip Added: release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.asc == Binary file - no diff available. Propchange: release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.asc -- svn:mime-type = application/pgp-signature Added: release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256 == --- release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256 (added) +++ release/kylin/apache-kylin-2.6.0/apache-kylin-2.6.0-source-release.zip.sha256 Sat Jan 12 16:12:33 2019 @@ -0,0 +1 @@ +3621750945823ff4f0c4124b6d5b5c7164d9b08686729352ea22b2f486958d2a \ No newline at end of file
[kylin] branch 2.6.x updated (6220fa0 -> 601deae)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch 2.6.x in repository https://gitbox.apache.org/repos/asf/kylin.git. discard 6220fa0 [maven-release-plugin] prepare for next development iteration discard 40f0d74 [maven-release-plugin] prepare release kylin-2.6.0 This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/2.6.x (601deae) \ O -- O -- O (6220fa0) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: assembly/pom.xml | 2 +- cache/pom.xml | 5 +++-- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 6 -- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 9 + query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 5 +++-- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 8 +--- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 42 insertions(+), 35 deletions(-)
svn commit: r31826 - in /dev/kylin/apache-kylin-2.6.0-rc1: ./ apache-kylin-2.6.0-source-release.zip apache-kylin-2.6.0-source-release.zip.asc apache-kylin-2.6.0-source-release.zip.sha256
Author: nju_yaho Date: Wed Jan 9 00:44:15 2019 New Revision: 31826 Log: Upload release artifacts to staging Added: dev/kylin/apache-kylin-2.6.0-rc1/ dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip (with props) dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip.asc dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip.sha256 Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip == Binary file - no diff available. Propchange: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip.asc == --- dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip.asc (added) +++ dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip.asc Wed Jan 9 00:44:15 2019 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEZe+yJYO9IoWFYgdVHlTyBZSQahwFAlw1O+0ACgkQHlTyBZSQ +ahxC0RAArPUydsmj4lICufqGk5bDTlyQdOSZ8a+lUoey7KfWZ35l3tiriYPeOmFM +/YM66+EXD1nF8y22V+N1ni1U4GJZGDRgHGM+xNqV8VwW7SEzEkvYTcZFmOTr3H69 +5PvEz4MOCMFYQI4QY+Vu1iwlRGBBIOzPQmkLgFPH1j1Sw/EcZ1AzACSxowBkbH0s +vp1S59J2R/1PMfewfuCtAIYH78F6huERRD2yKX1FA3td3mXjGvn8wM8pWM2iGwyK +SHmFKaoMVw9QDyc9OTBQgwMjZwK6eRqNqatEQSyFTocULaGngIVZoa+WxwgpDgRI +GD3Nwe37Hq/IkBefatezv3Ijk3LhgsPrp7Db+RNetPxE2h0ME5HkFZgttnfHVXdP +hZaNxy1zUFfWawgfU5g0oU5WuVGY5yKg9L+GCrwiX67RBzAdKGkYhIXq5hXZcYgq +67tBgROPiJq7MUx6Ko6Po5ViVhB7PPZTi/QlJsxBeB8y7qYGdReZj33u/XhwXVt4 +e9NdGsMiXh/IUTllDVGcdeELHKOV6qYgA7KEyc/ICISfug608n8hpBcjSwvMXiOB +qi/k6fafe25z8rUpVrAGd6tdjpgdGJ7C6vTH1eHRQx2b07Xlj6rNsHJng1OOv+nW +immW/YBwYXfGMZ6OTVE7blW7nK+HYAXPVff9f8EwoQcrf/7dixQ= +=ONeu +-END PGP SIGNATURE- Added: dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip.sha256 == --- dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip.sha256 (added) +++ dev/kylin/apache-kylin-2.6.0-rc1/apache-kylin-2.6.0-source-release.zip.sha256 Wed Jan 9 00:44:15 2019 @@ -0,0 +1 @@ +3621750945823ff4f0c4124b6d5b5c7164d9b08686729352ea22b2f486958d2a \ No newline at end of file
[kylin] annotated tag kylin-2.6.0 updated (8737bc1 -> 749bba6)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to annotated tag kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.6.0 was modified! *** from 8737bc1 (commit) to 749bba6 (tag) tagging 8737bc1f555a2789a67462c8f8420b6ab3be97ce (commit) replaces kylin-1.2 by kyotoYaho on Wed Jan 9 08:12:27 2019 +0800 - Log - [maven-release-plugin] copy for tag kylin-2.6.0 --- No new revisions were added by this update. Summary of changes:
[kylin] branch v2.6.0-release created (now 601deae)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch v2.6.0-release in repository https://gitbox.apache.org/repos/asf/kylin.git. at 601deae KYLIN-3580 Avoid using entrySet to add elements No new revisions were added by this update.
[kylin] branch kylin-2.6.0 updated (601deae -> 40f0d74)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. from 601deae KYLIN-3580 Avoid using entrySet to add elements add 40f0d74 [maven-release-plugin] prepare release kylin-2.6.0 No new revisions were added by this update. Summary of changes: assembly/pom.xml | 2 +- cache/pom.xml | 5 ++--- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 6 ++ engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 11 +-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 5 ++--- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 8 +++- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 36 insertions(+), 43 deletions(-)
[kylin] annotated tag kylin-2.6.0 deleted (was 392a034)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to annotated tag kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.6.0 was deleted! *** tag was 392a034 The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[kylin] annotated tag kylin-2.6.0 updated (40f0d74 -> 392a034)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to annotated tag kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.6.0 was modified! *** from 40f0d74 (commit) to 392a034 (tag) tagging 40f0d74a318a5d3b3e2f58a78eef1071cd5a2182 (commit) replaces kylin-1.2 by kyotoYaho on Tue Jan 8 20:17:08 2019 +0800 - Log - [maven-release-plugin] copy for tag kylin-2.6.0 --- No new revisions were added by this update. Summary of changes:
[kylin] branch 2.6.x updated: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch 2.6.x in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/2.6.x by this push: new 6220fa0 [maven-release-plugin] prepare for next development iteration 6220fa0 is described below commit 6220fa036dd81b949a41b4ecd59ba7ddef99 Author: kyotoYaho AuthorDate: Tue Jan 8 20:17:15 2019 +0800 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml | 2 +- cache/pom.xml | 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 28 insertions(+), 28 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index f3b1fe5..6436c3b 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/cache/pom.xml b/cache/pom.xml index d9bf908..7e1972f 100644 --- a/cache/pom.xml +++ b/cache/pom.xml @@ -26,7 +26,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-common/pom.xml b/core-common/pom.xml index b489369..80ec1c6 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 209df43..f68e3e1 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index f42e44f..9bf9e7b 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-job/pom.xml b/core-job/pom.xml index d10d239..deb2723 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 1e5c75e..f282fb7 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index d45abdc..c3a35f6 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-storage/pom.xml b/core-storage/pom.xml index 19dbba0..721b506 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/datasource-sdk/pom.xml b/datasource-sdk/pom.xml index 5d3a8c1..dff36bd 100644 --- a/datasource-sdk/pom.xml +++ b/datasource-sdk/pom.xml @@ -26,7 +26,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index 10b9fbd..afe649b 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index f7d9419..89da5fa 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index c372e6e..e8cb362 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index e24134c..4c9796d 100644
[kylin] branch 2.6.x updated: [maven-release-plugin] prepare release kylin-2.6.0
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch 2.6.x in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/2.6.x by this push: new 40f0d74 [maven-release-plugin] prepare release kylin-2.6.0 40f0d74 is described below commit 40f0d74a318a5d3b3e2f58a78eef1071cd5a2182 Author: kyotoYaho AuthorDate: Tue Jan 8 20:16:50 2019 +0800 [maven-release-plugin] prepare release kylin-2.6.0 --- assembly/pom.xml | 2 +- cache/pom.xml | 5 ++--- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 6 ++ engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 11 +-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 5 ++--- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 8 +++- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 36 insertions(+), 43 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index dd3211a..f3b1fe5 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/cache/pom.xml b/cache/pom.xml index 8e31435..d9bf908 100644 --- a/cache/pom.xml +++ b/cache/pom.xml @@ -15,8 +15,7 @@ limitations under the License. --> -http://www.w3.org/2001/XMLSchema-instance; xmlns="http://maven.apache.org/POM/4.0.0; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> +http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> 4.0.0 kylin-cache @@ -27,7 +26,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-common/pom.xml b/core-common/pom.xml index 594e39b..b489369 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 12202b1a..209df43 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index 56e8c6e..f42e44f 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-job/pom.xml b/core-job/pom.xml index dbce6ef..d10d239 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 6ba698c..1e5c75e 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index b503629..d45abdc 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-storage/pom.xml b/core-storage/pom.xml index bee83e1..19dbba0 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/datasource-sdk/pom.xml b/datasource-sdk/pom.xml index 2649d15..5d3a8c1 100644 --- a/datasource-sdk/pom.xml +++ b/datasource-sdk/pom.xml @@ -17,9 +17,7 @@ limitations under the License. --> -http://maven.apache.org/POM/4.0.0; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> +http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; xsi:schem
[kylin] annotated tag kylin-2.6.0 deleted (was 13b2593)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to annotated tag kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.6.0 was deleted! *** tag was 13b2593 The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[kylin] branch kylin-2.6.0 updated (21bac74 -> 601deae)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. discard 21bac74 [maven-release-plugin] prepare for next development iteration discard f09c92f [maven-release-plugin] prepare release kylin-2.6.0 This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/kylin-2.6.0 (601deae) \ O -- O -- O (21bac74) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: assembly/pom.xml | 2 +- cache/pom.xml | 5 +++-- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 6 -- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 9 + query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 5 +++-- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 8 +--- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 42 insertions(+), 35 deletions(-)
[kylin] annotated tag kylin-2.6.0 updated (f09c92f -> 13b2593)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to annotated tag kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.6.0 was modified! *** from f09c92f (commit) to 13b2593 (tag) tagging f09c92f23652ff80a4166fd4faf6b220b8f50d80 (commit) replaces kylin-1.2 by kyotoYaho on Tue Jan 8 10:47:20 2019 +0800 - Log - [maven-release-plugin] copy for tag kylin-2.6.0 --- No new revisions were added by this update. Summary of changes:
[kylin] branch kylin-2.6.0 updated: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/kylin-2.6.0 by this push: new 21bac74 [maven-release-plugin] prepare for next development iteration 21bac74 is described below commit 21bac74d0126eca04841cf26c456830b3b3ff026 Author: kyotoYaho AuthorDate: Tue Jan 8 10:47:28 2019 +0800 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml | 2 +- cache/pom.xml | 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 28 insertions(+), 28 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index f3b1fe5..6436c3b 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/cache/pom.xml b/cache/pom.xml index d9bf908..7e1972f 100644 --- a/cache/pom.xml +++ b/cache/pom.xml @@ -26,7 +26,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-common/pom.xml b/core-common/pom.xml index b489369..80ec1c6 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 209df43..f68e3e1 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index f42e44f..9bf9e7b 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-job/pom.xml b/core-job/pom.xml index d10d239..deb2723 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 1e5c75e..f282fb7 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index d45abdc..c3a35f6 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-storage/pom.xml b/core-storage/pom.xml index 19dbba0..721b506 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/datasource-sdk/pom.xml b/datasource-sdk/pom.xml index 5d3a8c1..dff36bd 100644 --- a/datasource-sdk/pom.xml +++ b/datasource-sdk/pom.xml @@ -26,7 +26,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index 10b9fbd..afe649b 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index f7d9419..89da5fa 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index c372e6e..e8cb362 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index e24134c..4c9796d
[kylin] branch kylin-2.6.0 updated: [maven-release-plugin] prepare release kylin-2.6.0
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/kylin-2.6.0 by this push: new f09c92f [maven-release-plugin] prepare release kylin-2.6.0 f09c92f is described below commit f09c92f23652ff80a4166fd4faf6b220b8f50d80 Author: kyotoYaho AuthorDate: Tue Jan 8 10:47:01 2019 +0800 [maven-release-plugin] prepare release kylin-2.6.0 --- assembly/pom.xml | 2 +- cache/pom.xml | 5 ++--- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 6 ++ engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 11 +-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 5 ++--- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 8 +++- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 36 insertions(+), 43 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index dd3211a..f3b1fe5 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/cache/pom.xml b/cache/pom.xml index 8e31435..d9bf908 100644 --- a/cache/pom.xml +++ b/cache/pom.xml @@ -15,8 +15,7 @@ limitations under the License. --> -http://www.w3.org/2001/XMLSchema-instance; xmlns="http://maven.apache.org/POM/4.0.0; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> +http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> 4.0.0 kylin-cache @@ -27,7 +26,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-common/pom.xml b/core-common/pom.xml index 594e39b..b489369 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 12202b1a..209df43 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index 56e8c6e..f42e44f 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-job/pom.xml b/core-job/pom.xml index dbce6ef..d10d239 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 6ba698c..1e5c75e 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index b503629..d45abdc 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-storage/pom.xml b/core-storage/pom.xml index bee83e1..19dbba0 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/datasource-sdk/pom.xml b/datasource-sdk/pom.xml index 2649d15..5d3a8c1 100644 --- a/datasource-sdk/pom.xml +++ b/datasource-sdk/pom.xml @@ -17,9 +17,7 @@ limitations under the License. --> -http://maven.apache.org/POM/4.0.0; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> +http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; xsi:schem
[kylin] branch kylin-2.6.0 created (now 601deae)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. at 601deae KYLIN-3580 Avoid using entrySet to add elements No new revisions were added by this update.
[kylin] branch kylin-2.6.0 deleted (was 601deae)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. was 601deae KYLIN-3580 Avoid using entrySet to add elements The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[kylin] branch kylin-2.6.0 updated (d503ffc -> 601deae)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. discard d503ffc [maven-release-plugin] prepare for next development iteration discard 34c306c [maven-release-plugin] prepare release kylin-2.6.0 This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/kylin-2.6.0 (601deae) \ O -- O -- O (d503ffc) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: assembly/pom.xml | 2 +- cache/pom.xml | 5 +++-- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 6 -- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 9 + query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 5 +++-- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 8 +--- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 42 insertions(+), 35 deletions(-)
[kylin] branch kylin-2.6.0 updated: [maven-release-plugin] prepare release kylin-2.6.0
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/kylin-2.6.0 by this push: new 34c306c [maven-release-plugin] prepare release kylin-2.6.0 34c306c is described below commit 34c306c502a04add22f8dabcde52a638cd5fbb11 Author: kyotoYaho AuthorDate: Mon Jan 7 23:56:19 2019 +0800 [maven-release-plugin] prepare release kylin-2.6.0 --- assembly/pom.xml | 2 +- cache/pom.xml | 5 ++--- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 6 ++ engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 11 +-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 5 ++--- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 8 +++- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 36 insertions(+), 43 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index dd3211a..f3b1fe5 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/cache/pom.xml b/cache/pom.xml index 8e31435..d9bf908 100644 --- a/cache/pom.xml +++ b/cache/pom.xml @@ -15,8 +15,7 @@ limitations under the License. --> -http://www.w3.org/2001/XMLSchema-instance; xmlns="http://maven.apache.org/POM/4.0.0; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> +http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> 4.0.0 kylin-cache @@ -27,7 +26,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-common/pom.xml b/core-common/pom.xml index 594e39b..b489369 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 12202b1a..209df43 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index 56e8c6e..f42e44f 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-job/pom.xml b/core-job/pom.xml index dbce6ef..d10d239 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 6ba698c..1e5c75e 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index b503629..d45abdc 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/core-storage/pom.xml b/core-storage/pom.xml index bee83e1..19dbba0 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0-SNAPSHOT +2.6.0 diff --git a/datasource-sdk/pom.xml b/datasource-sdk/pom.xml index 2649d15..5d3a8c1 100644 --- a/datasource-sdk/pom.xml +++ b/datasource-sdk/pom.xml @@ -17,9 +17,7 @@ limitations under the License. --> -http://maven.apache.org/POM/4.0.0; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd;> +http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; xsi:schem
[kylin] annotated tag kylin-2.6.0 updated (34c306c -> 73f23a0)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to annotated tag kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git. *** WARNING: tag kylin-2.6.0 was modified! *** from 34c306c (commit) to 73f23a0 (tag) tagging 34c306c502a04add22f8dabcde52a638cd5fbb11 (commit) replaces kylin-1.2 by kyotoYaho on Mon Jan 7 23:56:36 2019 +0800 - Log - [maven-release-plugin] copy for tag kylin-2.6.0 --- No new revisions were added by this update. Summary of changes:
[kylin] branch kylin-2.6.0 updated: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch kylin-2.6.0 in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/kylin-2.6.0 by this push: new d503ffc [maven-release-plugin] prepare for next development iteration d503ffc is described below commit d503ffc876239d8c88061fabe8deef780b367da5 Author: kyotoYaho AuthorDate: Mon Jan 7 23:56:43 2019 +0800 [maven-release-plugin] prepare for next development iteration --- assembly/pom.xml | 2 +- cache/pom.xml | 2 +- core-common/pom.xml| 2 +- core-cube/pom.xml | 2 +- core-dictionary/pom.xml| 2 +- core-job/pom.xml | 2 +- core-metadata/pom.xml | 2 +- core-metrics/pom.xml | 2 +- core-storage/pom.xml | 2 +- datasource-sdk/pom.xml | 2 +- engine-mr/pom.xml | 2 +- engine-spark/pom.xml | 2 +- jdbc/pom.xml | 2 +- kylin-it/pom.xml | 2 +- metrics-reporter-hive/pom.xml | 2 +- metrics-reporter-kafka/pom.xml | 2 +- pom.xml| 4 ++-- query/pom.xml | 2 +- server-base/pom.xml| 2 +- server/pom.xml | 2 +- source-hive/pom.xml| 2 +- source-jdbc/pom.xml| 2 +- source-kafka/pom.xml | 2 +- storage-hbase/pom.xml | 2 +- tomcat-ext/pom.xml | 2 +- tool-assembly/pom.xml | 2 +- tool/pom.xml | 2 +- 27 files changed, 28 insertions(+), 28 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index f3b1fe5..6436c3b 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -26,7 +26,7 @@ kylin org.apache.kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/cache/pom.xml b/cache/pom.xml index d9bf908..7e1972f 100644 --- a/cache/pom.xml +++ b/cache/pom.xml @@ -26,7 +26,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-common/pom.xml b/core-common/pom.xml index b489369..80ec1c6 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 209df43..f68e3e1 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml index f42e44f..9bf9e7b 100644 --- a/core-dictionary/pom.xml +++ b/core-dictionary/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-job/pom.xml b/core-job/pom.xml index d10d239..deb2723 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 1e5c75e..f282fb7 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-metrics/pom.xml b/core-metrics/pom.xml index d45abdc..c3a35f6 100644 --- a/core-metrics/pom.xml +++ b/core-metrics/pom.xml @@ -28,7 +28,7 @@ kylin org.apache.kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/core-storage/pom.xml b/core-storage/pom.xml index 19dbba0..721b506 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/datasource-sdk/pom.xml b/datasource-sdk/pom.xml index 5d3a8c1..dff36bd 100644 --- a/datasource-sdk/pom.xml +++ b/datasource-sdk/pom.xml @@ -26,7 +26,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index 10b9fbd..afe649b 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index f7d9419..89da5fa 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index c372e6e..e8cb362 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -28,7 +28,7 @@ org.apache.kylin kylin -2.6.0 +2.6.1-SNAPSHOT diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index e24134c..4c9796d
[kylin] branch v2.6.0-release deleted (was 601deae)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch v2.6.0-release in repository https://gitbox.apache.org/repos/asf/kylin.git. was 601deae KYLIN-3580 Avoid using entrySet to add elements The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[kylin] branch v2.6.0-release created (now 601deae)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch v2.6.0-release in repository https://gitbox.apache.org/repos/asf/kylin.git. at 601deae KYLIN-3580 Avoid using entrySet to add elements No new revisions were added by this update.
[kylin] branch master updated: KYLIN-3580 Avoid using entrySet to add elements
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 601deae KYLIN-3580 Avoid using entrySet to add elements
601deae is described below
commit 601deae4e72203c3309023848d70cd95941029c2
Author: kyotoYaho
AuthorDate: Mon Jan 7 20:38:19 2019 +0800
KYLIN-3580 Avoid using entrySet to add elements
---
.../java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
index ec2f496..d9aaf54 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
@@ -150,7 +150,10 @@ public class CuboidStatsUtil {
return ret == 0 ? o1.getKey().compareTo(o2.getKey()) : ret;
}
});
-sortedStatsSet.addAll(statistics.entrySet());
+//sortedStatsSet.addAll(statistics.entrySet()); KYLIN-3580
+for (Map.Entry entry : statistics.entrySet()) {
+sortedStatsSet.add(entry);
+}
for (Long cuboid : cuboids) {
if (statistics.get(cuboid) == null) {
// Get estimate row count for mandatory cuboid
[kylin] branch master updated: KYLIN-3725 disable javadoc check for all profiles
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/master by this push: new 5c9c3df KYLIN-3725 disable javadoc check for all profiles 5c9c3df is described below commit 5c9c3dfb8dbd665d864a036f9cbbc845edc2c172 Author: kyotoYaho AuthorDate: Mon Jan 7 15:39:42 2019 +0800 KYLIN-3725 disable javadoc check for all profiles --- pom.xml | 14 +++--- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 4db72bd..1bca3f2 100644 --- a/pom.xml +++ b/pom.xml @@ -1262,6 +1262,13 @@ +org.apache.maven.plugins +maven-javadoc-plugin + + org.apache.kylin.* + + + net.alchim31.maven scala-maven-plugin @@ -1382,13 +1389,6 @@ -org.apache.maven.plugins -maven-javadoc-plugin - - org.apache.kylin.* - - - de.thetaphi forbiddenapis
[kylin] branch master updated: KYLIN-2972 minor fix
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 1f36967 KYLIN-2972 minor fix
1f36967 is described below
commit 1f369678c78c80c7a4cb1fb2a3e5902cb6171126
Author: kyotoYaho
AuthorDate: Mon Jan 7 10:50:11 2019 +0800
KYLIN-2972 minor fix
---
.../src/main/java/org/apache/kylin/rest/request/SQLRequest.java| 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index 2672a19..f96f0da 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.request;
import java.io.Serializable;
+import java.util.Locale;
import java.util.Map;
import com.google.common.collect.Lists;
@@ -149,6 +150,6 @@ public class SQLRequest implements Serializable {
}
private String getNormProject() {
-return project == null ? null : project.toUpperCase();
+return project == null ? null : project.toUpperCase(Locale.ROOT);
}
}
[kylin] branch master updated: KYLIN-3740 fix NPE during step of Extract Fact Table Distinct Columns
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new b780759 KYLIN-3740 fix NPE during step of Extract Fact Table Distinct
Columns
b780759 is described below
commit b780759c6641f0dfbd57f91d4f3d568b8dbb46f6
Author: kyotoYaho
AuthorDate: Mon Jan 7 09:48:59 2019 +0800
KYLIN-3740 fix NPE during step of Extract Fact Table Distinct Columns
---
.../engine/mr/steps/FactDistinctColumnsJob.java| 22 +-
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
index 8f5d176..6f78de1 100755
---
a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
+++
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
@@ -78,6 +78,19 @@ public class FactDistinctColumnsJob extends
AbstractHadoopJob {
// add metadata to distributed cache
CubeManager cubeMgr =
CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cube = cubeMgr.getCube(cubeName);
+CubeSegment segment = cube.getSegmentById(segmentID);
+if (segment == null) {
+logger.warn("Failed to find segment {} in cube {}", segmentID,
cube);
+cube = cubeMgr.reloadCubeQuietly(cubeName);
+segment = cube.getSegmentById(segmentID);
+}
+if (segment == null) {
+logger.error("Failed to find {} in cube {}", segmentID, cube);
+for (CubeSegment s : cube.getSegments()) {
+logger.error(s.getName() + " with status " +
s.getStatus());
+}
+throw new IllegalStateException();
+}
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
job.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID,
segmentID);
@@ -88,15 +101,6 @@ public class FactDistinctColumnsJob extends
AbstractHadoopJob {
setJobClasspath(job, cube.getConfig());
-CubeSegment segment = cube.getSegmentById(segmentID);
-if (segment == null) {
-logger.error("Failed to find {} in cube {}", segmentID, cube);
-for (CubeSegment s : cube.getSegments()) {
-logger.error(s.getName() + " with status " +
s.getStatus());
-}
-throw new IllegalStateException();
-}
-
setupMapper(segment);
setupReducer(output, segment);
[kylin] branch master updated: KYLIN-2898 update LICENSE for couchbase/spymemcached
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git The following commit(s) were added to refs/heads/master by this push: new cc00df6 KYLIN-2898 update LICENSE for couchbase/spymemcached cc00df6 is described below commit cc00df69e2459686d71c7852bcbc94fd14213555 Author: kyotoYaho AuthorDate: Fri Dec 28 11:32:53 2018 +0800 KYLIN-2898 update LICENSE for couchbase/spymemcached --- LICENSE | 7 +++ 1 file changed, 7 insertions(+) diff --git a/LICENSE b/LICENSE index 26e015c..9713d1a 100644 --- a/LICENSE +++ b/LICENSE @@ -270,3 +270,10 @@ For HadoopMetrics2Reporter: == HadoopMetrics2Reporter is licensed under Apache 2.0 License. https://github.com/joshelser/dropwizard-hadoop-metrics2 + + +== +For couchbase/spymemcached: +== +couchbase/spymemcached is licensed under MIT License. +https://github.com/couchbase/spymemcached/blob/master/LICENSE.txt \ No newline at end of file
[kylin] 03/03: KYLIN-3540 estimate the row counts of source cuboids which are not built & remove mandatory cuboids recommendation
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4850dacece8a2e90f4333b50e2e8304635a730a2
Author: kyotoYaho
AuthorDate: Tue Dec 18 18:54:51 2018 +0800
KYLIN-3540 estimate the row counts of source cuboids which are not built &
remove mandatory cuboids recommendation
---
.../org/apache/kylin/common/KylinConfigBase.java | 18 +-
.../cube/cuboid/algorithm/BPUSCalculator.java | 2 +-
.../cube/cuboid/algorithm/CuboidRecommender.java | 25 ++-
.../kylin/cube/cuboid/algorithm/CuboidStats.java | 131 ++--
.../cube/cuboid/algorithm/CuboidStatsUtil.java | 226 -
.../cube/cuboid/algorithm/CuboidStatsUtilTest.java | 162 ---
.../engine/mr/common/CuboidRecommenderUtil.java| 44 +++-
.../engine/mr/common/CuboidStatsReaderUtil.java| 23 +++
.../kylin/rest/controller/CubeController.java | 4 +-
.../org/apache/kylin/rest/service/CubeService.java | 26 ++-
10 files changed, 495 insertions(+), 166 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f67f6b3..b63062e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -316,8 +316,7 @@ abstract public class KylinConfigBase implements
Serializable {
public String getReadHdfsWorkingDirectory() {
if (StringUtils.isNotEmpty(getHBaseClusterFs())) {
Path workingDir = new Path(getHdfsWorkingDirectory());
-return new Path(getHBaseClusterFs(),
Path.getPathWithoutSchemeAndAuthority(workingDir)).toString()
-+ "/";
+return new Path(getHBaseClusterFs(),
Path.getPathWithoutSchemeAndAuthority(workingDir)).toString() + "/";
}
return getHdfsWorkingDirectory();
@@ -644,8 +643,12 @@ abstract public class KylinConfigBase implements
Serializable {
return
Integer.parseInt(getOptional("kylin.cube.cubeplanner.recommend-cache-max-size",
"200"));
}
-public long getCubePlannerMandatoryRollUpThreshold() {
-return
Long.parseLong(getOptional("kylin.cube.cubeplanner.mandatory-rollup-threshold",
"1000"));
+public double getCubePlannerQueryUncertaintyRatio() {
+return
Double.parseDouble(getOptional("kylin.cube.cubeplanner.query-uncertainty-ratio",
"0.1"));
+}
+
+public double getCubePlannerBPUSMinBenefitRatio() {
+return
Double.parseDouble(getOptional("kylin.cube.cubeplanner.bpus-min-benefit-ratio",
"0.01"));
}
public int getCubePlannerAgreedyAlgorithmAutoThreshold() {
@@ -1910,12 +1913,13 @@ abstract public class KylinConfigBase implements
Serializable {
}
public int getSmallCellMetadataWarningThreshold() {
-return
Integer.parseInt(getOptional("kylin.metadata.jdbc.small-cell-meta-size-warning-threshold",
-String.valueOf(100 << 20))); //100mb
+return Integer.parseInt(
+
getOptional("kylin.metadata.jdbc.small-cell-meta-size-warning-threshold",
String.valueOf(100 << 20))); //100mb
}
public int getSmallCellMetadataErrorThreshold() {
-return
Integer.parseInt(getOptional("kylin.metadata.jdbc.small-cell-meta-size-error-threshold",
String.valueOf(1 << 30))); // 1gb
+return Integer.parseInt(
+
getOptional("kylin.metadata.jdbc.small-cell-meta-size-error-threshold",
String.valueOf(1 << 30))); // 1gb
}
public int getJdbcResourceStoreMaxCellSize() {
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
index 6316858..39c52da 100755
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
@@ -142,7 +142,7 @@ public class BPUSCalculator implements BenefitPolicy {
}
public double getMinBenefitRatio() {
-return 0.01;
+return cuboidStats.getBpusMinBenefitRatio();
}
@Override
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
index baacb51..0e6a844 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
@@ -154,12 +154,11 @@ public class CuboidRecommender {
[kylin] 01/03: KYLIN-3540 move queryService of CubeController to CubeService
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8cfe32cf4f61d218439e57e134ccc3413aa98f89
Author: kyotoYaho
AuthorDate: Wed Dec 19 10:29:38 2018 +0800
KYLIN-3540 move queryService of CubeController to CubeService
---
.../kylin/rest/controller/CubeController.java | 71 ++---
.../org/apache/kylin/rest/service/CubeService.java | 88 ++
2 files changed, 78 insertions(+), 81 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index dcaa259..11753ec 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -56,8 +56,6 @@ import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metrics.MetricsManager;
-import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -70,7 +68,6 @@ import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
import org.apache.kylin.rest.request.LookupSnapshotBuildRequest;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
@@ -80,7 +77,6 @@ import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.rest.service.ProjectService;
-import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.util.ValidateUtil;
import org.apache.kylin.source.kafka.util.KafkaClient;
import org.slf4j.Logger;
@@ -124,10 +120,6 @@ public class CubeController extends BasicController {
@Qualifier("projectService")
private ProjectService projectService;
-@Autowired
-@Qualifier("queryService")
-private QueryService queryService;
-
@RequestMapping(value = "/validate/{cubeName}", method =
RequestMethod.GET, produces = { "application/json" })
@ResponseBody
public EnvelopeResponse validateModelName(@PathVariable String
cubeName) {
@@ -873,7 +865,7 @@ public class CubeController extends BasicController {
Map queryMatchMap = null;
try {
hitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
-queryMatchMap = getCuboidQueryMatchCount(cubeName);
+queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName);
} catch (Exception e) {
logger.warn("Fail to query on system cube due to " + e);
}
@@ -899,7 +891,7 @@ public class CubeController extends BasicController {
// Get cuboid target info for displaying heat map of cuboid hit
Map displayHitFrequencyMap =
getTargetCuboidHitFrequency(cubeName);
// Get exactly matched cuboid query count
-Map queryMatchMap = getCuboidQueryMatchCount(cubeName);
+Map queryMatchMap =
cubeService.getCuboidQueryMatchCount(cubeName);
Set currentCuboidSet =
cube.getCuboidScheduler().getAllCuboidIds();
return cubeService.getCuboidTreeResponse(cuboidScheduler,
recommendCuboidStatsMap, displayHitFrequencyMap,
@@ -909,69 +901,16 @@ public class CubeController extends BasicController {
private Map getRecommendCuboidList(CubeInstance cube) throws
IOException {
// Get cuboid source info
Map optimizeHitFrequencyMap =
getSourceCuboidHitFrequency(cube.getName());
-Map> rollingUpCountSourceMap =
getCuboidRollingUpCount(cube.getName());
+Map> rollingUpCountSourceMap =
cubeService.getCuboidRollingUpStats(cube.getName());
return cubeService.getRecommendCuboidStatistics(cube,
optimizeHitFrequencyMap, rollingUpCountSourceMap);
}
private Map getSourceCuboidHitFrequency(String cubeName) {
-return getCuboidHitFrequency(cubeName, true);
+return cubeService.getCuboidHitFrequency(cubeName, true);
}
private Map getTargetCuboidHitFrequency(String cubeName) {
-return getCuboidHitFrequency(cubeName, false);
-}
-
-private Map getCuboidHitFrequency(String cubeName, boolean
isCuboidSource) {
-SQLRequest sqlRe
[kylin] branch master updated (0c92d1c -> 4850dac)
This is an automated email from the ASF dual-hosted git repository. nju_yaho pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git. from 0c92d1c minor, fix format in pom new 8cfe32c KYLIN-3540 move queryService of CubeController to CubeService new 4db6a37 KYLIN-3540 refactor the interface of querying on SYSTEM project new 4850dac KYLIN-3540 estimate the row counts of source cuboids which are not built & remove mandatory cuboids recommendation The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/kylin/common/KylinConfigBase.java | 18 +- .../cube/cuboid/algorithm/BPUSCalculator.java | 2 +- .../cube/cuboid/algorithm/CuboidRecommender.java | 25 ++- .../kylin/cube/cuboid/algorithm/CuboidStats.java | 131 ++-- .../cube/cuboid/algorithm/CuboidStatsUtil.java | 226 - .../cube/cuboid/algorithm/CuboidStatsUtilTest.java | 162 --- .../engine/mr/common/CuboidRecommenderUtil.java| 44 +++- .../engine/mr/common/CuboidStatsReaderUtil.java| 23 +++ .../kylin/rest/controller/CubeController.java | 73 +-- .../kylin/rest/controller/DashboardController.java | 66 +++--- .../org/apache/kylin/rest/service/CubeService.java | 85 ++-- .../kylin/rest/service/DashboardService.java | 34 ++-- .../apache/kylin/rest/service/QueryService.java| 14 +- 13 files changed, 617 insertions(+), 286 deletions(-)
[kylin] 02/03: KYLIN-3540 refactor the interface of querying on SYSTEM project
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4db6a37c7220c122cedb9fac1a2c735f10e27226
Author: kyotoYaho
AuthorDate: Wed Dec 19 10:40:33 2018 +0800
KYLIN-3540 refactor the interface of querying on SYSTEM project
---
.../kylin/rest/controller/DashboardController.java | 66 --
.../org/apache/kylin/rest/service/CubeService.java | 27 +++--
.../kylin/rest/service/DashboardService.java | 34 ++-
.../apache/kylin/rest/service/QueryService.java| 14 -
4 files changed, 73 insertions(+), 68 deletions(-)
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
index ee9fdcd..846d6d3 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
@@ -16,15 +16,12 @@
* limitations under the License.
*/
-
package org.apache.kylin.rest.controller;
import java.util.List;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metrics.MetricsManager;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.CubeService;
@@ -57,62 +54,69 @@ public class DashboardController extends BasicController {
@RequestMapping(value = "/metric/cube", method = { RequestMethod.GET })
@ResponseBody
-public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName",
required = false) String projectName, @RequestParam(value = "cubeName",
required = false) String cubeName) {
+public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName",
required = false) String projectName,
+@RequestParam(value = "cubeName", required = false) String
cubeName) {
checkAuthorization(projectName);
return dashboardService.getCubeMetrics(projectName, cubeName);
}
@RequestMapping(value = "/metric/query", method = RequestMethod.GET)
@ResponseBody
-public MetricsResponse getQueryMetrics(@RequestParam(value =
"projectName", required = false) String projectName, @RequestParam(value =
"cubeName", required = false) String cubeName, @RequestParam(value =
"startTime") String startTime, @RequestParam(value = "endTime") String endTime)
{
+public MetricsResponse getQueryMetrics(@RequestParam(value =
"projectName", required = false) String projectName,
+@RequestParam(value = "cubeName", required = false) String
cubeName,
+@RequestParam(value = "startTime") String startTime,
@RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
MetricsResponse queryMetrics = new MetricsResponse();
-SQLRequest sqlRequest = new SQLRequest();
-sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String sql = dashboardService.getQueryMetricsSQL(startTime, endTime,
projectName, cubeName);
-sqlRequest.setSql(sql);
-SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
-if(!sqlResponse.getIsException()){
-queryMetrics.increase("queryCount",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
-queryMetrics.increase("avgQueryLatency",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
-queryMetrics.increase("maxQueryLatency",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
-queryMetrics.increase("minQueryLatency",
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+SQLResponse sqlResponse = queryService.querySystemCube(sql);
+if (!sqlResponse.getIsException()) {
+queryMetrics.increase("queryCount",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
+queryMetrics.increase("avgQueryLatency",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+queryMetrics.increase("maxQueryLatency",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+queryMetrics.increase("minQueryLatency",
+
dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
}
return queryMetrics;
}
@RequestMapping(value = "/metr
[kylin] branch master updated: KYLIN-3365 Add unit test for the coprocessor code, CubeVisitService
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 91efa0c KYLIN-3365 Add unit test for the coprocessor code,
CubeVisitService
91efa0c is described below
commit 91efa0cff4c23246dd4e9c8eee87b7aeee54837e
Author: kyotoYaho
AuthorDate: Wed Dec 19 14:49:32 2018 +0800
KYLIN-3365 Add unit test for the coprocessor code, CubeVisitService
---
storage-hbase/pom.xml | 13 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java| 4 +-
.../coprocessor/endpoint/CubeVisitServiceTest.java | 568 +
3 files changed, 581 insertions(+), 4 deletions(-)
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 190dd8c..167280d 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -17,7 +17,8 @@
limitations under the License.
-->
-http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
4.0.0
kylin-storage-hbase
@@ -65,6 +66,13 @@
test
+org.apache.hadoop
+hadoop-hdfs
+${hadoop2.version}
+test-jar
+test
+
+
org.apache.hbase
hbase-common
provided
@@ -135,7 +143,8 @@
true
coprocessor
-
+
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index c9be666..d1aee2f 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -427,7 +427,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
}
-private ByteString serializeGTScanReq(GTScanRequest scanRequest) {
+public static ByteString serializeGTScanReq(GTScanRequest scanRequest) {
ByteString scanRequestByteString;
int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
while (true) {
@@ -445,7 +445,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
return scanRequestByteString;
}
-private ByteString serializeRawScans(List rawScans) {
+public static ByteString serializeRawScans(List rawScans) {
ByteString rawScanByteString;
int rawScanBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE;
while (true) {
diff --git
a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
new file mode 100644
index 000..b3e5a93
--- /dev/null
+++
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
@@ -0,0 +1,568 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HBaseTestingUt
[kylin] branch master updated: KYLIN-3544 refine guava cache for the recommended cuboids of cube planner
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new af77cfa KYLIN-3544 refine guava cache for the recommended cuboids of
cube planner
af77cfa is described below
commit af77cfa28b5f2b47b57dce71b00f69115275748c
Author: kyotoYaho
AuthorDate: Wed Dec 19 11:48:57 2018 +0800
KYLIN-3544 refine guava cache for the recommended cuboids of cube planner
---
.../kylin/cube/cuboid/algorithm/CuboidRecommender.java | 13 +++--
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
index 54c6764..baacb51 100644
---
a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
+++
b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
@@ -46,7 +46,7 @@ public class CuboidRecommender {
.removalListener(new RemovalListener>() {
@Override
public void onRemoval(RemovalNotification> notification) {
-logger.info("Dict with resource path " +
notification.getKey() + " is removed due to "
+logger.info("Recommended cuboids for cube " +
notification.getKey() + " is removed due to "
+ notification.getCause());
}
}).maximumSize(KylinConfig.getInstanceFromEnv().getCubePlannerRecommendCuboidCacheMaxSize())
@@ -66,8 +66,8 @@ public class CuboidRecommender {
}
public CuboidRecommender() {
-
Broadcaster.getInstance(KylinConfig.getInstanceFromEnv()).registerListener(new
CuboidRecommenderSyncListener(),
-"cube");
+Broadcaster.getInstance(KylinConfig.getInstanceFromEnv())
+.registerStaticListener(new CuboidRecommenderSyncListener(),
"cube", "cube_desc");
}
private static CuboidRecommender instance = new CuboidRecommender();
@@ -94,8 +94,7 @@ public class CuboidRecommender {
Map emptyMap = Maps.newHashMap();
cuboidRecommendCache.put(key, emptyMap);
try {
-Map recommendCuboid =
getRecommendCuboidList(cuboidStats, kylinConfig,
-true);
+Map recommendCuboid =
getRecommendCuboidList(cuboidStats, kylinConfig, true);
if (recommendCuboid != null) {
logger.info(String.format(Locale.ROOT, "Add
recommend cuboids for %s to cache", key));
@@ -105,7 +104,9 @@ public class CuboidRecommender {
return recommendCuboid;
} catch (Exception e) {
cuboidRecommendCache.invalidate(key);
-logger.error(String.format(Locale.ROOT, "Failed to
get recommend cuboids for %s in cache", key), e);
+logger.error(
+String.format(Locale.ROOT, "Failed to get
recommend cuboids for %s in cache", key),
+e);
throw e;
}
}
[kylin] branch master updated: KYLIN-2897 minor fix
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new a0e7039 KYLIN-2897 minor fix
a0e7039 is described below
commit a0e703986a5d965b3db8ee5be16dc3a26738c9eb
Author: Zhong
AuthorDate: Tue Nov 6 14:47:37 2018 +0800
KYLIN-2897 minor fix
---
core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6e1ff9e..149bb49 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1368,7 +1368,7 @@ abstract public class KylinConfigBase implements
Serializable {
}
public long getLazyQueryWaitingTimeoutMilliSeconds() {
-return
Long.parseLong(getOptional("kylin.query.lazy-query-waiting-timeout-milliseconds",
"6L"));
+return
Long.parseLong(getOptional("kylin.query.lazy-query-waiting-timeout-milliseconds",
"6"));
}
public int getQueryConcurrentRunningThresholdForProject() {
[kylin] branch 2.3.x updated: KYLIN-1239 change to htable owner tag to who...@kylin.apache.org
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.3.x by this push:
new 2e12bc1 KYLIN-1239 change to htable owner tag to
who...@kylin.apache.org
2e12bc1 is described below
commit 2e12bc1eb70cfe29978e515373384a6a76cf6585
Author: Zhong
AuthorDate: Mon Oct 8 10:07:40 2018 +0800
KYLIN-1239 change to htable owner tag to who...@kylin.apache.org
---
.../main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 56f867a..6e03b7e 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -51,7 +51,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println(descriptor);
System.out.println();
-descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
+descriptor.setValue(IRealizationConstants.HTableOwner,
"who...@kylin.apache.org");
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
[kylin] branch 2.4.x updated: KYLIN-1239 change to htable owner tag to who...@kylin.apache.org
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch 2.4.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.4.x by this push:
new 76edaaf KYLIN-1239 change to htable owner tag to
who...@kylin.apache.org
76edaaf is described below
commit 76edaafb881fa8d9cc0de9acf4131fa941095599
Author: Zhong
AuthorDate: Mon Oct 8 10:07:40 2018 +0800
KYLIN-1239 change to htable owner tag to who...@kylin.apache.org
---
.../main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 56f867a..6e03b7e 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -51,7 +51,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println(descriptor);
System.out.println();
-descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
+descriptor.setValue(IRealizationConstants.HTableOwner,
"who...@kylin.apache.org");
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
[kylin] branch 2.5.x updated: KYLIN-1239 change to htable owner tag to who...@kylin.apache.org
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.5.x by this push:
new 92957bf KYLIN-1239 change to htable owner tag to
who...@kylin.apache.org
92957bf is described below
commit 92957bf34470c8fd6521db3c56712769eb6579f6
Author: Zhong
AuthorDate: Mon Oct 8 10:07:40 2018 +0800
KYLIN-1239 change to htable owner tag to who...@kylin.apache.org
---
.../main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 56f867a..6e03b7e 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -51,7 +51,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println(descriptor);
System.out.println();
-descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
+descriptor.setValue(IRealizationConstants.HTableOwner,
"who...@kylin.apache.org");
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
[kylin] branch master updated: KYLIN-1239 change to htable owner tag to who...@kylin.apache.org
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 09c5668 KYLIN-1239 change to htable owner tag to
who...@kylin.apache.org
09c5668 is described below
commit 09c56680b178e4bb1b7200679e87fdeeae298e2e
Author: Zhong
AuthorDate: Mon Oct 8 10:07:40 2018 +0800
KYLIN-1239 change to htable owner tag to who...@kylin.apache.org
---
.../main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 026638a..f1b4657 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -52,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println(descriptor);
System.out.println();
-descriptor.setValue(IRealizationConstants.HTableOwner,
"dl-ebay-ky...@ebay.com");
+descriptor.setValue(IRealizationConstants.HTableOwner,
"who...@kylin.apache.org");
hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()),
descriptor);
}
}
svn commit: r29243 - /release/kylin/KEYS
Author: nju_yaho Date: Mon Sep 10 12:03:08 2018 New Revision: 29243 Log: checkin KEYS file Added: release/kylin/KEYS Added: release/kylin/KEYS == --- release/kylin/KEYS (added) +++ release/kylin/KEYS Mon Sep 10 12:03:08 2018 @@ -0,0 +1,480 @@ +This file contains the PGP keys of various developers. +Please don't use them for email unless you have to. Their main +purpose is code signing. + +Examples of importing this file in your keystore: + gpg --import KEYS.txt + (need pgp and other examples here) + +Examples of adding your key to this file: + pgp -kxa and append it to this file. + (pgpk -ll && pgpk -xa ) >> this file. + (gpg --list-sigs + && gpg --armor --export ) >> this file. + +--- + +pub 4096R/027DC364 2015-04-10 +uid Luke Han +sig 3027DC364 2015-04-10 Luke Han +sub 4096R/19BCFD87 2015-04-10 +sig 027DC364 2015-04-10 Luke Han + +-BEGIN PGP PUBLIC KEY BLOCK- +Version: GnuPG v1 + +mQINBFUno+8BEACxRnKkHFkA/qbYKyidAsMgvgsZZIwONlAoQE+01MKsGPbJU/DR +zVyzD5nFskaoZ3dLTeQfKP/BJ9qfXpm0zlS4PPHkrbKOeV+SawiiOvSLYCijz6vU +T1lbgW3yIpMkFp0zpIaZo7BpA2Q9s0qvPeZrDenu1X4IeNrUHs2QInYFDrUpyghw +BOrt4Yrx7KmtWDPia/Hvbd5nqy9coHZO+HYzBJhXPXK5arbrCDF+NEJFcUI3VlP1 +mPF0LG/SfeVtsjPVo+Ub9HIMlwd6NnzIveSXc5FWIaweQ5ko38TGgKLJ9UXkH5C+ +c/sPOaKVAYIcX5i94BLigjxAQAg15kDr1dX7gPnjQNZdLjMRWiTf4eSN5+rWK+j+ +IeA8QIcVGz7fmhyZ8J/9Gr9TdERvMuhoV/ll8aCGAQU5ApLfgVMIK8kW1vr8dFDh +x0tnrFZIPcNRg4K9ISR/DEFxYvSkXlxaBudIJiM5itgJ2dgltIIUX5uD8h2gi94A +4wJfI9m1F2v84lDjv6ve+O3O/P3OPErAKaKjn8w/PHFmGfVAGLkRIV/GQx39gDOf +QFMoGhqOdLeSIWLodg73HtpZAlVEVZ0+50cHtKqWfQ357Lw3aRJdO/WWrcDZQxut +lTiULde+vxgwWExpZf98HbpcwFNckA6+m0bwCCc9GEXkDUbPbuvpPvjJOQARAQAB +tB1MdWtlIEhhbiA8bHVrZWhhbkBhcGFjaGUub3JnPokCOAQTAQIAIgUCVSej7wIb +AwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQoRMvEgJ9w2RoDhAArSo9kHlQ +e84hP//+4VSccubHudPTX/+GdDcrAa0+QVS+lhnqe47sLsrvY2Hslm77AIZnYDOk +U76FpKkK4AcMjP+ayLT6KI/ds4k9ca3OvIgj9TYtFczOg2w1dq0sTKx5MXPXJpDL +o/Ee7Vuk10rKK9796B8keUHfFvexgfIFW39LaIOG0US5sx7IYNMIbpzEknl9dPi/ +/Mxa0+TTbeh+Jj+SkeGOV9SlcT3HWX+oXK3+saV5ChQHR4Db+wh0vhfgixi0YMbJ +iik0aPeATiLWvccpS3t5mBE5jP+Ss1xeptXaAmXJl9QC3J5lnIMZPhI7ie83agO8 +ZRcffUMfWw/4HJbB6kj9Cs82zIT8Qfz+w+3PyoLvU65IHmY8FxknQX2jg9XFMZ+C +c9NgJQyAOl2watglEJytjZ9UMfg36Zg0NYUmK+TMJzuMjtPbXgS7EyboMju9cdbS +1U4AStGFHUUcnw9r7jP9O9w7DtIjo9fdIDsfHkuNf/RMbZ4xAXWF/tbk++NpIjSY +qBXD2ZRUg+E6QfDvXOabpJ/14cVoYUlcFYeN9MpgDAKh94xmsyyxpnnyCWI3WTBj +aMT/p7xCCvYJ5XmyyV4zRHpQ/tHuZ/XnNfkeJavCxMXuslf5AyBP64uRB00EksDf +eMShtxE026rc9XyGf/pb7Ofg1zpbGl+Fhca5Ag0EVSej7wEQAKykheo7Hpq+m5Pl +NFGPp3Y8kN5hV5XoJLvKgbF4UnSB8/4IXRyLfTO+5F1ePNRiRBGtWAseWKtsN/qj +dgkc103Ckhl3NX/bH7o7mXsyOUThxAFCz/hzR1ca20H3lv63fB1L6wUD24aZ+m+9 +xf3RqiPNC/OYEvIN0NKeFNzpWWGNTaYcHReKiwwT8VpL0HzI8iu7oJI3Z/8Cmugk +N39FiR8lJTQspmRzUJerQHqB2E6M0B252DTD7syZZYHOgP56A9V9KsMApQ4mC/qz +ntdc0wTwLdAejrNs+qaSuIxSXhW9kPPJbazsDnvqEI3oAIU/4Zo+xt/QZpENgZTO +xEnzIoxby75Qq54vCZSxnVsjJfJv4ZB3DIq0bwEeOK75gcyGAShnxDCzdD0rkvwj +oyqiGu15bRKW+AiJAll2qs7PCV7sEUFx9pvz9QOx2JYyNKY246koe0j5nUDPY5uU +NnFdq2x+LpmuT3hZ/ptEhXzgQUEyw0LFGz8sMtkZ8Vk0gbJnv1nVqYVDjfcv9Ggy +DePBFkIpp/jWEVPWhlgQwYouNHF+W5Rs/r7eKpIwAQMw+aOcmn7BCxLkznZCvodG +373kIjy5to+hg0tIWiVcirGdCh1ktPxeRrvTkoFd001x8GU+z+E6JGYrIIuTi4JE +/cEqaV+wuKKqr2cFxxql9RXGYUWhABEBAAGJAh8EGAECAAkFAlUno+8CGwwACgkQ +oRMvEgJ9w2R2VQ/9FGp/O1F8TYITJWAyOUQONdk8pZriAk+9Oon9DkxhsYdDD1oB +td43wvFLr5dC7j9wvbnmYDWTlAwbeZYfZSS38YlrAH8/jwsuEtz6ez2faqb4qAz6 +OSGuJv3sp1T4fFUpd/O5KyGej90fmoFygSD7PdY9OEPKCHZpJuVqYMA+IaU2QXyL +giX4jspR5SJVaJJ+kyB7+IVIVwMqxhDQ0PU4v+7hx+kVyoL7f26P+SvMkPYzM2/w +HQrWMvuVT1NhPclk9wxdOJL1WJLNoR5U/wtZ/DBIV2gu457AyzuopHB8ye55J3q3 +mbrf24LrCUZxiofh+5GTjlC8EAcs79mAKRXUxVRah4rZovRg5jwCHgMsU0VFBzNH +fUq0euFIhyW14c8KOhVVCUcsDwONK0EEKJnG+cPhyjrzhdtfEy6StDCvg43PI7/w +Dodm8LJi7H1yG4LHDKalZ22st2K5DjYJWRBWXUYpCFWxQiOzGq8LdyYL3lQ/lBxt +ElBcVmy0Qvqf5/r9jVOpZWWvoAhJT0bj5B0Ca3CuGtkiVPJIMc2QuTAItWyRDN7T +EBG4pxGI2XsApbts0AkkINEYAKN+Z/o9BtrpEvodFPn2TBR3D9zPM+XiSZXTZ/gG +zJ5Mba6u6w6DZa238CtPdRTMh5ZAGhgkWYCrpQJ+CD9qflqNFzpTQ4FtPg8= +=YeBe +-END PGP PUBLIC KEY BLOCK- +pub 4096R/80181F78 2016-02-08 +uid Li Yang +sig 380181F78 2016-02-08 Li Yang +sub 4096R/31F92352 2016-02-08 +sig 80181F78 2016-02-08 Li Yang + +-BEGIN PGP PUBLIC KEY BLOCK- +Version: GnuPG v1 + +mQINBFa4kRoBEAC8VfF4w68DbWb/aeFV9P97xbfd0/D7rG3/n60t4vFE7WaPQNhK +iNq9Ib6eKLH3VU90lSkOBnStL0myvZRSZ6n7FNFO6XYKNKTcESIAvIba7WwdkJsH +HBda/OcHoi2Z2PieoTws3KgNyj34+XWDssJjRUNe8R4TrK7E0D8DGIRK832BX4te +38b0vKCK0Vg6MNMRPjWR2FguhBDPOGV45W3+MnwHJEhfWqbSHNwMpsLAgB37xNAp +9iR/jJI4poZqJpe8u+d4zKIpjyIQsCnanM6U032PJz2TTCwsayz5vZFz0Pswy7wz +JgvLoAzfJf8C4tU7KrWStjucxnum7DImQXEy/Bv7QeGPBYNkmipPhO4WWDyJuHUg +sx4e+bitBGqyk7xx6UIF/ybE5yKRuEGQkBbP27zbBXZJHBcOW1ukbVSBK+ogwFtG +2TZHTrDNDOUPiL1sUyOASekPisEh4kZ6Xvv0
svn commit: r29226 - /release/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-source-release.zip.asc.sha256
Author: nju_yaho Date: Sun Sep 9 11:28:27 2018 New Revision: 29226 Log: remove checksum files for signature files Removed: release/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-source-release.zip.asc.sha256
[kylin] branch document updated: KYLIN-3504 move the release notes from _docs23 to _docs
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch document
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/document by this push:
new e896163 KYLIN-3504 move the release notes from _docs23 to _docs
e896163 is described below
commit e89616333d86ce5c384e55222b36827378eeed3b
Author: Zhong
AuthorDate: Sun Sep 9 18:32:08 2018 +0800
KYLIN-3504 move the release notes from _docs23 to _docs
---
website/_docs/release_notes.md | 31 +++
website/_docs23/release_notes.md | 30 --
2 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/website/_docs/release_notes.md b/website/_docs/release_notes.md
index 34abcd5..f92e9ea 100644
--- a/website/_docs/release_notes.md
+++ b/website/_docs/release_notes.md
@@ -16,6 +16,37 @@ or send to Apache Kylin mailing list:
* Development relative: [d...@kylin.apache.org](mailto:d...@kylin.apache.org)
+## v2.4.1 - 2018-09-09
+_Tag:_ [kylin-2.4.1](https://github.com/apache/kylin/tree/kylin-2.4.1)
+This is a bug fix release after 2.4.0, with 22 bug fixes and enhancement.
Check [How to upgrade](/docs23/howto/howto_upgrade.html).
+
+__Improvement__
+* [KYLIN-3421] - Improve job scheduler fetch performance
+* [KYLIN-3424] - Missing invoke addCubingGarbageCollectionSteps in the cleanup
step for HBaseMROutput2Transition
+* [KYLIN-3422] - Support multi-path of domain for kylin connection
+* [KYLIN-3463] - Improve optimize job by avoiding creating empty output files
on HDFS
+* [KYLIN-3503] - Missing java.util.logging.config.file when starting kylin
instance
+* [KYLIN-3507] - Query NPE when project is not found
+
+__Bug__
+* [KYLIN-2662] - NegativeArraySizeException in "Extract Fact Table Distinct
Columns
+* [KYLIN-3025] - kylin odbc error : {fn CONVERT} for bigint type in tableau
10.4
+* [KYLIN-3255] - Cannot save cube
+* [KYLIN-3347] - QueryService Exception when using calcite function ex : {fn
CURRENT_TIMESTAMP(0)}
+* [KYLIN-3391] - BadQueryDetector only detect first query
+* [KYLIN-3403] - Querying sample cube with filter "KYLIN_CAL_DT.WEEK_BEG_DT >=
CAST('2001-09-09' AS DATE)" returns unexpected empty result set
+* [KYLIN-3428] - java.lang.OutOfMemoryError: Requested array size exceeds VM
limit
+* [KYLIN-3438] - mapreduce.job.queuename does not work at 'Convert Cuboid Data
to HFile' Step
+* [KYLIN-3451] - Cloned cube doesn't have Mandatory Cuboids copied
+* [KYLIN-3456] - Cube level's snapshot config does not work
+* [KYLIN-3460] - {fn CURRENT_DATE()} parse error
+* [KYLIN-3461] - "metastore.sh refresh-cube-signature" not updating cube
signature as expected
+* [KYLIN-3476] - Fix TupleExpression verification when parsing sql
+* [KYLIN-3492] - Wrong constant value in
KylinConfigBase.getDefaultVarcharPrecision
+* [KYLIN-3500] - kylin 2.4 use jdbc datasource :Unknown column 'A.A.CRT_DATE'
in 'where clause'
+* [KYLIN-3505] - DataType.getType wrong usage of cache
+
+
## v2.4.0 - 2018-06-23
_Tag:_ [kylin-2.4.0](https://github.com/apache/kylin/tree/kylin-2.4.0)
This is a major release after 2.3.x, with 8 new features and more than 30 bug
fixes bug fixes and enhancement. Check [How to
upgrade](/docs/howto/howto_upgrade.html).
diff --git a/website/_docs23/release_notes.md b/website/_docs23/release_notes.md
index 21dd26b..d95a00e 100644
--- a/website/_docs23/release_notes.md
+++ b/website/_docs23/release_notes.md
@@ -15,36 +15,6 @@ or send to Apache Kylin mailing list:
* User relative: [u...@kylin.apache.org](mailto:u...@kylin.apache.org)
* Development relative: [d...@kylin.apache.org](mailto:d...@kylin.apache.org)
-## v2.4.1 - 2018-09-09
-_Tag:_ [kylin-2.4.1](https://github.com/apache/kylin/tree/kylin-2.4.1)
-This is a bug fix release after 2.4.0, with 22 bug fixes and enhancement.
Check [How to upgrade](/docs23/howto/howto_upgrade.html).
-
-__Improvement__
-* [KYLIN-3421] - Improve job scheduler fetch performance
-* [KYLIN-3424] - Missing invoke addCubingGarbageCollectionSteps in the cleanup
step for HBaseMROutput2Transition
-* [KYLIN-3422] - Support multi-path of domain for kylin connection
-* [KYLIN-3463] - Improve optimize job by avoiding creating empty output files
on HDFS
-* [KYLIN-3503] - Missing java.util.logging.config.file when starting kylin
instance
-* [KYLIN-3507] - Query NPE when project is not found
-
-__Bug__
-* [KYLIN-2662] - NegativeArraySizeException in "Extract Fact Table Distinct
Columns
-* [KYLIN-3025] - kylin odbc error : {fn CONVERT} for bigint type in tableau
10.4
-* [KYLIN-3255] - Cannot save cube
-* [KYLIN-3347] - QueryService Exception when using calcite function ex : {fn
CURRENT_TIMESTAMP(0)}
-* [KYLIN-3391] - BadQueryDetector only detect first query
-* [KYLIN-3403] - Querying sample cube with filter "KYLIN_CAL_DT.WEEK_BEG_DT >=
CAST('2001-09-09' AS DATE)" returns unexpected empty result set
[kylin] branch document updated: KYLIN-3504 Update release notes and download for 2.4.1
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch document
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/document by this push:
new 6fbcbd3 KYLIN-3504 Update release notes and download for 2.4.1
6fbcbd3 is described below
commit 6fbcbd314d8a2601b35650d88ea4c70534f3ce08
Author: Zhong
AuthorDate: Sun Sep 9 18:26:04 2018 +0800
KYLIN-3504 Update release notes and download for 2.4.1
---
website/_docs23/release_notes.md | 31 +++
website/download/index.cn.md | 10 +-
website/download/index.md| 10 +-
3 files changed, 41 insertions(+), 10 deletions(-)
diff --git a/website/_docs23/release_notes.md b/website/_docs23/release_notes.md
index fa8027a..21dd26b 100644
--- a/website/_docs23/release_notes.md
+++ b/website/_docs23/release_notes.md
@@ -15,6 +15,37 @@ or send to Apache Kylin mailing list:
* User relative: [u...@kylin.apache.org](mailto:u...@kylin.apache.org)
* Development relative: [d...@kylin.apache.org](mailto:d...@kylin.apache.org)
+## v2.4.1 - 2018-09-09
+_Tag:_ [kylin-2.4.1](https://github.com/apache/kylin/tree/kylin-2.4.1)
+This is a bug fix release after 2.4.0, with 22 bug fixes and enhancement.
Check [How to upgrade](/docs23/howto/howto_upgrade.html).
+
+__Improvement__
+* [KYLIN-3421] - Improve job scheduler fetch performance
+* [KYLIN-3424] - Missing invoke addCubingGarbageCollectionSteps in the cleanup
step for HBaseMROutput2Transition
+* [KYLIN-3422] - Support multi-path of domain for kylin connection
+* [KYLIN-3463] - Improve optimize job by avoiding creating empty output files
on HDFS
+* [KYLIN-3503] - Missing java.util.logging.config.file when starting kylin
instance
+* [KYLIN-3507] - Query NPE when project is not found
+
+__Bug__
+* [KYLIN-2662] - NegativeArraySizeException in "Extract Fact Table Distinct
Columns
+* [KYLIN-3025] - kylin odbc error : {fn CONVERT} for bigint type in tableau
10.4
+* [KYLIN-3255] - Cannot save cube
+* [KYLIN-3347] - QueryService Exception when using calcite function ex : {fn
CURRENT_TIMESTAMP(0)}
+* [KYLIN-3391] - BadQueryDetector only detect first query
+* [KYLIN-3403] - Querying sample cube with filter "KYLIN_CAL_DT.WEEK_BEG_DT >=
CAST('2001-09-09' AS DATE)" returns unexpected empty result set
+* [KYLIN-3428] - java.lang.OutOfMemoryError: Requested array size exceeds VM
limit
+* [KYLIN-3438] - mapreduce.job.queuename does not work at 'Convert Cuboid Data
to HFile' Step
+* [KYLIN-3451] - Cloned cube doesn't have Mandatory Cuboids copied
+* [KYLIN-3456] - Cube level's snapshot config does not work
+* [KYLIN-3460] - {fn CURRENT_DATE()} parse error
+* [KYLIN-3461] - "metastore.sh refresh-cube-signature" not updating cube
signature as expected
+* [KYLIN-3476] - Fix TupleExpression verification when parsing sql
+* [KYLIN-3492] - Wrong constant value in
KylinConfigBase.getDefaultVarcharPrecision
+* [KYLIN-3500] - kylin 2.4 use jdbc datasource :Unknown column 'A.A.CRT_DATE'
in 'where clause'
+* [KYLIN-3505] - DataType.getType wrong usage of cache
+
+
## v2.3.2 - 2018-07-08
_Tag:_ [kylin-2.3.2](https://github.com/apache/kylin/tree/kylin-2.3.2)
This is a bug fix release after 2.3.1, with 12 bug fixes and enhancement.
Check [How to upgrade](/docs23/howto/howto_upgrade.html).
diff --git a/website/download/index.cn.md b/website/download/index.cn.md
index 9d64c33..134b64e 100644
--- a/website/download/index.cn.md
+++ b/website/download/index.cn.md
@@ -5,13 +5,13 @@ title: 下载
您可以按照这些[步骤](https://www.apache.org/info/verification.html)
并使用这些[KEYS](https://kylin.apache.org/KEYS)来验证下载文件的有效性.
- v2.4.0
-- This is a major release after 2.3.x, with 8 new features and more than 30
bug fixes and enhancement. For the detail list please check release notes.
+ v2.4.1
+- This is a bug fix release after 2.4.0, with 22 bug fixes and enhancement.
For the detail list please check release notes.
- [Release notes](/docs/release_notes.html) and [upgrade
guide](/docs/howto/howto_upgrade.html)
-- 源代码下载:
[apache-kylin-2.4.0-src.tar.gz](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.4.0/apache-kylin-2.4.0-source-release.zip)
\[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.4.0/apache-kylin-2.4.0-source-release.zip.asc)\]
\[[sha1](https://www.apache.org/dist/kylin/apache-kylin-2.4.0/apache-kylin-2.4.0-source-release.zip.sha1)\]
+- 源代码下载:
[apache-kylin-2.4.1-src.tar.gz](http://www.apache.org/dyn/closer.cgi/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-source-release.zip)
\[[asc](https://www.apache.org/dist/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-source-release.zip.asc)\]
\[[sha256](https://www.apache.org/dist/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-source-release.zip.sha256)\]
- 二进制包下载:
- - for HBase 1.x (includes HDP 2.3+, AWS EMR 5.0+, Azure HDInsight 3.4 - 3.6)
-
[apache-kylin-2.4.0-bin-hbase1x.tar.g
[kylin] branch 2.4.x updated: KYLIN-3504 update CURRENT_KYLIN_VERSION to 2.4.2
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch 2.4.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.4.x by this push:
new 450aadb KYLIN-3504 update CURRENT_KYLIN_VERSION to 2.4.2
450aadb is described below
commit 450aadbf36d55fec4c299eee61fd6feb48d12349
Author: Zhong
AuthorDate: Sun Sep 9 17:59:59 2018 +0800
KYLIN-3504 update CURRENT_KYLIN_VERSION to 2.4.2
---
core-common/src/main/java/org/apache/kylin/common/KylinVersion.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index d9b942a..a1541bd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -95,7 +95,7 @@ public class KylinVersion implements Comparable {
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
-private static final KylinVersion CURRENT_KYLIN_VERSION = new
KylinVersion("2.4.1.20500");
+private static final KylinVersion CURRENT_KYLIN_VERSION = new
KylinVersion("2.4.2");
private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");
svn commit: r29224 - /release/kylin/apache-kylin-2.4.1/
Author: nju_yaho Date: Sun Sep 9 09:55:07 2018 New Revision: 29224 Log: checkin release binary artifacts Added: release/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-bin-cdh57.tar.gz - copied unchanged from r29223, dev/kylin/apache-kylin-2.4.1-rc1/apache-kylin-2.4.1-bin-cdh57.tar.gz release/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-bin-cdh57.tar.gz.asc - copied unchanged from r29223, dev/kylin/apache-kylin-2.4.1-rc1/apache-kylin-2.4.1-bin-cdh57.tar.gz.asc release/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-bin-cdh57.tar.gz.sha256 - copied unchanged from r29223, dev/kylin/apache-kylin-2.4.1-rc1/apache-kylin-2.4.1-bin-cdh57.tar.gz.sha256 release/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-bin-hbase1x.tar.gz - copied unchanged from r29223, dev/kylin/apache-kylin-2.4.1-rc1/apache-kylin-2.4.1-bin-hbase1x.tar.gz release/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-bin-hbase1x.tar.gz.asc - copied unchanged from r29223, dev/kylin/apache-kylin-2.4.1-rc1/apache-kylin-2.4.1-bin-hbase1x.tar.gz.asc release/kylin/apache-kylin-2.4.1/apache-kylin-2.4.1-bin-hbase1x.tar.gz.sha256 - copied unchanged from r29223, dev/kylin/apache-kylin-2.4.1-rc1/apache-kylin-2.4.1-bin-hbase1x.tar.gz.sha256
