Repository: ambari Updated Branches: refs/heads/trunk 8bcc953df -> a749f97ab
AMBARI-20113 : hive20 view : fixed : Not able to compute the table statistics for partitioned table (nitirajrathore) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a749f97a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a749f97a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a749f97a Branch: refs/heads/trunk Commit: a749f97abbc44217fbb6f8e4868deaf6c4811fab Parents: 8bcc953 Author: Nitiraj Singh Rathore <nitiraj.rath...@gmail.com> Authored: Wed Mar 1 23:30:35 2017 +0530 Committer: Nitiraj Singh Rathore <nitiraj.rath...@gmail.com> Committed: Wed Mar 1 23:30:35 2017 +0530 ---------------------------------------------------------------------- .../generators/AnalyzeTableQueryGenerator.java | 49 ++++- .../view/hive20/resources/browser/DDLProxy.java | 6 +- .../hive20/resources/browser/DDLService.java | 4 +- .../AnalyzeTableQueryGeneratorSpecTest.groovy | 194 +++++++++++++++++++ 4 files changed, 242 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java index 1d68407..dbe9d43 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java @@ -18,23 +18,58 @@ package org.apache.ambari.view.hive20.internal.query.generators; +import com.google.common.base.Function; +import com.google.common.base.Joiner; import com.google.common.base.Optional; +import com.google.common.collect.FluentIterable; import org.apache.ambari.view.hive20.exceptions.ServiceException; +import org.apache.ambari.view.hive20.internal.dto.ColumnInfo; +import org.apache.ambari.view.hive20.internal.dto.TableMeta; + +import javax.annotation.Nullable; + +import static org.apache.ambari.view.hive20.internal.query.generators.QueryGenerationUtils.isNullOrEmpty; public class AnalyzeTableQueryGenerator implements QueryGenerator { - private final String databaseName; - private final String tableName; + private TableMeta tableMeta; private final Boolean shouldAnalyzeColumns; - public AnalyzeTableQueryGenerator(String databaseName, String tableName, Boolean shouldAnalyzeColumns) { - this.databaseName = databaseName; - this.tableName = tableName; + public AnalyzeTableQueryGenerator(TableMeta tableMeta, Boolean shouldAnalyzeColumns) { + this.tableMeta = tableMeta; this.shouldAnalyzeColumns = shouldAnalyzeColumns; } @Override public Optional<String> getQuery() throws ServiceException { - return Optional.of("ANALYZE TABLE " + "`" + databaseName + "`.`" + tableName + "`" + " COMPUTE STATISTICS " + - (shouldAnalyzeColumns? " FOR COLUMNS ": "") + ";"); + StringBuilder query = new StringBuilder("ANALYZE TABLE " ); + query.append("`").append(tableMeta.getDatabase()).append("`").append(".").append("`").append(tableMeta.getTable()).append("`"); + + if( null != tableMeta.getPartitionInfo() && !isNullOrEmpty(tableMeta.getPartitionInfo().getColumns())){ + query.append(" PARTITION (") + .append(Joiner.on(",") + .join(FluentIterable.from(tableMeta.getPartitionInfo().getColumns()) + .transform( + new Function<ColumnInfo, Object>() { + @Nullable + @Override + public Object apply(@Nullable ColumnInfo columnInfo) { + return columnInfo.getName(); + } + }) + ) + ) + .append(")"); + } + + + query.append(" COMPUTE STATISTICS "); + + if(shouldAnalyzeColumns){ + query.append(" FOR COLUMNS "); + } + + query.append(";"); + + return Optional.of(query.toString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java index e433dc4..0c93ba3 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java @@ -332,8 +332,10 @@ public class DDLProxy { } } - public Job analyzeTable(String databaseName, String tableName, Boolean shouldAnalyzeColumns, JobResourceManager resourceManager) throws ServiceException { - AnalyzeTableQueryGenerator queryGenerator = new AnalyzeTableQueryGenerator(databaseName, tableName, shouldAnalyzeColumns); + public Job analyzeTable(String databaseName, String tableName, Boolean shouldAnalyzeColumns, JobResourceManager resourceManager, ConnectionConfig hiveConnectionConfig) throws ServiceException { + TableMeta tableMeta = this.getTableProperties(context, hiveConnectionConfig, databaseName, tableName); + + AnalyzeTableQueryGenerator queryGenerator = new AnalyzeTableQueryGenerator(tableMeta, shouldAnalyzeColumns); Optional<String> analyzeTable = queryGenerator.getQuery(); String jobTitle = "Analyze table " + databaseName + "." + tableName; if(analyzeTable.isPresent()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java index 89b9d84..f5a4781 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java @@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; -import javax.ws.rs.FormParam; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; @@ -184,7 +183,8 @@ public class DDLService extends BaseService { shouldAnalyzeColumns = Boolean.valueOf(analyzeColumns.trim()); } try { - Job job = proxy.analyzeTable(databaseName, tableName, shouldAnalyzeColumns, getResourceManager()); + ConnectionConfig hiveConnectionConfig = getHiveConnectionConfig(); + Job job = proxy.analyzeTable(databaseName, tableName, shouldAnalyzeColumns, getResourceManager(), hiveConnectionConfig); JSONObject response = new JSONObject(); response.put("job", job); return Response.status(Response.Status.ACCEPTED).entity(response).build(); http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy new file mode 100644 index 0000000..7bc5850 --- /dev/null +++ b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy @@ -0,0 +1,194 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.ambari.view.hive20.internal.query.generators; + +import com.google.gson.Gson; +import org.apache.ambari.view.hive20.internal.dto.TableMeta; +import spock.lang.Specification; +import com.google.common.base.Optional; + +class AnalyzeTableQueryGeneratorSpecTest extends Specification { + def "analyze with partition and for columns"() { + setup: + String tableMetaJson = "{" + + "\"database\": \"d1\"," + + "\"table\": \"t2\"," + + "\"columns\": [{" + + "\"name\": \"col_name1\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name1 comment\"" + + "}, {" + + "\"name\": \"col_name2\"," + + "\"type\": \"decimal(10,2)\"," + + "\"comment\": \"col_name2 comment\"" + + "}]," + + "\"partitionInfo\": {" + + "\"columns\": [{" + + "\"name\": \"col_name4\"," + + "\"type\": \"char(1)\"," + + "\"comment\": \"col_name4 comment\"" + + "}, {" + + "\"name\": \"col_name3\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name3 comment\"" + + "}]" + + "}" + + "}"; + + TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class); + AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, true); + + when: + Optional<String> databaseDeleteQuery = generator.getQuery() + + then: + databaseDeleteQuery.isPresent() + + when: + String query = databaseDeleteQuery.get(); + + then: + query == "ANALYZE TABLE `d1`.`t2` PARTITION (col_name4,col_name3) COMPUTE STATISTICS FOR COLUMNS ;" + } + def "analyze with partition"() { + setup: + String tableMetaJson = "{" + + "\"database\": \"d1\"," + + "\"table\": \"t2\"," + + "\"columns\": [{" + + "\"name\": \"col_name1\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name1 comment\"" + + "}, {" + + "\"name\": \"col_name2\"," + + "\"type\": \"decimal(10,2)\"," + + "\"comment\": \"col_name2 comment\"" + + "}]," + + "\"partitionInfo\": {" + + "\"columns\": [{" + + "\"name\": \"col_name4\"," + + "\"type\": \"char(1)\"," + + "\"comment\": \"col_name4 comment\"" + + "}, {" + + "\"name\": \"col_name3\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name3 comment\"" + + "}]" + + "}" + + "}"; + + TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class); + AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, false); + + when: + Optional<String> databaseDeleteQuery = generator.getQuery() + + then: + databaseDeleteQuery.isPresent() + + when: + String query = databaseDeleteQuery.get(); + + then: + query == "ANALYZE TABLE `d1`.`t2` PARTITION (col_name4,col_name3) COMPUTE STATISTICS ;" + } + + def "analyze without partition"() { + setup: + String tableMetaJson = "{" + + "\"database\": \"d1\"," + + "\"table\": \"t2\"," + + "\"columns\": [{" + + "\"name\": \"col_name1\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name1 comment\"" + + "}, {" + + "\"name\": \"col_name2\"," + + "\"type\": \"decimal(10,2)\"," + + "\"comment\": \"col_name2 comment\"" + + "}," + + "{" + + "\"name\": \"col_name4\"," + + "\"type\": \"char(1)\"," + + "\"comment\": \"col_name4 comment\"" + + "}, {" + + "\"name\": \"col_name3\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name3 comment\"" + + "}" + + "]" + + "}"; + + TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class); + AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, true); + + when: + Optional<String> databaseDeleteQuery = generator.getQuery() + + then: + databaseDeleteQuery.isPresent() + + when: + String query = databaseDeleteQuery.get(); + + then: + query == "ANALYZE TABLE `d1`.`t2` COMPUTE STATISTICS FOR COLUMNS ;" + } + + def "analyze for table only"() { + setup: + String tableMetaJson = "{" + + "\"database\": \"d1\"," + + "\"table\": \"t2\"," + + "\"columns\": [{" + + "\"name\": \"col_name1\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name1 comment\"" + + "}, {" + + "\"name\": \"col_name2\"," + + "\"type\": \"decimal(10,2)\"," + + "\"comment\": \"col_name2 comment\"" + + "}," + + "{" + + "\"name\": \"col_name4\"," + + "\"type\": \"char(1)\"," + + "\"comment\": \"col_name4 comment\"" + + "}, {" + + "\"name\": \"col_name3\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name3 comment\"" + + "}" + + "]" + + "}"; + + TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class); + AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, false); + + when: + Optional<String> databaseDeleteQuery = generator.getQuery() + + then: + databaseDeleteQuery.isPresent() + + when: + String query = databaseDeleteQuery.get(); + + then: + query == "ANALYZE TABLE `d1`.`t2` COMPUTE STATISTICS ;" + } +}
