Repository: ambari Updated Branches: refs/heads/trunk 5269052eb -> c77f4ce8b
AMBARI-20598 : added support for stats of partitioned table for hive server version 2.1 and above (nitirajrathore) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c77f4ce8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c77f4ce8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c77f4ce8 Branch: refs/heads/trunk Commit: c77f4ce8bbf140efcae48128ec01391c908c4f46 Parents: 5269052 Author: Nitiraj Singh Rathore <nitiraj.rath...@gmail.com> Authored: Tue Mar 28 16:50:47 2017 +0530 Committer: Nitiraj Singh Rathore <nitiraj.rath...@gmail.com> Committed: Tue Mar 28 16:50:47 2017 +0530 ---------------------------------------------------------------------- .../ambari/view/hive20/ConnectionDelegate.java | 4 ++ .../view/hive20/HiveJdbcConnectionDelegate.java | 6 ++ .../ambari/view/hive20/actor/JdbcConnector.java | 38 +++++++++- .../view/hive20/actor/StatementExecutor.java | 15 ++++ .../actor/message/GetDatabaseMetadataJob.java | 24 +++++++ .../hive20/actor/message/ResultInformation.java | 19 +++++ .../view/hive20/actor/message/job/Result.java | 19 ++++- .../ambari/view/hive20/client/DDLDelegator.java | 3 + .../view/hive20/client/DDLDelegatorImpl.java | 53 ++++++++++---- .../hive20/client/DatabaseMetadataWrapper.java | 37 ++++++++++ .../view/hive20/internal/dto/ColumnStats.java | 5 ++ .../view/hive20/internal/dto/TableStats.java | 12 ++++ .../parsers/DatabaseMetadataExtractor.java | 46 ++++++++++++ .../internal/parsers/TableMetaParser.java | 3 +- .../internal/parsers/TableMetaParserImpl.java | 9 ++- .../view/hive20/resources/browser/DDLProxy.java | 23 ++++-- .../ui/app/components/table-statistics.js | 73 ++++++++++++++++---- .../resources/ui/app/services/stats-service.js | 15 +--- .../templates/components/table-statistics.hbs | 25 ++++++- 19 files changed, 372 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/ConnectionDelegate.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/ConnectionDelegate.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/ConnectionDelegate.java index f3c4e91..6f03480 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/ConnectionDelegate.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/ConnectionDelegate.java @@ -23,6 +23,7 @@ import org.apache.ambari.view.hive20.actor.message.GetColumnMetadataJob; import org.apache.hive.jdbc.HiveConnection; import org.apache.hive.jdbc.HiveStatement; +import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; @@ -31,6 +32,9 @@ public interface ConnectionDelegate { Optional<ResultSet> execute(String statement) throws SQLException; Optional<ResultSet> execute(HiveConnection connection, String statement) throws SQLException; ResultSet getColumnMetadata(HiveConnection connection, GetColumnMetadataJob job) throws SQLException; + + DatabaseMetaData getDatabaseMetadata(HiveConnection connection) throws SQLException; + void cancel() throws SQLException; void closeResultSet(); void closeStatement(); http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/HiveJdbcConnectionDelegate.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/HiveJdbcConnectionDelegate.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/HiveJdbcConnectionDelegate.java index 5cc60fb..12149e8 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/HiveJdbcConnectionDelegate.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/HiveJdbcConnectionDelegate.java @@ -73,6 +73,12 @@ public class HiveJdbcConnectionDelegate implements ConnectionDelegate { } @Override + public DatabaseMetaData getDatabaseMetadata(HiveConnection connection) throws SQLException { + DatabaseMetaData metaData = connection.getMetaData(); + return metaData; + } + + @Override public void cancel() throws SQLException { if (currentStatement != null) { currentStatement.cancel(); http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/JdbcConnector.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/JdbcConnector.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/JdbcConnector.java index 4b218b1..f93ecbf 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/JdbcConnector.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/JdbcConnector.java @@ -30,6 +30,7 @@ import org.apache.ambari.view.hive20.actor.message.Connect; import org.apache.ambari.view.hive20.actor.message.FetchError; import org.apache.ambari.view.hive20.actor.message.FetchResult; import org.apache.ambari.view.hive20.actor.message.GetColumnMetadataJob; +import org.apache.ambari.view.hive20.actor.message.GetDatabaseMetadataJob; import org.apache.ambari.view.hive20.actor.message.HiveJob; import org.apache.ambari.view.hive20.actor.message.HiveMessage; import org.apache.ambari.view.hive20.actor.message.ResultInformation; @@ -51,8 +52,11 @@ import org.apache.ambari.view.hive20.actor.message.lifecycle.FreeConnector; import org.apache.ambari.view.hive20.actor.message.lifecycle.InactivityCheck; import org.apache.ambari.view.hive20.actor.message.lifecycle.KeepAlive; import org.apache.ambari.view.hive20.actor.message.lifecycle.TerminateInactivityCheck; +import org.apache.ambari.view.hive20.client.DatabaseMetadataWrapper; +import org.apache.ambari.view.hive20.exceptions.ServiceException; import org.apache.ambari.view.hive20.internal.Connectable; import org.apache.ambari.view.hive20.internal.ConnectionException; +import org.apache.ambari.view.hive20.internal.parsers.DatabaseMetadataExtractor; import org.apache.ambari.view.hive20.persistence.Storage; import org.apache.ambari.view.hive20.persistence.utils.ItemNotFound; import org.apache.ambari.view.hive20.resources.jobs.viewJobs.Job; @@ -64,6 +68,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.concurrent.duration.Duration; +import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayDeque; @@ -78,7 +83,7 @@ import java.util.concurrent.TimeUnit; */ public class JdbcConnector extends HiveActor { - private final Logger LOG = LoggerFactory.getLogger(getClass()); + private static final Logger LOG = LoggerFactory.getLogger(JdbcConnector.class); public static final String SUFFIX = "validating the login"; @@ -190,6 +195,8 @@ public class JdbcConnector extends HiveActor { runStatementJob((SQLStatementJob) message); } else if (message instanceof GetColumnMetadataJob) { runGetMetaData((GetColumnMetadataJob) message); + } else if (message instanceof GetDatabaseMetadataJob) { + runGetDatabaseMetaData((GetDatabaseMetadataJob) message); } else if (message instanceof ExecuteNextStatement) { executeNextStatement(); } else if (message instanceof ResultInformation) { @@ -252,6 +259,12 @@ public class JdbcConnector extends HiveActor { processFailure(failure); return; } + Optional<DatabaseMetaData> databaseMetaDataOptional = message.getDatabaseMetaData(); + if (databaseMetaDataOptional.isPresent()) { + DatabaseMetaData databaseMetaData = databaseMetaDataOptional.get(); + processDatabaseMetadata(databaseMetaData); + return; + } if (statementQueue.size() == 0) { // This is the last resultSet processResult(message.getResultSet()); @@ -286,6 +299,19 @@ public class JdbcConnector extends HiveActor { } } + private void processDatabaseMetadata(DatabaseMetaData databaseMetaData) { + executing = false; + isFailure = false; + // Send for sync execution + try { + DatabaseMetadataWrapper databaseMetadataWrapper = new DatabaseMetadataExtractor(databaseMetaData).extract(); + commandSender.tell(databaseMetadataWrapper, self()); + } catch (ServiceException e) { + commandSender.tell(new ExecutionFailed(e.getMessage(), e), self()); + } + cleanUpWithTermination(); + } + private void stopStatementExecutor() { if (statementExecutor != null) { statementExecutor.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -387,6 +413,16 @@ public class JdbcConnector extends HiveActor { statementExecutor.tell(message, self()); } + private void runGetDatabaseMetaData(GetDatabaseMetadataJob message) { + if (!checkConnection()) return; + resetToInitialState(); + executing = true; + executionType = message.getType(); + commandSender = getSender(); + statementExecutor = getStatementExecutor(); + statementExecutor.tell(message, self()); + } + private ActorRef getStatementExecutor() { return getContext().actorOf(Props.create(StatementExecutor.class, hdfsApi, storage, connectable.getConnection().get(), connectionDelegate) .withDispatcher("akka.actor.result-dispatcher"), http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/StatementExecutor.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/StatementExecutor.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/StatementExecutor.java index c3ed14b..d73a284 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/StatementExecutor.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/StatementExecutor.java @@ -23,6 +23,7 @@ import akka.actor.Props; import com.google.common.base.Optional; import org.apache.ambari.view.hive20.ConnectionDelegate; import org.apache.ambari.view.hive20.actor.message.GetColumnMetadataJob; +import org.apache.ambari.view.hive20.actor.message.GetDatabaseMetadataJob; import org.apache.ambari.view.hive20.actor.message.HiveMessage; import org.apache.ambari.view.hive20.actor.message.ResultInformation; import org.apache.ambari.view.hive20.actor.message.RunStatement; @@ -36,6 +37,7 @@ import org.apache.hive.jdbc.HiveStatement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; import java.util.UUID; @@ -70,6 +72,8 @@ public class StatementExecutor extends HiveActor { runStatement((RunStatement) message); } else if (message instanceof GetColumnMetadataJob) { getColumnMetaData((GetColumnMetadataJob) message); + }else if (message instanceof GetDatabaseMetadataJob) { + getDatabaseMetaData((GetDatabaseMetadataJob) message); } } @@ -151,4 +155,15 @@ public class StatementExecutor extends HiveActor { ", tablePattern: " + message.getTablePattern() + ", ColumnPattern: " + message.getColumnPattern(), e)), self()); } } + + private void getDatabaseMetaData(GetDatabaseMetadataJob message) { + try { + DatabaseMetaData metaData = connectionDelegate.getDatabaseMetadata(connection); + sender().tell(new ResultInformation(-1, metaData), self()); + } catch (SQLException e) { + LOG.error("Failed to get database metadata.", e); + sender().tell(new ResultInformation(-1, + new Failure("Failed to get database metadata.", e)), self()); + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/GetDatabaseMetadataJob.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/GetDatabaseMetadataJob.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/GetDatabaseMetadataJob.java new file mode 100644 index 0000000..b7fe37f --- /dev/null +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/GetDatabaseMetadataJob.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.view.hive20.actor.message; + +public class GetDatabaseMetadataJob extends HiveJob { + public GetDatabaseMetadataJob(String username) { + super(Type.SYNC, username); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/ResultInformation.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/ResultInformation.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/ResultInformation.java index 5b5e17c..0f86010 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/ResultInformation.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/ResultInformation.java @@ -21,6 +21,7 @@ package org.apache.ambari.view.hive20.actor.message; import com.google.common.base.Optional; import org.apache.ambari.view.hive20.actor.message.job.Failure; +import java.sql.DatabaseMetaData; import java.sql.ResultSet; /** @@ -41,6 +42,7 @@ public class ResultInformation { private final Failure failure; private final boolean cancelled; + private DatabaseMetaData databaseMetaData; private ResultInformation(int id, ResultSet resultSet, Failure failure, boolean cancelled) { this.id = id; @@ -57,6 +59,11 @@ public class ResultInformation { this(id, null, null, false); } + public ResultInformation(int id, ResultSet resultSet, DatabaseMetaData metaData, Failure failure, boolean cancelled ) { + this(id, null, null, false); + this.databaseMetaData = metaData; + } + public ResultInformation(int id, Failure failure) { this(id, null, failure, false); } @@ -65,6 +72,10 @@ public class ResultInformation { this(id, null, null, cancelled); } + public ResultInformation(int id, DatabaseMetaData metaData) { + this(id, null, metaData, null, false); + } + public int getId() { return id; } @@ -80,4 +91,12 @@ public class ResultInformation { public boolean isCancelled() { return cancelled; } + + public Optional<DatabaseMetaData> getDatabaseMetaData() { + return Optional.fromNullable(databaseMetaData); + } + + public void setDatabaseMetaData(DatabaseMetaData databaseMetaData) { + this.databaseMetaData = databaseMetaData; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/job/Result.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/job/Result.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/job/Result.java index f8c3ba0..2d8fc4d 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/job/Result.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/actor/message/job/Result.java @@ -20,19 +20,34 @@ package org.apache.ambari.view.hive20.actor.message.job; import com.google.common.collect.ImmutableList; import org.apache.ambari.view.hive20.client.ColumnDescription; +import org.apache.ambari.view.hive20.client.DatabaseMetadataWrapper; import org.apache.ambari.view.hive20.client.Row; +import java.sql.DatabaseMetaData; import java.util.List; public class Result { - private final List<ColumnDescription> columns; - private final List<Row> rows; + private List<ColumnDescription> columns; + private List<Row> rows; + private DatabaseMetadataWrapper databaseMetadata; public Result(List<Row> rows, List<ColumnDescription> columns) { this.rows = ImmutableList.copyOf(rows); this.columns = columns; } + public Result(DatabaseMetadataWrapper databaseMetadata){ + this.databaseMetadata = databaseMetadata; + } + + public DatabaseMetadataWrapper getDatabaseMetadata() { + return databaseMetadata; + } + + public void setDatabaseMetadata(DatabaseMetadataWrapper databaseMetadata) { + this.databaseMetadata = databaseMetadata; + } + public List<Row> getRows() { return rows; } http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegator.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegator.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegator.java index baa82b4..2f5331e 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegator.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegator.java @@ -20,6 +20,7 @@ package org.apache.ambari.view.hive20.client; import java.util.List; +import org.apache.ambari.view.hive20.exceptions.ServiceException; import org.apache.ambari.view.hive20.internal.dto.DatabaseInfo; import org.apache.ambari.view.hive20.internal.dto.TableInfo; @@ -40,4 +41,6 @@ public interface DDLDelegator { Cursor<Row, ColumnDescription> getTableListCursor(ConnectionConfig config, String database, String like); Cursor<Row, ColumnDescription> getTableDescriptionCursor(ConnectionConfig config, String database, String table, String like, boolean extended); + + DatabaseMetadataWrapper getDatabaseMetadata(ConnectionConfig config) throws ServiceException; } http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegatorImpl.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegatorImpl.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegatorImpl.java index d6756ef..c20c847 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegatorImpl.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DDLDelegatorImpl.java @@ -18,15 +18,20 @@ package org.apache.ambari.view.hive20.client; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; - +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.actor.Inbox; +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.hive20.actor.message.Connect; import org.apache.ambari.view.hive20.actor.message.ExecuteJob; import org.apache.ambari.view.hive20.actor.message.GetColumnMetadataJob; +import org.apache.ambari.view.hive20.actor.message.GetDatabaseMetadataJob; import org.apache.ambari.view.hive20.actor.message.HiveJob; import org.apache.ambari.view.hive20.actor.message.SQLStatementJob; import org.apache.ambari.view.hive20.actor.message.job.AuthenticationFailed; @@ -37,24 +42,20 @@ import org.apache.ambari.view.hive20.actor.message.job.NoMoreItems; import org.apache.ambari.view.hive20.actor.message.job.NoResult; import org.apache.ambari.view.hive20.actor.message.job.Result; import org.apache.ambari.view.hive20.actor.message.job.ResultSetHolder; +import org.apache.ambari.view.hive20.exceptions.ServiceException; import org.apache.ambari.view.hive20.internal.dto.DatabaseInfo; import org.apache.ambari.view.hive20.internal.dto.TableInfo; import org.apache.ambari.view.hive20.utils.HiveActorConfiguration; import org.apache.ambari.view.hive20.utils.ServiceFormattedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; -import com.google.common.base.Joiner; -import com.google.common.base.Optional; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.actor.Inbox; import scala.concurrent.duration.Duration; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + public class DDLDelegatorImpl implements DDLDelegator { public static final String NO_VALUE_MARKER = "NO_VALUE"; @@ -207,6 +208,23 @@ public class DDLDelegatorImpl implements DDLDelegator { return getResultFromDB(execute); } + @Override + public DatabaseMetadataWrapper getDatabaseMetadata(ConnectionConfig config) throws ServiceException { + Connect connect = config.createConnectMessage(); + HiveJob job = new GetDatabaseMetadataJob(config.getUsername()); + ExecuteJob execute = new ExecuteJob(connect, job); + + LOG.info("Fetching databaseMetadata."); + Optional<Result> resultOptional = getResultFromDB(execute); + if(resultOptional.isPresent()){ + Result result = resultOptional.get(); + DatabaseMetadataWrapper databaseMetadata = result.getDatabaseMetadata(); + return databaseMetadata; + }else{ + throw new ServiceException("Cannot fetch database version."); + } + } + private Optional<Result> getResultFromDB(ExecuteJob job) { List<ColumnDescription> descriptions = null; List<Row> rows = Lists.newArrayList(); @@ -226,6 +244,11 @@ public class DDLDelegatorImpl implements DDLDelegator { return Optional.absent(); } + if (submitResult instanceof DatabaseMetadataWrapper) { + LOG.info("Query returned with no result."); + return Optional.of(new Result((DatabaseMetadataWrapper)submitResult)); + } + if (submitResult instanceof ExecutionFailed) { ExecutionFailed error = (ExecutionFailed) submitResult; LOG.error("Failed to get the table description"); http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DatabaseMetadataWrapper.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DatabaseMetadataWrapper.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DatabaseMetadataWrapper.java new file mode 100644 index 0000000..5ea9fed --- /dev/null +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/client/DatabaseMetadataWrapper.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive20.client; + +public class DatabaseMetadataWrapper { + private final int databaseMajorVersion; + private final int databaseMinorVersion; + + public DatabaseMetadataWrapper(int databaseMajorVersion, int databaseMinorVersion) { + this.databaseMajorVersion = databaseMajorVersion; + this.databaseMinorVersion = databaseMinorVersion; + } + + @Override + public String toString() { + return "DatabaseMetadataWrapper{" + + "databaseMajorVersion=" + databaseMajorVersion + + ", databaseMinorVersion=" + databaseMinorVersion + + '}'; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnStats.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnStats.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnStats.java index 190ecd3..1654a1f 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnStats.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/ColumnStats.java @@ -44,6 +44,7 @@ public class ColumnStats { private String numTrues; private String numFalse; private String comment; + private String columnStatsAccurate; public String getDatabaseName() { return databaseName; @@ -167,4 +168,8 @@ public class ColumnStats { sb.append('}'); return sb.toString(); } + + public void setColumnStatsAccurate(String columnStatsAccurate) { + this.columnStatsAccurate = columnStatsAccurate; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/TableStats.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/TableStats.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/TableStats.java index 3048d22..5d0f307 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/TableStats.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/dto/TableStats.java @@ -18,6 +18,9 @@ package org.apache.ambari.view.hive20.internal.dto; + +import org.apache.ambari.view.hive20.client.DatabaseMetadataWrapper; + /** * this will be returned as a part of TableMeta which table info is called. * It includes the part of DetailedTableInfo which contain statistics related data. @@ -29,6 +32,7 @@ public class TableStats { public static final String RAW_DATA_SIZE = "rawDataSize"; public static final String TOTAL_SIZE = "totalSize"; + private DatabaseMetadataWrapper databaseMetadata; private Boolean isTableStatsEnabled; private Integer numFiles; private Integer numRows; @@ -96,4 +100,12 @@ public class TableStats { sb.append('}'); return sb.toString(); } + + public DatabaseMetadataWrapper getDatabaseMetadata() { + return databaseMetadata; + } + + public void setDatabaseMetadata(DatabaseMetadataWrapper databaseMetadata) { + this.databaseMetadata = databaseMetadata; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/DatabaseMetadataExtractor.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/DatabaseMetadataExtractor.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/DatabaseMetadataExtractor.java new file mode 100644 index 0000000..1ae146a --- /dev/null +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/DatabaseMetadataExtractor.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.hive20.internal.parsers; + +import org.apache.ambari.view.hive20.client.DatabaseMetadataWrapper; +import org.apache.ambari.view.hive20.exceptions.ServiceException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.DatabaseMetaData; +import java.sql.SQLException; + +public class DatabaseMetadataExtractor { + private static final Logger LOG = LoggerFactory.getLogger(DatabaseMetadataExtractor.class); + + private final DatabaseMetaData databaseMetaData; + + public DatabaseMetadataExtractor(DatabaseMetaData databaseMetaData) { + this.databaseMetaData = databaseMetaData; + } + + public DatabaseMetadataWrapper extract() throws ServiceException { + try { + return new DatabaseMetadataWrapper(databaseMetaData.getDatabaseMajorVersion(), databaseMetaData.getDatabaseMinorVersion()); + } catch (SQLException e) { + LOG.error("Error occurred while fetching version from database metadata.", e); + throw new ServiceException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParser.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParser.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParser.java index aae23c8..251aace 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParser.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParser.java @@ -18,6 +18,7 @@ package org.apache.ambari.view.hive20.internal.parsers; +import org.apache.ambari.view.hive20.client.DatabaseMetadataWrapper; import org.apache.ambari.view.hive20.client.Row; import java.util.List; @@ -26,5 +27,5 @@ import java.util.List; * */ public interface TableMetaParser<T> { - T parse(String database, String table, List<Row> createTableStatementRows, List<Row> describeFormattedRows); + T parse(String database, String table, List<Row> createTableStatementRows, List<Row> describeFormattedRows, DatabaseMetadataWrapper databaseMetadata); } http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParserImpl.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParserImpl.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParserImpl.java index f2a1933..711cab9 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParserImpl.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/parsers/TableMetaParserImpl.java @@ -18,6 +18,7 @@ package org.apache.ambari.view.hive20.internal.parsers; +import org.apache.ambari.view.hive20.client.DatabaseMetadataWrapper; import org.apache.ambari.view.hive20.client.Row; import org.apache.ambari.view.hive20.internal.dto.ColumnInfo; import org.apache.ambari.view.hive20.internal.dto.DetailedTableInfo; @@ -55,10 +56,11 @@ public class TableMetaParserImpl implements TableMetaParser<TableMeta> { private ViewInfoParser viewInfoParser; @Override - public TableMeta parse(String database, String table, List<Row> createTableStatementRows, List<Row> describeFormattedRows) { + public TableMeta parse(String database, String table, List<Row> createTableStatementRows, List<Row> describeFormattedRows, DatabaseMetadataWrapper databaseMetadata) { String createTableStatement = createTableStatementParser.parse(createTableStatementRows); DetailedTableInfo tableInfo = detailedTableInfoParser.parse(describeFormattedRows); TableStats tableStats = getTableStats(tableInfo); + tableStats.setDatabaseMetadata(databaseMetadata); StorageInfo storageInfo = storageInfoParser.parse(describeFormattedRows); List<ColumnInfo> columns = columnInfoParser.parse(describeFormattedRows); PartitionInfo partitionInfo = partitionInfoParser.parse(describeFormattedRows); @@ -118,7 +120,10 @@ public class TableMetaParserImpl implements TableMetaParser<TableMeta> { tableStats.setTotalSize(Integer.valueOf(totalSize.trim())); } - tableStats.setColumnStatsAccurate(columnStatsAccurate); + if(!Strings.isNullOrEmpty(columnStatsAccurate) && !Strings.isNullOrEmpty(columnStatsAccurate.trim())) { + tableStats.setTableStatsEnabled(true); + tableStats.setColumnStatsAccurate(columnStatsAccurate); + } return tableStats; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java index 0c93ba3..77857f9 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java @@ -29,6 +29,7 @@ import org.apache.ambari.view.hive20.ConnectionSystem; import org.apache.ambari.view.hive20.client.ConnectionConfig; import org.apache.ambari.view.hive20.client.DDLDelegator; import org.apache.ambari.view.hive20.client.DDLDelegatorImpl; +import org.apache.ambari.view.hive20.client.DatabaseMetadataWrapper; import org.apache.ambari.view.hive20.client.HiveClientException; import org.apache.ambari.view.hive20.client.Row; import org.apache.ambari.view.hive20.exceptions.ServiceException; @@ -146,8 +147,14 @@ public class DDLProxy { DDLDelegator delegator = new DDLDelegatorImpl(context, ConnectionSystem.getInstance().getActorSystem(), ConnectionSystem.getInstance().getOperationController(context)); List<Row> createTableStatementRows = delegator.getTableCreateStatement(connectionConfig, databaseName, tableName); List<Row> describeFormattedRows = delegator.getTableDescriptionFormatted(connectionConfig, databaseName, tableName); + DatabaseMetadataWrapper databaseMetadata = new DatabaseMetadataWrapper(1, 0); + try { + databaseMetadata = delegator.getDatabaseMetadata(connectionConfig); + } catch (ServiceException e) { + LOG.error("Exception while fetching hive version", e); + } - return tableMetaParser.parse(databaseName, tableName, createTableStatementRows, describeFormattedRows); + return tableMetaParser.parse(databaseName, tableName, createTableStatementRows, describeFormattedRows, databaseMetadata); } private Optional<DatabaseInfo> selectDatabase(final String databaseId) { @@ -353,6 +360,7 @@ public class DDLProxy { List<String[]> rows = results.getRows(); Map<Integer, String> headerMap = new HashMap<>(); boolean header = true; + ColumnStats columnStats = new ColumnStats(); for(String[] row : rows){ if(header){ for(int i = 0 ; i < row.length; i++){ @@ -364,10 +372,14 @@ public class DDLProxy { } else if(row.length > 0 ){ if(columnName.equals(row[0])){ // the first column of the row contains column name - return createColumnStats(row, headerMap); + createColumnStats(row, headerMap, columnStats); + }else if( row.length > 1 && row[0].equalsIgnoreCase("COLUMN_STATS_ACCURATE")){ + columnStats.setColumnStatsAccurate(row[1]); } } } + + return columnStats; }else{ throw new ServiceException("Cannot find any result for this jobId: " + jobId); } @@ -375,9 +387,6 @@ public class DDLProxy { LOG.error("Exception occurred while fetching results for column statistics with jobId: {}", jobId, e); throw new ServiceException(e); } - - LOG.error("Column stats not found in the fetched results."); - throw new ServiceException("Could not find the column stats in the result."); } /** @@ -386,13 +395,13 @@ public class DDLProxy { * indexes : 0 1 2 3 4 5 6 7 8 9 10 * @param row * @param headerMap + * @param columnStats * @return */ - private ColumnStats createColumnStats(String[] row, Map<Integer, String> headerMap) throws ServiceException { + private ColumnStats createColumnStats(String[] row, Map<Integer, String> headerMap, ColumnStats columnStats) throws ServiceException { if(null == row){ throw new ServiceException("row cannot be null."); } - ColumnStats columnStats = new ColumnStats(); for(int i = 0 ; i < row.length; i++){ switch(headerMap.get(i)){ case ColumnStats.COLUMN_NAME: http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/resources/ui/app/components/table-statistics.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/components/table-statistics.js b/contrib/views/hive20/src/main/resources/ui/app/components/table-statistics.js index 0310cbc..c6846d3 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/components/table-statistics.js +++ b/contrib/views/hive20/src/main/resources/ui/app/components/table-statistics.js @@ -20,13 +20,52 @@ import Ember from 'ember'; import UILoggerMixin from '../mixins/ui-logger'; export default Ember.Component.extend(UILoggerMixin, { + columnStatsKeys : [ + {dataKey: 'min', label: 'MIN'}, + {dataKey: 'max', label: 'MAX'}, + {dataKey: 'numNulls', label: 'NUMBER OF NULLS'}, + {dataKey: 'distinctCount', label: 'DISTINCT COUNT'}, + {dataKey: 'avgColLen', label: 'AVERAGE COLUMN LENGTH'}, + {dataKey: 'maxColLen', label: 'MAX COLUMN LENGTH'}, + {dataKey: 'numTrues', label: 'NUMBER OF TRUE'}, + {dataKey: 'numFalse', label: 'NUMBER OF FALSE'}, + ], + statsService: Ember.inject.service(), analyseWithStatistics: false, + partitionStatSupportedVersion: "2.1", + isTablePartitioned: Ember.computed("table.partitionInfo.columns", function(){ + return this.get("table.partitionInfo.columns") && this.get("table.partitionInfo.columns.length") > 0; + }), + partitionStatSupported: Ember.computed("table.tableStats.databaseMetadata.databaseMajorVersion", + "table.tableStats.databaseMetadata.databaseMinorVersion", function(){ + if(this.get('table.tableStats.databaseMetadata.databaseMajorVersion') > 2){ + return true; + }else if(this.get('table.tableStats.databaseMetadata.databaseMajorVersion') === 2 + && this.get('table.tableStats.databaseMetadata.databaseMinorVersion') >= 1){ + return true; + } + return false; + }), + showStats:Ember.computed("partitionStatSupported", "isTablePartitioned", function(){ + if(!this.get("isTablePartitioned")) { + return true; + }else{ + if(this.get("partitionStatSupported")){ + return true; + }else{ + return false; + } + } + }), tableStats: Ember.computed.oneWay('table.tableStats'), + tableStatisticsEnabled: Ember.computed.oneWay('table.tableStats.isTableStatsEnabled'), + basicStatsAccurate: Ember.computed.oneWay('columnStatsAccurate.BASIC_STATS'), + columnStatsAccurate: Ember.computed('table.tableStats.columnStatsAccurate', function () { let columnStatsJson = this.get('table.tableStats.columnStatsAccurate'); return Ember.isEmpty(columnStatsJson) ? {} : JSON.parse(columnStatsJson.replace(/\\\"/g, '"')); @@ -37,12 +76,14 @@ export default Ember.Component.extend(UILoggerMixin, { return !stats ? [] : Object.keys(stats); }), - columns: Ember.computed('table.columns', 'columnsWithStatistics', function () { + columns: Ember.computed('table.columns', function () { let cols = this.get('table.columns'); - let colsWithStatistics = this.get('columnsWithStatistics'); + if(this.get("table.partitionInfo.columns")){ // show stats for all columns + cols = cols.concat(this.get("table.partitionInfo.columns")); + } return cols.map((col) => { let copy = Ember.Object.create(col); - copy.set('hasStatistics', colsWithStatistics.contains(copy.name)); + copy.set('hasStatistics', true); copy.set('isFetchingStats', false); copy.set('statsError', false); copy.set('showStats', true); @@ -50,14 +91,6 @@ export default Ember.Component.extend(UILoggerMixin, { }); }), - allColumnsHasStatistics: Ember.computed('table.columns', 'columnsWithStatistics', function () { - let colsNames = this.get('table.columns').getEach('name'); - let colsWithStatistics = this.get('columnsWithStatistics'); - - let colsNotIn = colsNames.filter((item) => !colsWithStatistics.contains(item)); - return colsNotIn.length === 0; - }), - performTableAnalysis(withColumns = false) { const tableName = this.get('table.table'); const databaseName = this.get('table.database'); @@ -94,7 +127,23 @@ export default Ember.Component.extend(UILoggerMixin, { return this.get('statsService').fetchColumnStatsResult(databaseName, tableName, column.name, job); }).then((data) => { column.set('isFetchingStats', false); - column.set('stats', data); + let colStatAccurate = data["columnStatsAccurate"]; + let colStatAccurateJson = Ember.isEmpty(colStatAccurate) ? {} : JSON.parse(colStatAccurate.replace(/\\\"/g, '"')); + if(this.get("partitionStatSupported")){ + if(!colStatAccurateJson["COLUMN_STATS"] || colStatAccurateJson["COLUMN_STATS"][column.name] === "false"){ + column.set('statsWarn', true); + column.set('statsWarnMsg', "Column statistics might be stale. Please consider recomputing with 'include columns' option checked."); + } + }else if( !this.get("partitionStatSupported") && !(this.get("columnsWithStatistics").contains(column.get("name")))){ + column.set('statsWarn', true); + column.set('statsWarnMsg', "Column statistics might be stale. Please consider recomputing with 'include columns' option checked."); + } + + let statsData = this.get("columnStatsKeys").map((item) => { + return {label: item.label, value: data[item.dataKey]}; + }); + + column.set('stats', statsData); }).catch((err) => { column.set('isFetchingStats', false); column.set('statsError', true); http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/resources/ui/app/services/stats-service.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/services/stats-service.js b/contrib/views/hive20/src/main/resources/ui/app/services/stats-service.js index 620bf9b..9d46acd 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/services/stats-service.js +++ b/contrib/views/hive20/src/main/resources/ui/app/services/stats-service.js @@ -18,17 +18,6 @@ import Ember from 'ember'; -const columnStatsKeys = [ - {dataKey: 'min', label: 'MIN'}, - {dataKey: 'max', label: 'MAX'}, - {dataKey: 'numNulls', label: 'NUMBER OF NULLS'}, - {dataKey: 'distinctCount', label: 'DISTINCT COUNT'}, - {dataKey: 'avgColLen', label: 'AVERAGE COLUMN LENGTH'}, - {dataKey: 'maxColLen', label: 'MAX COLUMN LENGTH'}, - {dataKey: 'numTrues', label: 'NUMBER OF TRUE'}, - {dataKey: 'numFalse', label: 'NUMBER OF FALSE'}, - ]; - export default Ember.Service.extend({ jobs: Ember.inject.service(), store: Ember.inject.service(), @@ -68,9 +57,7 @@ export default Ember.Service.extend({ fetchColumnStatsResult(databaseName, tableName, columnName, job) { return this.get('store').adapterFor('table').fetchColumnStats(databaseName, tableName, columnName, job.get('id')).then((data) => { let columnStats = data.columnStats; - return columnStatsKeys.map((item) => { - return {label: item.label, value: columnStats[item.dataKey]}; - }); + return columnStats; }); } }); http://git-wip-us.apache.org/repos/asf/ambari/blob/c77f4ce8/contrib/views/hive20/src/main/resources/ui/app/templates/components/table-statistics.hbs ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/templates/components/table-statistics.hbs b/contrib/views/hive20/src/main/resources/ui/app/templates/components/table-statistics.hbs index 5f62fca..cdcf762 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/templates/components/table-statistics.hbs +++ b/contrib/views/hive20/src/main/resources/ui/app/templates/components/table-statistics.hbs @@ -16,6 +16,7 @@ * limitations under the License. }} +{{#if showStats}} <div class="row"> <div class="alert"> <p class="lead"> @@ -36,6 +37,12 @@ </div> <div class="row"> {{#if tableStatisticsEnabled}} + {{#if (not basicStatsAccurate) }} + <div class="alert alert-warning"> + <p> Table statistics might be stale. Please consider recomputing statistics. + </p> + </div> + {{/if}} <div class="stats-section"> <p><strong>TABLE STATISTICS</strong></p> <table class="table table-bordered table-hover"> @@ -108,6 +115,12 @@ {{/if}} {{#if (and column.stats column.showStats)}} + {{#if column.statsWarn}} + <div class="alert alert-warning"> + <p> {{column.statsWarnMsg}} + </p> + </div> + {{/if}} <div class="col-stats-details"> <table class="table table-bordered table-hover "> <thead> @@ -126,10 +139,8 @@ </tbody> </table> </div> + {{/if}} {{/if}} - {{else}} - No statistics computed - {{/if}} </td> </tr> {{/each}} @@ -155,3 +166,11 @@ </div> {{/modal-dialog}} {{/if}} +{{else}} + <div class="alert alert-danger"> + <p> Statistics not supported for partitioned table in Hive Server version lesser than {{partitionStatSupportedVersion}}. + Your current Hive Server version is {{table.tableStats.databaseMetadata.databaseMajorVersion}}.{{table.tableStats.databaseMetadata.databaseMinorVersion}} + </p> + </div> +{{/if}} +
