[impala] 03/03: IMPALA-10234: Add support for cookie authentication to impala-shell
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1c72c5a8f93ae3aa7e30b8bb59e0a18a654d0d34
Author: Attila Jeges
AuthorDate: Thu Oct 22 19:00:13 2020 +0200
IMPALA-10234: Add support for cookie authentication to impala-shell
IMPALA-8584 added support for cookie authentication to Impala.
This change adds cookie authentication support to impala-shell
as well when using 'hs2-http' protocol.
Testing:
- Unit tests were added to test cookie handling methods.
- Tested e2e manually with nginx HTTP proxy.
TODO:
- Test with Knox HTTP proxy as well.
Change-Id: Icb0bc6e0f58f236866ca9913a2e63d97d5148f51
Reviewed-on: http://gerrit.cloudera.org:8080/16660
Reviewed-by: Attila Jeges
Tested-by: Impala Public Jenkins
---
.../impala/customcluster/LdapImpalaShellTest.java | 38 +-
shell/ImpalaHttpClient.py | 44 ++-
shell/cookie_util.py | 69 +++
shell/impala_client.py | 12 +-
shell/impala_shell.py | 6 +-
shell/make_shell_tarball.sh| 1 +
shell/packaging/make_python_package.sh | 1 +
tests/shell/test_cookie_util.py| 129 +
8 files changed, 286 insertions(+), 14 deletions(-)
diff --git
a/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java
b/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java
index bcefb19..b0c2f42 100644
--- a/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java
+++ b/fe/src/test/java/org/apache/impala/customcluster/LdapImpalaShellTest.java
@@ -33,6 +33,8 @@ import
org.apache.directory.server.annotations.CreateTransport;
import org.apache.directory.server.core.annotations.ApplyLdifFiles;
import org.apache.directory.server.core.integ.CreateLdapServerRule;
import org.apache.impala.testutil.ImpalaJdbcClient;
+import org.apache.impala.util.Metrics;
+import com.google.common.collect.Range;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.Before;
@@ -58,6 +60,8 @@ public class LdapImpalaShellTest {
// Includes a special character to test HTTP path encoding.
private static final String delegateUser_ = "proxyUser$";
+ Metrics metrics = new Metrics();
+
public void setUp(String extraArgs) throws Exception {
String uri =
String.format("ldap://localhost:%s";,
serverRule.getLdapServer().getPort());
@@ -68,6 +72,7 @@ public class LdapImpalaShellTest {
uri, dn, extraArgs);
int ret = CustomClusterRunner.StartImpalaCluster(ldapArgs);
assertEquals(ret, 0);
+verifyMetrics(zero, zero, zero, zero);
}
/**
@@ -83,6 +88,31 @@ public class LdapImpalaShellTest {
return Boolean.parseBoolean(RunShellCommand.Run(cmd, true, "",
"").replace("\n", ""));
}
+ private void verifyMetrics(Range expectedBasicSuccess,
+ Range expectedBasicFailure, Range expectedCookieSuccess,
+ Range expectedCookieFailure) throws Exception {
+long actualBasicSuccess = (long) metrics.getMetric(
+
"impala.thrift-server.hiveserver2-http-frontend.total-basic-auth-success");
+assertTrue("Expected: " + expectedBasicSuccess + ", Actual: " +
actualBasicSuccess,
+expectedBasicSuccess.contains(actualBasicSuccess));
+long actualBasicFailure = (long) metrics.getMetric(
+
"impala.thrift-server.hiveserver2-http-frontend.total-basic-auth-failure");
+assertTrue("Expected: " + expectedBasicFailure + ", Actual: " +
actualBasicFailure,
+expectedBasicFailure.contains(actualBasicFailure));
+
+long actualCookieSuccess = (long) metrics.getMetric(
+
"impala.thrift-server.hiveserver2-http-frontend.total-cookie-auth-success");
+assertTrue("Expected: " + expectedCookieSuccess + ", Actual: " +
actualCookieSuccess,
+expectedCookieSuccess.contains(actualCookieSuccess));
+long actualCookieFailure = (long) metrics.getMetric(
+
"impala.thrift-server.hiveserver2-http-frontend.total-cookie-auth-failure");
+assertTrue("Expected: " + expectedCookieFailure + ", Actual: " +
actualCookieFailure,
+expectedCookieFailure.contains(actualCookieFailure));
+ }
+
+ private static final Range zero = Range.closed(0L, 0L);
+ private static final Range one = Range.closed(1L, 1L);
+
/**
* Tests ldap authentication using impala-shell.
*/
@@ -110,11 +140,15 @@ public class LdapImpalaShellTest {
protocolsToTest = Arrays.asList("beeswax", "hs2", "hs2-http");
}
-for (String protocol: protocolsToTest) {
- protocol = String.format(protocolTemplate, protocol);
+for (String p: protocolsToTest) {
+ String protocol = String.format(protocolTemplate, p);
validCommand[1] = protocol;
Ru
[impala] 01/03: IMPALA-7876: COMPUTE STATS TABLESAMPLE is not updating number of estimated rows
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit fa525dfdf72f6f612821a14e683cd7f16d2c423a
Author: Abhishek Rawat
AuthorDate: Wed Nov 11 07:58:11 2020 -0800
IMPALA-7876: COMPUTE STATS TABLESAMPLE is not updating number of estimated
rows
'COMPUTE STATS TABLESAMPLE' uses a child query with following function
'ROUND(COUNT(*) / )' for computing the row count.
The 'ROUND()' fn returns the row count as a DECIMAL type. The
'CatalogOpExecutor' (CatalogOpExecutor::SetTableStats) expects the row
count as a BIGINT type. Due to this data type mismatch the table stats
(Extrap #Rows) doesn't get set.
Adding an explicit CAST to BIGINT for the ROUND function results in the
table stats (Extrap #Rows) getting set properly.
Fixed both 'custom_cluster/test_stats_extrapolation.py' and
'metadata/test_stats_extrapolation.py' so that they can catch issues
like this, where table stats are not set when using
'COMPUTE STATS TABLESAMPLE'.
Testing:
- Ran core tests.
Change-Id: I88a0a777c2be9cc18b3ff293cf1c06fb499ca052
Reviewed-on: http://gerrit.cloudera.org:8080/16712
Reviewed-by: Tim Armstrong
Tested-by: Impala Public Jenkins
---
.../org/apache/impala/analysis/ComputeStatsStmt.java| 8 +---
tests/common/impala_test_suite.py | 2 +-
tests/custom_cluster/test_stats_extrapolation.py| 9 +
tests/metadata/test_stats_extrapolation.py | 17 -
4 files changed, 27 insertions(+), 9 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
index 6bb2b17..906a972 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
@@ -322,7 +322,7 @@ public class ComputeStatsStmt extends StatementBase {
*
* 2. COMPUTE STATS with TABLESAMPLE
* 2.1 Row counts:
- * SELECT ROUND(COUNT(*) / )
+ * SELECT CAST(ROUND(COUNT(*) / ) AS BIGINT)
* FROM tbl TABLESAMPLE SYSTEM() REPEATABLE ()
*
* 2.1 Column stats:
@@ -544,8 +544,10 @@ public class ComputeStatsStmt extends StatementBase {
StringBuilder tableStatsQueryBuilder = new StringBuilder("SELECT ");
String countSql = "COUNT(*)";
if (isSampling()) {
- // Extrapolate the count based on the effective sampling rate.
- countSql = String.format("ROUND(COUNT(*) / %.10f)",
effectiveSamplePerc_);
+ // Extrapolate the count based on the effective sampling rate. Add an
explicit CAST
+ // to BIGINT, which is the expected data type for row count.
+ countSql = String.format("CAST(ROUND(COUNT(*) / %.10f) AS BIGINT)",
+effectiveSamplePerc_);
}
List tableStatsSelectList = Lists.newArrayList(countSql);
// Add group by columns for incremental stats or with extrapolation
disabled.
diff --git a/tests/common/impala_test_suite.py
b/tests/common/impala_test_suite.py
index 5d1a1fa..e5fcc70 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -930,7 +930,7 @@ class ImpalaTestSuite(BaseTestSuite):
"""Returns True if 'a' and 'b' are within 'diff_perc' percent of each
other,
False otherwise. 'diff_perc' must be a float in [0,1]."""
if a == b: return True # Avoid division by 0
-assert abs(a - b) / float(max(a,b)) <= diff_perc
+assert abs(a - b) / float(max(abs(a), abs(b))) <= diff_perc
def _get_table_location(self, table_name, vector):
""" Returns the HDFS location of the table """
diff --git a/tests/custom_cluster/test_stats_extrapolation.py
b/tests/custom_cluster/test_stats_extrapolation.py
index cd1accd..9b21921 100644
--- a/tests/custom_cluster/test_stats_extrapolation.py
+++ b/tests/custom_cluster/test_stats_extrapolation.py
@@ -48,8 +48,17 @@ class TestStatsExtrapolation(CustomClusterTestSuite):
# Test COMPUTE STATS TABLESAMPLE
part_test_tbl = unique_database + ".alltypes"
self.clone_table("functional.alltypes", part_test_tbl, True, vector)
+# Since our test tables are small, set the minimum sample size to 0 to
make sure
+# we exercise the sampling code paths.
+self.client.execute("set COMPUTE_STATS_MIN_SAMPLE_SIZE=0")
self.client.execute(
"compute stats {0} tablesample system (13)".format(part_test_tbl))
+# Check that table stats were set.
+table_stats = self.client.execute("show table stats
{0}".format(part_test_tbl))
+col_names = [fs.name.upper() for fs in table_stats.schema.fieldSchemas]
+extrap_rows_idx = col_names.index("EXTRAP #ROWS")
+for row in table_stats.data:
+ assert int(row.split("\t")[extrap_rows_idx]) >= 0
# Check that column stats were set.
col_stats = self.client.execute("show
[impala] branch master updated (5a00a4c -> 1c72c5a)
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 5a00a4c IMPALA-10320: Specify expression selectivity for BoolLiteral.
new fa525df IMPALA-7876: COMPUTE STATS TABLESAMPLE is not updating number
of estimated rows
new 8a4539d IMPALA-10279: Import CPC functionality from DataSketches
new 1c72c5a IMPALA-10234: Add support for cookie authentication to
impala-shell
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exprs/CMakeLists.txt|3 +-
be/src/exprs/datasketches-test.cc | 45 +
be/src/thirdparty/datasketches/HllUtil.hpp | 18 +-
be/src/thirdparty/datasketches/README.md |9 +-
be/src/thirdparty/datasketches/binomial_bounds.hpp | 458 ++
.../{common_defs.hpp => ceiling_power_of_2.hpp}| 25 +-
be/src/thirdparty/datasketches/common_defs.hpp | 15 +
.../thirdparty/datasketches/compression_data.hpp | 6022
.../datasketches/conditional_back_inserter.hpp | 68 +
.../datasketches/conditional_forward.hpp | 70 +
be/src/thirdparty/datasketches/cpc_common.hpp | 62 +
be/src/thirdparty/datasketches/cpc_compressor.hpp | 147 +
.../datasketches/cpc_compressor_impl.hpp | 742 +++
be/src/thirdparty/datasketches/cpc_confidence.hpp | 167 +
be/src/thirdparty/datasketches/cpc_sketch.hpp | 311 +
be/src/thirdparty/datasketches/cpc_sketch_impl.hpp | 810 +++
be/src/thirdparty/datasketches/cpc_union.hpp | 102 +
be/src/thirdparty/datasketches/cpc_union_impl.hpp | 346 ++
be/src/thirdparty/datasketches/cpc_util.hpp| 137 +
be/src/thirdparty/datasketches/icon_estimator.hpp | 274 +
be/src/thirdparty/datasketches/kxp_byte_lookup.hpp | 81 +
be/src/thirdparty/datasketches/serde.hpp | 30 +-
be/src/thirdparty/datasketches/u32_table.hpp | 84 +
be/src/thirdparty/datasketches/u32_table_impl.hpp | 266 +
.../apache/impala/analysis/ComputeStatsStmt.java |8 +-
.../impala/customcluster/LdapImpalaShellTest.java | 38 +-
shell/ImpalaHttpClient.py | 44 +-
shell/cookie_util.py | 69 +
shell/impala_client.py | 12 +-
shell/impala_shell.py |6 +-
shell/make_shell_tarball.sh|1 +
shell/packaging/make_python_package.sh |1 +
tests/common/impala_test_suite.py |2 +-
tests/custom_cluster/test_stats_extrapolation.py |9 +
tests/metadata/test_stats_extrapolation.py | 17 +-
tests/shell/test_cookie_util.py| 129 +
36 files changed, 10560 insertions(+), 68 deletions(-)
create mode 100644 be/src/thirdparty/datasketches/binomial_bounds.hpp
copy be/src/thirdparty/datasketches/{common_defs.hpp =>
ceiling_power_of_2.hpp} (68%)
create mode 100644 be/src/thirdparty/datasketches/compression_data.hpp
create mode 100644 be/src/thirdparty/datasketches/conditional_back_inserter.hpp
create mode 100644 be/src/thirdparty/datasketches/conditional_forward.hpp
create mode 100644 be/src/thirdparty/datasketches/cpc_common.hpp
create mode 100644 be/src/thirdparty/datasketches/cpc_compressor.hpp
create mode 100644 be/src/thirdparty/datasketches/cpc_compressor_impl.hpp
create mode 100644 be/src/thirdparty/datasketches/cpc_confidence.hpp
create mode 100644 be/src/thirdparty/datasketches/cpc_sketch.hpp
create mode 100644 be/src/thirdparty/datasketches/cpc_sketch_impl.hpp
create mode 100644 be/src/thirdparty/datasketches/cpc_union.hpp
create mode 100644 be/src/thirdparty/datasketches/cpc_union_impl.hpp
create mode 100644 be/src/thirdparty/datasketches/cpc_util.hpp
create mode 100644 be/src/thirdparty/datasketches/icon_estimator.hpp
create mode 100644 be/src/thirdparty/datasketches/kxp_byte_lookup.hpp
create mode 100644 be/src/thirdparty/datasketches/u32_table.hpp
create mode 100644 be/src/thirdparty/datasketches/u32_table_impl.hpp
create mode 100644 shell/cookie_util.py
create mode 100644 tests/shell/test_cookie_util.py
