[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r202108057 --- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh --- @@ -0,0 +1,59 @@ +#/usr/bin/env bash + +set -u +set -x +set -e + +function cleanup { +set +e +set +u +kdestroy +pushd ${PY_ENV_PATH}/bin +. deactivate "" +popd +rm -rf $PY_ENV_PATH +} + +trap cleanup EXIT + +echo "LAUNCHING SCRIPT" + +LOCAL_PY=$1 +PRINC=$2 +KEYTAB_LOC=$3 +KRB5_CFG_FILE=$4 +PQS_PORT=$5 +PYTHON_SCRIPT=$6 + +PY_ENV_PATH=$( mktemp -d ) + +conda create -y -p $PY_ENV_PATH || virtualenv $PY_ENV_PATH + +pushd ${PY_ENV_PATH}/bin + +# conda activate does stuff with unbound variables :( +set +u +. activate "" + +popd + +set -u +echo "INSTALLING COMPONENTS" +pip install -e file:///${LOCAL_PY}/requests-kerberos +pip install -e file:///${LOCAL_PY}/phoenixdb-module + +export KRB5_CONFIG=$KRB5_CFG_FILE +cat $KRB5_CONFIG +export KRB5_TRACE=/dev/stdout + +#echo "RUNNING KINIT" +kinit -kt $KEYTAB_LOC $PRINC --- End diff -- > I just tried on the command line and MAC OS (Heimdal) kinit does not require a directory Yeah, convention is to use `${tmpdir}/krb5cc_$(current-user uid)`. > pass the when executing python or just continue running in the same shell, which is why I stopped attempts to make ny further reductions to the shell script Oh right, I forgot they would bash the environment. Let's just let this be for now. Will be easier to come back to it later. ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r202106445 --- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh --- @@ -0,0 +1,59 @@ +#/usr/bin/env bash + +set -u +set -x +set -e + +function cleanup { +set +e +set +u +kdestroy --- End diff -- Ok, cool. I didn't think kdestroy was doing more than just cleaning up those token :) ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201717820 --- Diff: phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java --- @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LocalHBaseCluster; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; +import org.apache.hadoop.hbase.security.token.TokenProvider; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.phoenix.query.ConfigurationFactory; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.queryserver.client.ThinClientUtil; +import org.apache.phoenix.queryserver.server.QueryServer; +import org.apache.phoenix.util.InstanceResolver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.*; +import java.lang.reflect.Field; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import java.nio.file.Paths; +import java.util.Map; + +import static org.junit.Assert.*; + +@Category(NeedsOwnMiniClusterTest.class) +public class SecureQueryServerPhoenixDBIT { +private static final Log LOG = LogFactory.getLog(SecureQueryServerPhoenixDBIT.class); + +private static final File TEMP_DIR = new File(getTempDirForClass()); +private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs"); +private static final List USER_KEYTAB_FILES = new ArrayList<>(); + +private static final String SPNEGO_PRINCIPAL = "HTTP/localhost"; +private static final String PQS_PRINCIPAL = "phoenixqs/localhost"; +private static final String SERVICE_PRINCIPAL = "securecluster/localhost"; +private static File KEYTAB; + +private static MiniKdc KDC; +private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); +private static LocalHBaseCluster HBASE_CLUSTER; +private static int NUM_CREATED_USERS; + +private static ExecutorService PQS_EXECUTOR; +private static QueryServer PQS; +private static int PQS_PORT; +private static String PQS_URL; + +private static String getTempDirForClass() { +StringBuilder sb = new StringBuilder(32); +sb.append(System.getProperty("user.dir")).append(File.separator); +sb.append("target").append(File.separator); +sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName()); +return sb.toString(); +} + +private static void updateDefaultRealm() throws Exception { +// (at least) one other phoenix test triggers the caching of this field before the KDC is up +
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201716375 --- Diff: python/phoenixdb-module/phoenixdb/__init__.py --- @@ -1,11 +1,10 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# Copyright 2015 Lukas Lalinsky --- End diff -- I am not sure how that went back in, it is possible that I may have copied __init__.py from the time I was doing this work on my own before I found out that this has been moved to phoenix. I will change the header ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201482973 --- Diff: phoenix-queryserver/pom.xml --- @@ -47,6 +47,11 @@ org.apache.maven.plugins maven-failsafe-plugin + + +**/SecureQueryServerPhoenixDBIT.java --- End diff -- There are a few prerequisites - Either anaconda or virtual env *must* to be installed - System *must* provide either MIT or Heimdal kerberos utilities and libraries ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201482297 --- Diff: phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java --- @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LocalHBaseCluster; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; +import org.apache.hadoop.hbase.security.token.TokenProvider; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.phoenix.query.ConfigurationFactory; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.queryserver.client.ThinClientUtil; +import org.apache.phoenix.queryserver.server.QueryServer; +import org.apache.phoenix.util.InstanceResolver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.*; +import java.lang.reflect.Field; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import java.nio.file.Paths; +import java.util.Map; + +import static org.junit.Assert.*; + +@Category(NeedsOwnMiniClusterTest.class) +public class SecureQueryServerPhoenixDBIT { +private static final Log LOG = LogFactory.getLog(SecureQueryServerPhoenixDBIT.class); + +private static final File TEMP_DIR = new File(getTempDirForClass()); +private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs"); +private static final List USER_KEYTAB_FILES = new ArrayList<>(); + +private static final String SPNEGO_PRINCIPAL = "HTTP/localhost"; +private static final String PQS_PRINCIPAL = "phoenixqs/localhost"; +private static final String SERVICE_PRINCIPAL = "securecluster/localhost"; +private static File KEYTAB; + +private static MiniKdc KDC; +private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); +private static LocalHBaseCluster HBASE_CLUSTER; +private static int NUM_CREATED_USERS; + +private static ExecutorService PQS_EXECUTOR; +private static QueryServer PQS; +private static int PQS_PORT; +private static String PQS_URL; + +private static String getTempDirForClass() { +StringBuilder sb = new StringBuilder(32); +sb.append(System.getProperty("user.dir")).append(File.separator); +sb.append("target").append(File.separator); +sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName()); +return sb.toString(); +} + +private static void updateDefaultRealm() throws Exception { +// (at least) one other phoenix test triggers the caching of this field before the KDC is up +
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201482215 --- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh --- @@ -0,0 +1,59 @@ +#/usr/bin/env bash + +set -u +set -x +set -e + +function cleanup { +set +e +set +u +kdestroy --- End diff -- Not to be overly pedantic, but you would want to still pass krb5ccname and just call kdestroy to make sure proper cleanup is done ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user pu239ppy commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201481976 --- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh --- @@ -0,0 +1,59 @@ +#/usr/bin/env bash + +set -u +set -x +set -e + +function cleanup { +set +e +set +u +kdestroy +pushd ${PY_ENV_PATH}/bin +. deactivate "" +popd +rm -rf $PY_ENV_PATH +} + +trap cleanup EXIT + +echo "LAUNCHING SCRIPT" + +LOCAL_PY=$1 +PRINC=$2 +KEYTAB_LOC=$3 +KRB5_CFG_FILE=$4 +PQS_PORT=$5 +PYTHON_SCRIPT=$6 + +PY_ENV_PATH=$( mktemp -d ) + +conda create -y -p $PY_ENV_PATH || virtualenv $PY_ENV_PATH + +pushd ${PY_ENV_PATH}/bin + +# conda activate does stuff with unbound variables :( +set +u +. activate "" + +popd + +set -u +echo "INSTALLING COMPONENTS" +pip install -e file:///${LOCAL_PY}/requests-kerberos +pip install -e file:///${LOCAL_PY}/phoenixdb-module + +export KRB5_CONFIG=$KRB5_CFG_FILE +cat $KRB5_CONFIG +export KRB5_TRACE=/dev/stdout + +#echo "RUNNING KINIT" +kinit -kt $KEYTAB_LOC $PRINC --- End diff -- I tried something similar +File KRB5CCNAME = File.createTempFile("krb5ccname", null); +kinitEnv.put("KRB5CCNAME", KRB5CCNAME.getAbsolutePath()); This stalled, although looking at the code now it probably should have been a directory, which is why kinit stalled I can try this again ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201418146 --- Diff: phoenix-queryserver/pom.xml --- @@ -47,6 +47,11 @@ org.apache.maven.plugins maven-failsafe-plugin + + +**/SecureQueryServerPhoenixDBIT.java --- End diff -- You not intending for this test to be executed during the normal build process? ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201078376 --- Diff: python/requests-kerberos/LICENSE --- @@ -0,0 +1,15 @@ +ISC License --- End diff -- Just calling out that this is allowed: ISC is a Category-A license per https://www.apache.org/legal/resolved.html ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201077800 --- Diff: python/phoenixdb-module/phoenixdb/__init__.py --- @@ -1,11 +1,10 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# Copyright 2015 Lukas Lalinsky --- End diff -- Any reason for the re-add of this? We don't need this after the IP Clearance process, I think. NOTICE file should be sufficient. ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201076404 --- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh --- @@ -0,0 +1,59 @@ +#/usr/bin/env bash + +set -u +set -x +set -e + +function cleanup { +set +e +set +u +kdestroy --- End diff -- If we use a custom directory for the `kinit`, then this just becomes removing that custom directory. ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201075953 --- Diff: phoenix-queryserver/src/it/bin/test_phoenixdb.sh --- @@ -0,0 +1,59 @@ +#/usr/bin/env bash + +set -u +set -x +set -e + +function cleanup { +set +e +set +u +kdestroy +pushd ${PY_ENV_PATH}/bin +. deactivate "" +popd +rm -rf $PY_ENV_PATH +} + +trap cleanup EXIT + +echo "LAUNCHING SCRIPT" + +LOCAL_PY=$1 +PRINC=$2 +KEYTAB_LOC=$3 +KRB5_CFG_FILE=$4 +PQS_PORT=$5 +PYTHON_SCRIPT=$6 + +PY_ENV_PATH=$( mktemp -d ) + +conda create -y -p $PY_ENV_PATH || virtualenv $PY_ENV_PATH + +pushd ${PY_ENV_PATH}/bin + +# conda activate does stuff with unbound variables :( +set +u +. activate "" + +popd + +set -u +echo "INSTALLING COMPONENTS" +pip install -e file:///${LOCAL_PY}/requests-kerberos +pip install -e file:///${LOCAL_PY}/phoenixdb-module + +export KRB5_CONFIG=$KRB5_CFG_FILE +cat $KRB5_CONFIG +export KRB5_TRACE=/dev/stdout + +#echo "RUNNING KINIT" +kinit -kt $KEYTAB_LOC $PRINC --- End diff -- Can we kinit to a custom location? e.g. the `-c` option. Then, later, we just set the variable `KRB5CCNAME` in the shell ENV. This would help prevent us from bashing the user's ticket (if they already have one). ---
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201077202 --- Diff: phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java --- @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LocalHBaseCluster; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; +import org.apache.hadoop.hbase.security.token.TokenProvider; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.phoenix.query.ConfigurationFactory; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.queryserver.client.ThinClientUtil; +import org.apache.phoenix.queryserver.server.QueryServer; +import org.apache.phoenix.util.InstanceResolver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.*; +import java.lang.reflect.Field; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import java.nio.file.Paths; +import java.util.Map; + +import static org.junit.Assert.*; + +@Category(NeedsOwnMiniClusterTest.class) +public class SecureQueryServerPhoenixDBIT { +private static final Log LOG = LogFactory.getLog(SecureQueryServerPhoenixDBIT.class); + +private static final File TEMP_DIR = new File(getTempDirForClass()); +private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs"); +private static final List USER_KEYTAB_FILES = new ArrayList<>(); + +private static final String SPNEGO_PRINCIPAL = "HTTP/localhost"; +private static final String PQS_PRINCIPAL = "phoenixqs/localhost"; +private static final String SERVICE_PRINCIPAL = "securecluster/localhost"; +private static File KEYTAB; + +private static MiniKdc KDC; +private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); +private static LocalHBaseCluster HBASE_CLUSTER; +private static int NUM_CREATED_USERS; + +private static ExecutorService PQS_EXECUTOR; +private static QueryServer PQS; +private static int PQS_PORT; +private static String PQS_URL; + +private static String getTempDirForClass() { +StringBuilder sb = new StringBuilder(32); +sb.append(System.getProperty("user.dir")).append(File.separator); +sb.append("target").append(File.separator); +sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName()); +return sb.toString(); +} + +private static void updateDefaultRealm() throws Exception { +// (at least) one other phoenix test triggers the caching of this field before the KDC is up +
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201077355 --- Diff: phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java --- @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LocalHBaseCluster; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; +import org.apache.hadoop.hbase.security.token.TokenProvider; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.phoenix.query.ConfigurationFactory; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.queryserver.client.ThinClientUtil; +import org.apache.phoenix.queryserver.server.QueryServer; +import org.apache.phoenix.util.InstanceResolver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.*; +import java.lang.reflect.Field; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import java.nio.file.Paths; +import java.util.Map; + +import static org.junit.Assert.*; + +@Category(NeedsOwnMiniClusterTest.class) +public class SecureQueryServerPhoenixDBIT { +private static final Log LOG = LogFactory.getLog(SecureQueryServerPhoenixDBIT.class); + +private static final File TEMP_DIR = new File(getTempDirForClass()); +private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs"); +private static final List USER_KEYTAB_FILES = new ArrayList<>(); + +private static final String SPNEGO_PRINCIPAL = "HTTP/localhost"; +private static final String PQS_PRINCIPAL = "phoenixqs/localhost"; +private static final String SERVICE_PRINCIPAL = "securecluster/localhost"; +private static File KEYTAB; + +private static MiniKdc KDC; +private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); +private static LocalHBaseCluster HBASE_CLUSTER; +private static int NUM_CREATED_USERS; + +private static ExecutorService PQS_EXECUTOR; +private static QueryServer PQS; +private static int PQS_PORT; +private static String PQS_URL; + +private static String getTempDirForClass() { +StringBuilder sb = new StringBuilder(32); +sb.append(System.getProperty("user.dir")).append(File.separator); +sb.append("target").append(File.separator); +sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName()); +return sb.toString(); +} + +private static void updateDefaultRealm() throws Exception { +// (at least) one other phoenix test triggers the caching of this field before the KDC is up +
[GitHub] phoenix pull request #307: PHOENIX-4688 Kerberize python phoenixdb
Github user joshelser commented on a diff in the pull request: https://github.com/apache/phoenix/pull/307#discussion_r201076875 --- Diff: phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java --- @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LocalHBaseCluster; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; +import org.apache.hadoop.hbase.security.token.TokenProvider; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.phoenix.query.ConfigurationFactory; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.queryserver.client.ThinClientUtil; +import org.apache.phoenix.queryserver.server.QueryServer; +import org.apache.phoenix.util.InstanceResolver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.*; +import java.lang.reflect.Field; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import java.nio.file.Paths; +import java.util.Map; + +import static org.junit.Assert.*; + +@Category(NeedsOwnMiniClusterTest.class) +public class SecureQueryServerPhoenixDBIT { +private static final Log LOG = LogFactory.getLog(SecureQueryServerPhoenixDBIT.class); + +private static final File TEMP_DIR = new File(getTempDirForClass()); +private static final File KEYTAB_DIR = new File(TEMP_DIR, "keytabs"); +private static final List USER_KEYTAB_FILES = new ArrayList<>(); + +private static final String SPNEGO_PRINCIPAL = "HTTP/localhost"; +private static final String PQS_PRINCIPAL = "phoenixqs/localhost"; +private static final String SERVICE_PRINCIPAL = "securecluster/localhost"; +private static File KEYTAB; + +private static MiniKdc KDC; +private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); +private static LocalHBaseCluster HBASE_CLUSTER; +private static int NUM_CREATED_USERS; + +private static ExecutorService PQS_EXECUTOR; +private static QueryServer PQS; +private static int PQS_PORT; +private static String PQS_URL; + +private static String getTempDirForClass() { +StringBuilder sb = new StringBuilder(32); +sb.append(System.getProperty("user.dir")).append(File.separator); +sb.append("target").append(File.separator); +sb.append(SecureQueryServerPhoenixDBIT.class.getSimpleName()); +return sb.toString(); +} + +private static void updateDefaultRealm() throws Exception { +// (at least) one other phoenix test triggers the caching of this field before the KDC is up +
[GitHub] phoenix pull request #307: Phoenix 4688 Kerberize python phoenixdb
GitHub user pu239ppy opened a pull request: https://github.com/apache/phoenix/pull/307 Phoenix 4688 Kerberize python phoenixdb Lets rip out httplib and replace with requests and use requests kerberos Notes - This PR mirrors requests kerberos until such time that the maintainers of reuests-kerberos can merge https://github.com/requests/requests-kerberos/pull/115 - This is trivial comparing to the integration test required You can merge this pull request into a Git repository by running: $ git pull https://github.com/pu239ppy/phoenix PHOENIX-4688 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/phoenix/pull/307.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #307 commit 6a5448237fee59c36b167f445bfdb23ce27a308f Author: Lev Bronshtein Date: 2018-06-06T15:06:42Z moved to a separate subdirectory commit 6373e0332b24010d7b357065fface0bb7a97f0d5 Author: Lev Bronshtein Date: 2018-06-06T15:11:31Z whoops should have been ls -la commit c3c86b912f33b964f8cc607b1c43b3eb3ff56b3d Author: Lev Bronshtein Date: 2018-06-06T15:14:35Z added my fork of requests-kerberos module commit 03fc4c53b9448704d569fd4af574eecd1ea5536d Author: Lev Bronshtein Date: 2018-06-06T16:00:45Z Now with KERBEROS commit 5fb158af09150e5ab35eb9b96fd93550b93629a2 Author: Lev Bronshtein Date: 2018-06-06T17:40:08Z documentation commit e919c76760f5574477a22bee0028b58d6e1460b2 Author: Lev Bronshtein Date: 2018-06-25T22:06:52Z phoenixdb qualifier commit 3f23299553673b35aebe0f258142d999e1fe54f9 Author: Lev Bronshtein Date: 2018-06-26T02:53:59Z no need to maintain a separate directory name for forked project commit 7f2f19c30538d3db54110e7296829fffa87113c4 Author: Lev Bronshtein Date: 2018-06-26T13:34:54Z add test script to run python commit 0207cc5e9c292eda1859a4ce96e802c4bf3044fd Author: Lev Bronshtein Date: 2018-06-26T13:37:32Z make excutable commit 56c7a9a9c07d003d76813cb48243b10598d6cef2 Author: Lev Bronshtein Date: 2018-06-26T14:49:30Z pass command line parameters commit b2c7c206d830baafc39cdf833153505425187967 Author: Lev Bronshtein Date: 2018-06-26T14:54:07Z phoenix URL commit 4b6ebc1153643e9e033e84f1cb636f872468dfdd Author: Lev Bronshtein Date: 2018-06-26T21:51:38Z lets not do heredoc commit 8203449d342fcee41d0d72227ea80a7b73f62879 Author: Lev Bronshtein Date: 2018-06-26T21:52:00Z get STDOUT/ERR commit 81dd5b35d18466e0758d3b56960f33ac1d84a365 Author: Lev Bronshtein Date: 2018-06-27T11:22:33Z typo in realms commit be0f774c10fc790166f6af060d06e7e3b575df07 Author: Lev Bronshtein Date: 2018-06-27T11:23:13Z few safegurds commit ddfd1e324df83aec7c7bb425adfe435c7c36e11d Author: Lev Bronshtein Date: 2018-06-27T11:42:54Z Add KDC port to list of params commit b04a8eed246ea1987a282a34ad832d08d8b390ed Author: Lev Bronshtein Date: 2018-06-27T12:45:12Z use krb5.conf generated by the MINI KDC commit 2a4969b9cad2ad178c85804ed458b63f62e0d8dc Author: Lev Bronshtein Date: 2018-06-27T13:00:01Z use example from README commit 032879b8abeae71db016ca26a6b4f27000fb504a Author: Lev Bronshtein Date: 2018-06-27T13:00:21Z comments commit 6500a024beaa161841b731fdec09523d1c57daf4 Author: Lev Bronshtein Date: 2018-06-27T13:01:49Z lets just hardcode this, what difference doe sit make commit d7830fcfcc43fac8ff4de176554c39983e718383 Author: Lev Bronshtein Date: 2018-06-27T16:54:16Z avoiding unbound variable mech_oid commit 7fa5c5d76c74350f91d2373ff452f59810c77da7 Author: Lev Bronshtein Date: 2018-06-27T16:55:18Z have to pass PQS port as it changes on every run commit e473255835df659386f6b221c31184f6aeabc2c8 Author: Lev Bronshtein Date: 2018-06-27T17:00:38Z pass PQS port to python commit 7b17feb11cf3d73daaa0584e9359fa64998d2738 Author: Lev Bronshtein Date: 2018-06-27T17:53:34Z OS agnostic path commit 312bb27c06006b7ff12cf32ff8e422e3940a08f5 Author: Lev Bronshtein Date: 2018-06-27T19:20:36Z shell script inherits proxy settings form caller no need to set, cook up a custom heimdal krb5.conf if mac commit 10021e341d25b530fda7a69f1cd9ef37917a3c14 Author: Lev Bronshtein Date: 2018-06-27T19:42:50Z tell shell script where to find python script commit 7652edf8a4574f073f2911fe0062ba95ba799167 Author: Lev Bronshtein Date: 2018-06-27T21:38:58Z no longer need to do any cleanup commit 1aa9147566b6e2c9f5a0eb154f6c09113f143956 Author: Lev Bronshtein Date: 2018-06-28T16:55:13Z call kinit and pass along credentials commit 933328e01fd02909cda72436104f7a85e0705cb5 Author: Lev Bronshtein Date: 2018-06-28T18:47:06Z stalls while trying to execute kinit, I will leave this for someone else to figure out commit 690b5e9c116121962b585c5a97ca2ff7fe30f992 Au