http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java index ac60880..aa26fd6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java @@ -77,8 +77,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)), - false); + ConfigValueFactory.fromAnyRef(true))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL); @@ -221,8 +220,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, ConfigValueFactory.fromAnyRef(true)) .withValue(ExecConstants.USER_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, - ConfigValueFactory.fromAnyRef(100)) - ,false); + ConfigValueFactory.fromAnyRef(100))); updateTestCluster(1, newConfig, connectionProps); @@ -259,8 +257,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)) - ,false); + ConfigValueFactory.fromAnyRef(true))); updateTestCluster(1, newConfig, connectionProps); @@ -315,8 +312,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED, ConfigValueFactory.fromAnyRef(true)) .withValue(ExecConstants.BIT_ENCRYPTION_SASL_MAX_WRAPPED_SIZE, - ConfigValueFactory.fromAnyRef(10000)) - ,false); + ConfigValueFactory.fromAnyRef(10000))); updateTestCluster(1, newConfig, connectionProps); @@ -369,8 +365,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.USE_LOGIN_PRINCIPAL, ConfigValueFactory.fromAnyRef(true)) .withValue(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)) - ,false); + ConfigValueFactory.fromAnyRef(true))); updateTestCluster(1, newConfig, connectionProps); @@ -426,8 +421,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.USE_LOGIN_PRINCIPAL, ConfigValueFactory.fromAnyRef(true)) .withValue(ExecConstants.BIT_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)) - ,false); + ConfigValueFactory.fromAnyRef(true))); updateTestCluster(1, newConfig, connectionProps); @@ -469,8 +463,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)), - false); + ConfigValueFactory.fromAnyRef(true))); updateTestCluster(1, newConfig, connectionProps); fail(); @@ -501,8 +494,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)), - false); + ConfigValueFactory.fromAnyRef(true))); updateTestCluster(1, newConfig, connectionProps); fail(); @@ -538,8 +530,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)), - false); + ConfigValueFactory.fromAnyRef(true))); updateTestCluster(1, newConfig, connectionProps); fail(); @@ -573,8 +564,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(BootStrapContext.SERVICE_KEYTAB_LOCATION, ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString())) .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, - ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))), - false); + ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))); updateTestCluster(1, newConfig, connectionProps); } @@ -602,8 +592,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(BootStrapContext.SERVICE_KEYTAB_LOCATION, ConfigValueFactory.fromAnyRef(krbHelper.serverKeytab.toString())) .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, - ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))) - , false); + ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos")))); updateTestCluster(1, newConfig, connectionProps); @@ -638,8 +627,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain", "kerberos"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)) - , false); + ConfigValueFactory.fromAnyRef(true))); updateTestCluster(1, newConfig, connectionProps); } catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java index 2fb08e3..0578d71 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java @@ -76,8 +76,7 @@ public class TestUserBitSSL extends BaseTestQuery { .withValue(ExecConstants.SSL_TRUSTSTORE_PASSWORD, ConfigValueFactory.fromAnyRef("drill123")) .withValue(ExecConstants.SSL_PROTOCOL, - ConfigValueFactory.fromAnyRef("TLSv1.2")), - false); + ConfigValueFactory.fromAnyRef("TLSv1.2"))); initProps = new Properties(); initProps.setProperty(DrillProperties.ENABLE_TLS, "true"); @@ -92,7 +91,7 @@ public class TestUserBitSSL extends BaseTestQuery { @AfterClass public static void cleanTest() throws Exception { DrillConfig restoreConfig = - new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties()), false); + new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())); updateTestCluster(1, restoreConfig); } @@ -253,7 +252,7 @@ public class TestUserBitSSL extends BaseTestQuery { .withValue(ExecConstants.SSL_KEYSTORE_TYPE, ConfigValueFactory.fromAnyRef("JKS")) .withValue(ExecConstants.SSL_KEYSTORE_PATH, ConfigValueFactory.fromAnyRef(keyStorePath)) .withValue(ExecConstants.SSL_KEYSTORE_PASSWORD, ConfigValueFactory.fromAnyRef("test_password")) - .withValue(ExecConstants.SSL_PROTOCOL, ConfigValueFactory.fromAnyRef("TLSv1.2")), false); + .withValue(ExecConstants.SSL_PROTOCOL, ConfigValueFactory.fromAnyRef("TLSv1.2"))); updateTestCluster(1, sslConfig, connectionProps); @@ -298,7 +297,7 @@ public class TestUserBitSSL extends BaseTestQuery { .withValue(ExecConstants.SSL_KEYSTORE_TYPE, ConfigValueFactory.fromAnyRef("JKS")) .withValue(ExecConstants.SSL_KEYSTORE_PATH, ConfigValueFactory.fromAnyRef(unknownKsPath)) .withValue(ExecConstants.SSL_KEYSTORE_PASSWORD, ConfigValueFactory.fromAnyRef("drill123")) - .withValue(ExecConstants.SSL_PROTOCOL, ConfigValueFactory.fromAnyRef("TLSv1.2")), false); + .withValue(ExecConstants.SSL_PROTOCOL, ConfigValueFactory.fromAnyRef("TLSv1.2"))); updateTestCluster(1, sslConfig, connectionProps); @@ -326,7 +325,7 @@ public class TestUserBitSSL extends BaseTestQuery { .withValue(ExecConstants.SSL_KEYSTORE_TYPE, ConfigValueFactory.fromAnyRef("JKS")) .withValue(ExecConstants.SSL_KEYSTORE_PATH, ConfigValueFactory.fromAnyRef(unknownKsPath)) .withValue(ExecConstants.SSL_KEYSTORE_PASSWORD, ConfigValueFactory.fromAnyRef("drill123")) - .withValue(ExecConstants.SSL_PROTOCOL, ConfigValueFactory.fromAnyRef("TLSv1.2")), false); + .withValue(ExecConstants.SSL_PROTOCOL, ConfigValueFactory.fromAnyRef("TLSv1.2"))); updateTestCluster(1, sslConfig, connectionProps); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java index 470f95f..621309a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java @@ -23,6 +23,7 @@ import org.apache.drill.test.BaseTestQuery; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.config.DrillProperties; import org.apache.drill.exec.ExecConstants; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -51,7 +52,7 @@ public class TestUserBitSSLServer extends BaseTestQuery { .withValue(ExecConstants.SSL_KEYSTORE_PATH, ConfigValueFactory.fromAnyRef(ksPath)) .withValue(ExecConstants.SSL_KEYSTORE_PASSWORD, ConfigValueFactory.fromAnyRef("drill123")) .withValue(ExecConstants.SSL_KEY_PASSWORD, ConfigValueFactory.fromAnyRef("drill123")) - .withValue(ExecConstants.SSL_PROTOCOL, ConfigValueFactory.fromAnyRef("TLSv1.2")), false); + .withValue(ExecConstants.SSL_PROTOCOL, ConfigValueFactory.fromAnyRef("TLSv1.2"))); initProps = new Properties(); initProps.setProperty(DrillProperties.ENABLE_TLS, "true"); initProps.setProperty(DrillProperties.TRUSTSTORE_PATH, tsPath); @@ -59,11 +60,17 @@ public class TestUserBitSSLServer extends BaseTestQuery { initProps.setProperty(DrillProperties.DISABLE_HOST_VERIFICATION, "true"); } + @AfterClass + public static void cleanTest() throws Exception { + DrillConfig restoreConfig = + new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())); + updateTestCluster(1, restoreConfig); + } + @Test public void testInvalidKeystorePath() throws Exception { DrillConfig testConfig = new DrillConfig(DrillConfig.create(sslConfig) - .withValue(ExecConstants.SSL_KEYSTORE_PATH, ConfigValueFactory.fromAnyRef("/bad/path")), - false); + .withValue(ExecConstants.SSL_KEYSTORE_PATH, ConfigValueFactory.fromAnyRef("/bad/path"))); // Start an SSL enabled cluster boolean failureCaught = false; @@ -78,8 +85,7 @@ public class TestUserBitSSLServer extends BaseTestQuery { @Test public void testInvalidKeystorePassword() throws Exception { DrillConfig testConfig = new DrillConfig(DrillConfig.create(sslConfig) - .withValue(ExecConstants.SSL_KEYSTORE_PASSWORD, ConfigValueFactory.fromAnyRef("badpassword")), - false); + .withValue(ExecConstants.SSL_KEYSTORE_PASSWORD, ConfigValueFactory.fromAnyRef("badpassword"))); // Start an SSL enabled cluster boolean failureCaught = false; @@ -94,8 +100,7 @@ public class TestUserBitSSLServer extends BaseTestQuery { @Test public void testInvalidKeyPassword() throws Exception { DrillConfig testConfig = new DrillConfig(DrillConfig.create(sslConfig) - .withValue(ExecConstants.SSL_KEY_PASSWORD, ConfigValueFactory.fromAnyRef("badpassword")), - false); + .withValue(ExecConstants.SSL_KEY_PASSWORD, ConfigValueFactory.fromAnyRef("badpassword"))); // Start an SSL enabled cluster boolean failureCaught = false; @@ -111,8 +116,7 @@ public class TestUserBitSSLServer extends BaseTestQuery { // Should pass because the keystore password will be used. public void testNoKeyPassword() throws Exception { DrillConfig testConfig = new DrillConfig(DrillConfig.create(sslConfig) - .withValue(ExecConstants.SSL_KEY_PASSWORD, ConfigValueFactory.fromAnyRef("")), - false); + .withValue(ExecConstants.SSL_KEY_PASSWORD, ConfigValueFactory.fromAnyRef(""))); // Start an SSL enabled cluster boolean failureCaught = false; http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java index b957462..5a4ebf9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java @@ -52,8 +52,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(false)), - false); + ConfigValueFactory.fromAnyRef(false))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.USER, "anonymous"); @@ -72,8 +71,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { final DrillConfig newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties()) .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, - ConfigValueFactory.fromAnyRef(false)), - false); + ConfigValueFactory.fromAnyRef(false))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.USER, "anonymous"); @@ -96,8 +94,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { final DrillConfig newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties()) .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, - ConfigValueFactory.fromAnyRef(false)), - false); + ConfigValueFactory.fromAnyRef(false))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.AUTH_MECHANISM, "kerberos"); @@ -120,8 +117,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { public void testDisableDrillbitAuth_EnableClientEncryption() throws Exception { final DrillConfig newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties()) .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, - ConfigValueFactory.fromAnyRef(false)), - false); + ConfigValueFactory.fromAnyRef(false))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.USER, "anonymous"); @@ -152,8 +148,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(false)), - false); + ConfigValueFactory.fromAnyRef(false))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.USER, "anonymous"); @@ -184,8 +179,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(false)), - false); + ConfigValueFactory.fromAnyRef(false))); final Properties connectionProps = new Properties(); @@ -213,8 +207,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)), - false); + ConfigValueFactory.fromAnyRef(true))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.USER, "anonymous"); @@ -244,8 +237,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(false)), - false); + ConfigValueFactory.fromAnyRef(false))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.USER, "anonymous"); @@ -274,8 +266,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))) .withValue(ExecConstants.USER_ENCRYPTION_SASL_ENABLED, - ConfigValueFactory.fromAnyRef(true)), - false); + ConfigValueFactory.fromAnyRef(true))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.USER, "anonymous"); @@ -300,8 +291,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { public void testDisableDrillbitClientAuth() throws Exception { final DrillConfig newConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties()) .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, - ConfigValueFactory.fromAnyRef(false)), - false); + ConfigValueFactory.fromAnyRef(false))); final Properties connectionProps = new Properties(); @@ -325,8 +315,7 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { .withValue(ExecConstants.IMPERSONATION_ENABLED, ConfigValueFactory.fromAnyRef(true)) .withValue(ExecConstants.IMPERSONATION_MAX_CHAINED_USER_HOPS, - ConfigValueFactory.fromAnyRef(3)), - false); + ConfigValueFactory.fromAnyRef(3))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.USER, "anonymous"); @@ -337,6 +326,5 @@ public class TestUserBitSaslCompatibility extends BaseTestQuery { fail(); } } - } http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java index 61eb85d..733de1f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java @@ -57,8 +57,7 @@ public class TestOptionsAuthEnabled extends BaseTestQuery { final DrillConfig config = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties()) .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)) .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL, - ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE)), - false); + ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))); final Properties connectionProps = new Properties(); connectionProps.setProperty(DrillProperties.USER, PROCESS_USER); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestMainLoginPageModel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestMainLoginPageModel.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestMainLoginPageModel.java index 3b36704..8e89e0c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestMainLoginPageModel.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestMainLoginPageModel.java @@ -73,7 +73,7 @@ public class TestMainLoginPageModel { public void testAuthEnabledWithNoMech() { final DrillConfig config = new DrillConfig(DrillConfig.create() .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, - ConfigValueFactory.fromAnyRef(true)), false); + ConfigValueFactory.fromAnyRef(true))); when(context.getConfig()).thenReturn(config); final MainLoginPageModel model = logInLogOutResources.new MainLoginPageModel(null); assertTrue(model.isFormEnabled()); @@ -90,7 +90,7 @@ public class TestMainLoginPageModel { .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)) .withValue(ExecConstants.HTTP_AUTHENTICATION_MECHANISMS, - ConfigValueFactory.fromIterable(Lists.newArrayList("form"))), false); + ConfigValueFactory.fromIterable(Lists.newArrayList("form")))); when(context.getConfig()).thenReturn(config); final MainLoginPageModel model = logInLogOutResources.new MainLoginPageModel(null); assertTrue(model.isFormEnabled()); @@ -107,7 +107,7 @@ public class TestMainLoginPageModel { .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)) .withValue(ExecConstants.HTTP_AUTHENTICATION_MECHANISMS, - ConfigValueFactory.fromIterable(Lists.newArrayList("spnego"))), false); + ConfigValueFactory.fromIterable(Lists.newArrayList("spnego")))); when(context.getConfig()).thenReturn(config); final MainLoginPageModel model = logInLogOutResources.new MainLoginPageModel(null); assertTrue(!model.isFormEnabled()); @@ -124,7 +124,7 @@ public class TestMainLoginPageModel { .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)) .withValue(ExecConstants.HTTP_AUTHENTICATION_MECHANISMS, - ConfigValueFactory.fromIterable(Lists.newArrayList("form", "spnego"))), false); + ConfigValueFactory.fromIterable(Lists.newArrayList("form", "spnego")))); when(context.getConfig()).thenReturn(config); final MainLoginPageModel model = logInLogOutResources.new MainLoginPageModel(null); assertTrue(model.isFormEnabled()); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java index 2b7da56..2f078c2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java @@ -100,8 +100,7 @@ public class TestDrillSpnegoAuthenticator { .withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)) .withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, - ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())), - false); + ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString()))); // Create mock objects for optionManager and AuthConfiguration final SystemOptionManager optionManager = Mockito.mock(SystemOptionManager.class); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java index 51171cd..14253e2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java @@ -101,8 +101,7 @@ public class TestSpnegoAuthentication { .withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)) .withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, - ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())), - false); + ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString()))); final ScanResult scanResult = ClassPathScanner.fromPrescan(newConfig); final AuthenticatorProviderImpl authenticatorProvider = Mockito.mock(AuthenticatorProviderImpl.class); @@ -133,8 +132,7 @@ public class TestSpnegoAuthentication { .withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)) .withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, - ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())), - false); + ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString()))); final ScanResult scanResult = ClassPathScanner.fromPrescan(newConfig); final AuthenticatorProviderImpl authenticatorProvider = Mockito.mock(AuthenticatorProviderImpl.class); @@ -166,8 +164,7 @@ public class TestSpnegoAuthentication { .withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)) .withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, - ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())), - false); + ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString()))); final ScanResult scanResult = ClassPathScanner.fromPrescan(newConfig); final AuthenticatorProviderImpl authenticatorProvider = Mockito.mock(AuthenticatorProviderImpl.class); @@ -201,8 +198,7 @@ public class TestSpnegoAuthentication { .withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)) .withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, - ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())), - false); + ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString()))); final ScanResult scanResult = ClassPathScanner.fromPrescan(newConfig); final AuthenticatorProviderImpl authenticatorProvider = Mockito.mock(AuthenticatorProviderImpl.class); @@ -230,8 +226,7 @@ public class TestSpnegoAuthentication { final DrillConfig newConfig = new DrillConfig(DrillConfig.create() .withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, - ConfigValueFactory.fromAnyRef(true)), - false); + ConfigValueFactory.fromAnyRef(true))); final ScanResult scanResult = ClassPathScanner.fromPrescan(newConfig); final AuthenticatorProviderImpl authenticatorProvider = Mockito.mock(AuthenticatorProviderImpl.class); @@ -294,8 +289,7 @@ public class TestSpnegoAuthentication { .withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)) .withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, - ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())), - false); + ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString()))); final SystemOptionManager optionManager = Mockito.mock(SystemOptionManager.class); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoConfig.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoConfig.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoConfig.java index 7803b9a..d3c77c9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoConfig.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoConfig.java @@ -89,8 +89,7 @@ public class TestSpnegoConfig { .withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))) .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL, - ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE)), - false); + ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))); final SpnegoConfig spnegoConfig = new SpnegoConfig(newConfig); spnegoConfig.validateSpnegoConfig(); @@ -107,7 +106,7 @@ public class TestSpnegoConfig { @Test public void testSpnegoConfigOnlyKeytab() throws Exception { try { - final DrillConfig newConfig = new DrillConfig(DrillConfig.create().withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)).withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))).withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())).withValue(ExecConstants.USER_AUTHENTICATOR_IMPL, ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE)), false); + final DrillConfig newConfig = new DrillConfig(DrillConfig.create().withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)).withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))).withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())).withValue(ExecConstants.USER_AUTHENTICATOR_IMPL, ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))); final SpnegoConfig spnegoConfig = new SpnegoConfig(newConfig); spnegoConfig.validateSpnegoConfig(); @@ -124,7 +123,7 @@ public class TestSpnegoConfig { @Test public void testSpnegoConfigOnlyPrincipal() throws Exception { try { - final DrillConfig newConfig = new DrillConfig(DrillConfig.create().withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)).withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))).withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)).withValue(ExecConstants.USER_AUTHENTICATOR_IMPL, ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE)), false); + final DrillConfig newConfig = new DrillConfig(DrillConfig.create().withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)).withValue(ExecConstants.AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("plain"))).withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)).withValue(ExecConstants.USER_AUTHENTICATOR_IMPL, ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))); final SpnegoConfig spnegoConfig = new SpnegoConfig(newConfig); spnegoConfig.validateSpnegoConfig(); @@ -152,8 +151,7 @@ public class TestSpnegoConfig { .withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())) .withValue(ExecConstants.USER_AUTHENTICATOR_IMPL, - ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE)), - false); + ConfigValueFactory.fromAnyRef(UserAuthenticatorTestImpl.TYPE))); final SpnegoConfig spnegoConfig = new SpnegoConfig(newConfig); spnegoConfig.validateSpnegoConfig(); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java index cdc3162..16348bc 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java @@ -21,7 +21,6 @@ import java.util.LinkedList; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.proto.CoordinationProtos; -import org.apache.drill.exec.store.parquet.ParquetGroupScan; import org.apache.hadoop.fs.BlockLocation; import org.junit.Test; @@ -29,10 +28,7 @@ import com.google.common.collect.ImmutableRangeMap; import com.google.common.collect.Range; public class TestAffinityCalculator extends ExecTest { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestAffinityCalculator.class); - - String port = "1234"; - final String path = "path"; + private final String port = "1234"; public BlockLocation[] buildBlockLocations(String[] hosts, long blockSize) { String[] names = new String[hosts.length]; @@ -50,34 +46,6 @@ public class TestAffinityCalculator extends ExecTest { return blockLocations; } - public BlockLocation[] buildBlockLocations2(String[] hosts, long blockSize) { - String[] names = new String[hosts.length]; - - for (int i = 0; i < hosts.length; i++) { - hosts[i] = "host" + i; - names[i] = "host:" + port; - } - - BlockLocation[] blockLocations = new BlockLocation[4]; - blockLocations[0] = new BlockLocation(new String[]{names[0]}, new String[]{hosts[0]}, 0, blockSize); - blockLocations[1] = new BlockLocation(new String[]{names[1]}, new String[]{hosts[1]}, blockSize, blockSize); - blockLocations[3] = new BlockLocation(new String[]{names[3]}, new String[]{hosts[3]}, blockSize*2, blockSize); - blockLocations[2] = new BlockLocation(new String[]{names[2]}, new String[]{hosts[2]}, blockSize*3, blockSize); - - return blockLocations; - } - public void buildRowGroups(LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups, int numberOfBlocks, long blockSize, int numberOfRowGroups) { - long rowGroupSize = numberOfBlocks * blockSize / numberOfRowGroups; - - rowGroups.clear(); - - for (int i = 0; i < numberOfRowGroups; i++) { - // buildRowGroups method seems not be used at all. Pass -1 as rowCount. - // Maybe remove this method completely ? - rowGroups.add(new ParquetGroupScan.RowGroupInfo(path, (long)i*rowGroupSize, (long)rowGroupSize, i, -1)); - } - } - public LinkedList<CoordinationProtos.DrillbitEndpoint> buildEndpoints(int numberOfEndpoints) { LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = new LinkedList<>(); @@ -87,61 +55,6 @@ public class TestAffinityCalculator extends ExecTest { return endPoints; } -// @Test -// public void testSetEndpointBytes(@Injectable final FileSystem fs, @Injectable final FileStatus file) throws Throwable{ -// final long blockSize = 256*1024*1024; -// LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups = new LinkedList<>(); -// int numberOfHosts = 4; -// int numberOfBlocks = 3; -// String port = "1234"; -// String[] hosts = new String[numberOfHosts]; -// -// final BlockLocation[] blockLocations = buildBlockLocations(hosts, blockSize); -// final LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = buildEndpoints(numberOfHosts); -// buildRowGroups(rowGroups, numberOfBlocks, blockSize, 3); -// -// new NonStrictExpectations() {{ -// fs.getFileBlockLocations(file, 0, 3*blockSize); result = blockLocations; -// fs.getFileStatus(new Path(path)); result = file; -// file.getLen(); result = 3*blockSize; -// }}; -// -// -// BlockMapBuilder ac = new BlockMapBuilder(fs, endPoints); -// for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) { -// ac.setEndpointBytes(rowGroup); -// } -// ParquetGroupScan.RowGroupInfo rg = rowGroups.get(0); -// Long b = rg.getEndpointBytes().get(endPoints.get(0)); -// assertEquals(blockSize,b.longValue()); -// b = rg.getEndpointBytes().get(endPoints.get(3)); -// assertNull(b); -// -// buildRowGroups(rowGroups, numberOfBlocks, blockSize, 2); -// -// ac = new BlockMapBuilder(fs, endPoints); -// for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) { -// ac.setEndpointBytes(rowGroup); -// } -// rg = rowGroups.get(0); -// b = rg.getEndpointBytes().get(endPoints.get(0)); -// assertEquals(blockSize*3/2,b.longValue()); -// b = rg.getEndpointBytes().get(endPoints.get(3)); -// assertEquals(blockSize / 2, b.longValue()); -// -// buildRowGroups(rowGroups, numberOfBlocks, blockSize, 6); -// -// ac = new BlockMapBuilder(fs, endPoints); -// for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) { -// ac.setEndpointBytes(rowGroup); -// } -// rg = rowGroups.get(0); -// b = rg.getEndpointBytes().get(endPoints.get(0)); -// assertEquals(blockSize/2,b.longValue()); -// b = rg.getEndpointBytes().get(endPoints.get(3)); -// assertNull(b); -// } - @Test public void testBuildRangeMap() { BlockLocation[] blocks = buildBlockLocations(new String[4], 256*1024*1024); @@ -157,77 +70,4 @@ public class TestAffinityCalculator extends ExecTest { long tB = System.nanoTime(); System.out.println(String.format("Took %f ms to build range map", (tB - tA) / 1e6)); } - /* - @Test - public void testApplyAssignments(@Injectable final DrillbitContext context, @Injectable final ParquetStorageEngine engine, - @Injectable final FileSystem fs, @Injectable final FileStatus file) throws IOException { - - final long blockSize = 256*1024*1024; - LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups = new LinkedList<>(); - int numberOfHosts = 4; - int numberOfBlocks = 4; - String port = "1234"; - String[] hosts = new String[numberOfHosts]; - - final BlockLocation[] blockLocations = buildBlockLocations2(hosts, blockSize); - final LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = buildEndpoints(numberOfHosts); - - new NonStrictExpectations() {{ - engine.getFileSystem(); result = fs; - engine.getContext(); result = context; - context.getBits(); result = endPoints; - fs.getFileBlockLocations(file, 0, 3*blockSize); result = blockLocations; - fs.getFileStatus(new Path(path)); result = file; - file.getLen(); result = 3*blockSize; - }}; - - buildRowGroups(rowGroups, numberOfBlocks, blockSize, 4); - ParquetGroupScan scan = new ParquetGroupScan(rowGroups, engine); - - List<EndpointAffinity> affinities = scan.getOperatorAffinity(); - - for (EndpointAffinity affinity : affinities) { - CoordinationProtos.DrillbitEndpoint db = affinity.getEndpoint(); - assertEquals((float)0.25, affinity.getAffinity(), .01); - } - - scan.applyAssignments(endPoints); - - for (int i = 0; i < endPoints.size(); i++) { - List<ParquetRowGroupScan.RowGroupReadEntry> rowGroupReadEntries = scan.getSpecificScan(i).getRowGroupReadEntries(); - assertEquals(1, rowGroupReadEntries.size()); - switch(i) { - case 0: assertEquals(0,rowGroupReadEntries.get(0).getRowGroupIndex()); - break; - case 1: assertEquals(1,rowGroupReadEntries.get(0).getRowGroupIndex()); - break; - case 2: assertEquals(3,rowGroupReadEntries.get(0).getRowGroupIndex()); - break; - case 3: assertEquals(2,rowGroupReadEntries.get(0).getRowGroupIndex()); - break; - } - } - - scan.applyAssignments(endPoints.subList(2,4)); - - List<ParquetRowGroupScan.RowGroupReadEntry> rowGroupReadEntries = scan.getSpecificScan(0).getRowGroupReadEntries(); - assertEquals(2, rowGroupReadEntries.size()); - assertEquals(3,rowGroupReadEntries.get(0).getRowGroupIndex()); - - rowGroupReadEntries = scan.getSpecificScan(1).getRowGroupReadEntries(); - assertEquals(2, rowGroupReadEntries.size()); - assertEquals(2,rowGroupReadEntries.get(0).getRowGroupIndex()); - - LinkedList<CoordinationProtos.DrillbitEndpoint> dupList = new LinkedList<>(); - dupList.add(endPoints.get(0)); - dupList.add(endPoints.get(0)); - scan.applyAssignments(dupList); - - rowGroupReadEntries = scan.getSpecificScan(0).getRowGroupReadEntries(); - assertEquals(2, rowGroupReadEntries.size()); - rowGroupReadEntries = scan.getSpecificScan(1).getRowGroupReadEntries(); - assertEquals(2, rowGroupReadEntries.size()); - } - */ - } http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java index 397c512..b8e7750 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java @@ -21,6 +21,8 @@ import static org.apache.drill.exec.store.parquet.TestFileGenerator.populateFiel import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + import io.netty.buffer.DrillBuf; import java.io.File; @@ -30,8 +32,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import mockit.Injectable; - +import org.apache.drill.exec.ops.FragmentContextImpl; import org.apache.drill.test.BaseTestQuery; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; @@ -42,7 +43,6 @@ import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.RootAllocatorFactory; -import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.proto.BitControl; import org.apache.drill.exec.proto.UserBitShared.QueryType; @@ -600,23 +600,18 @@ public class ParquetRecordReaderTest extends BaseTestQuery { @Test @Ignore - public void testPerformance(@Injectable final DrillbitContext bitContext, - @Injectable UserClientConnection connection) throws Exception { + public void testPerformance() throws Exception { + final DrillbitContext bitContext = mock(DrillbitContext.class); + final UserClientConnection connection = mock(UserClientConnection.class); + final DrillConfig c = DrillConfig.create(); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); - final FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry); - -// new NonStrictExpectations() { -// { -// context.getAllocator(); result = BufferAllocator.getAllocator(DrillConfig.create()); -// } -// }; + final FragmentContextImpl context = new FragmentContextImpl(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry); final String fileName = "/tmp/parquet_test_performance.parquet"; final HashMap<String, FieldInfo> fields = new HashMap<>(); final ParquetTestProperties props = new ParquetTestProperties(1, 20 * 1000 * 1000, DEFAULT_BYTES_PER_PAGE, fields); populateFieldInfoMap(props); - //generateParquetFile(fileName, props); final Configuration dfsConfig = new Configuration(); final List<Footer> footers = ParquetFileReader.readFooters(dfsConfig, new Path(fileName)); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java index 85501d1..98e7ef2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java @@ -21,7 +21,7 @@ package org.apache.drill.exec.store.parquet; import org.apache.commons.io.FileUtils; import org.apache.drill.PlanTestBase; import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.ops.FragmentContextImpl; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.proto.BitControl; import org.apache.hadoop.conf.Configuration; @@ -45,14 +45,14 @@ import static org.junit.Assert.assertEquals; public class TestParquetFilterPushDown extends PlanTestBase { private static final String CTAS_TABLE = "order_ctas"; - private static FragmentContext fragContext; + private static FragmentContextImpl fragContext; private static FileSystem fs; @BeforeClass public static void initFSAndCreateFragContext() throws Exception { fs = getLocalFileSystem(); - fragContext = new FragmentContext(bits[0].getContext(), + fragContext = new FragmentContextImpl(bits[0].getContext(), BitControl.PlanFragment.getDefaultInstance(), null, bits[0].getContext().getFunctionImplementationRegistry()); dirTestWatcher.copyResourceToRoot(Paths.get("parquetFilterPush")); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java index 463021f..f721ee4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java @@ -17,33 +17,32 @@ */ package org.apache.drill.exec.store.parquet; -import mockit.Mock; -import mockit.MockUp; import com.google.common.collect.Lists; -import mockit.integration.junit4.JMockit; +import org.apache.commons.io.filefilter.FalseFileFilter; +import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.drill.PlanTestBase; import org.apache.drill.categories.UnlikelyTest; import org.apache.commons.io.FileUtils; -import org.apache.drill.exec.store.dfs.MetadataContext; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import java.util.Collection; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@RunWith(JMockit.class) public class TestParquetMetadataCache extends PlanTestBase { private static final String TABLE_NAME_1 = "parquetTable1"; private static final String TABLE_NAME_2 = "parquetTable2"; @@ -631,14 +630,7 @@ public class TestParquetMetadataCache extends PlanTestBase { File rootMetadataFile = FileUtils.getFile(dataDir, Metadata.METADATA_FILENAME); assertTrue(String.format("Metadata cache file '%s' isn't deleted", rootMetadataFile.getPath()), rootMetadataFile.delete()); - // mock Metadata tableModified method to avoid occasional metadata files updating - new MockUp<Metadata>() { - @Mock - boolean tableModified(List<String> directories, org.apache.hadoop.fs.Path metaFilePath, - org.apache.hadoop.fs.Path parentDir, MetadataContext metaContext) { - return false; - } - }; + setTimestampToZero(dataDir); String query = String.format("select dir0, dir1, o_custkey, o_orderdate from `%s` " + " where dir0=1994 or dir1='Q3'", rootMetaCorruptedTable); int expectedRowCount = 60; @@ -671,14 +663,7 @@ public class TestParquetMetadataCache extends PlanTestBase { assertTrue(String.format("Metadata cache file '%s' isn't deleted", secondInnerMetadataFile.getPath()), secondInnerMetadataFile.delete()); - // mock Metadata tableModified method to avoid occasional metadata files updating - new MockUp<Metadata>() { - @Mock - boolean tableModified(List<String> directories, org.apache.hadoop.fs.Path metaFilePath, - org.apache.hadoop.fs.Path parentDir, MetadataContext metaContext) { - return false; - } - }; + setTimestampToZero(dataDir); String query = String.format("select dir0, dir1, o_custkey, o_orderdate from `%s` " + " where dir0=1994 or dir1='Q3'", innerMetaCorruptedTable); @@ -696,6 +681,14 @@ public class TestParquetMetadataCache extends PlanTestBase { } } + private void setTimestampToZero(File dataDir) throws IOException { + Collection<File> dirs = FileUtils.listFilesAndDirs(dataDir, FalseFileFilter.INSTANCE, TrueFileFilter.INSTANCE); + + for (File dir: dirs) { + Files.setLastModifiedTime(dir.toPath(), FileTime.from(0, TimeUnit.MILLISECONDS)); + } + } + @Test // DRILL-4264 @Category(UnlikelyTest.class) public void testMetadataCacheFieldWithDots() throws Exception { http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java index 6404e27..f904603 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java @@ -22,6 +22,7 @@ import static org.junit.Assert.fail; import java.util.concurrent.CountDownLatch; +import org.apache.drill.exec.ops.FragmentContextImpl; import org.apache.drill.test.BaseTestQuery; import org.apache.drill.common.concurrent.ExtendedLatch; import org.apache.drill.exec.ops.QueryContext; @@ -45,7 +46,7 @@ public class TestCountDownLatchInjection extends BaseTestQuery { /** * Class whose methods we want to simulate count down latches at run-time for testing * purposes. The class must have access to {@link org.apache.drill.exec.ops.QueryContext} or - * {@link org.apache.drill.exec.ops.FragmentContext}. + * {@link FragmentContextImpl}. */ private static class DummyClass { private static final ControlsInjector injector = ControlsInjectorFactory.getInjector(DummyClass.class); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java index 571a793..0537566 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java @@ -22,6 +22,7 @@ import static org.junit.Assert.fail; import java.util.concurrent.CountDownLatch; +import org.apache.drill.exec.ops.FragmentContextImpl; import org.apache.drill.test.BaseTestQuery; import org.apache.drill.common.concurrent.ExtendedLatch; import org.apache.drill.common.config.DrillConfig; @@ -53,7 +54,7 @@ public class TestPauseInjection extends BaseTestQuery { /** * Class whose methods we want to simulate pauses at run-time for testing * purposes. The class must have access to {@link org.apache.drill.exec.ops.QueryContext} or - * {@link org.apache.drill.exec.ops.FragmentContext}. + * {@link FragmentContextImpl}. */ private static class DummyClass { private static final Logger logger = org.slf4j.LoggerFactory.getLogger(DummyClass.class); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java index 7a62ef3..f56df77 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java @@ -20,9 +20,9 @@ package org.apache.drill.exec.util; import static org.junit.Assert.*; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.test.DrillTest; import org.apache.drill.test.OperatorFixture; -import org.apache.drill.test.OperatorFixture.OperatorFixtureBuilder; import org.junit.Test; public class TestQueryMemoryAlloc extends DrillTest { @@ -32,75 +32,85 @@ public class TestQueryMemoryAlloc extends DrillTest { @Test public void testDefaultOptions() throws Exception { - OperatorFixtureBuilder builder = OperatorFixture.builder(); + OperatorFixture.Builder builder = OperatorFixture.builder(); builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05); builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB); try (OperatorFixture fixture = builder.build()) { + final OptionManager optionManager = fixture.getOptionManager(); - // Out-of-box memory, use query memory per node as floor. + optionManager.setLocalOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05); + optionManager.setLocalOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB); - long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), fixture.options(), 8 * ONE_GB); + // Out-of-box memory, use query memory per node as floor. + long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 8 * ONE_GB); assertEquals(2 * ONE_GB, mem); // Up to 40 GB, query memory dominates. - mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), fixture.options(), 40 * ONE_GB); + mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 40 * ONE_GB); assertEquals(2 * ONE_GB, mem); // After 40 GB, the percent dominates - mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), fixture.options(), 100 * ONE_GB); + mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 100 * ONE_GB); assertEquals(5 * ONE_GB, mem); } } @Test public void testCustomFloor() throws Exception { - OperatorFixtureBuilder builder = OperatorFixture.builder(); + OperatorFixture.Builder builder = OperatorFixture.builder(); builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05); builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 3 * ONE_GB); try (OperatorFixture fixture = builder.build()) { + final OptionManager optionManager = fixture.getOptionManager(); - // Out-of-box memory, use query memory per node as floor. + optionManager.setLocalOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05); + optionManager.setLocalOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 3 * ONE_GB); - long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), fixture.options(), 8 * ONE_GB); + // Out-of-box memory, use query memory per node as floor. + long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 8 * ONE_GB); assertEquals(3 * ONE_GB, mem); // Up to 60 GB, query memory dominates. - mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), fixture.options(), 60 * ONE_GB); + mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 60 * ONE_GB); assertEquals(3 * ONE_GB, mem); // After 60 GB, the percent dominates - mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), fixture.options(), 100 * ONE_GB); + mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 100 * ONE_GB); assertEquals(5 * ONE_GB, mem); } } @Test public void testCustomPercent() throws Exception { - OperatorFixtureBuilder builder = OperatorFixture.builder(); + OperatorFixture.Builder builder = OperatorFixture.builder(); builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.10); builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB); try (OperatorFixture fixture = builder.build()) { + final OptionManager optionManager = fixture.getOptionManager(); + + optionManager.setLocalOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.10); + optionManager.setLocalOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB); // Out-of-box memory, use query memory per node as floor. - long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), fixture.options(), 8 * ONE_GB); + long mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 8 * ONE_GB); assertEquals(2 * ONE_GB, mem); // Up to 20 GB, query memory dominates. - mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), fixture.options(), 20 * ONE_GB); + mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 20 * ONE_GB); assertEquals(2 * ONE_GB, mem); // After 20 GB, the percent dominates - mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), fixture.options(), 30 * ONE_GB); + mem = MemoryAllocationUtilities.computeQueryMemory(fixture.config(), optionManager, 30 * ONE_GB); assertEquals(3 * ONE_GB, mem); } } @@ -116,21 +126,26 @@ public class TestQueryMemoryAlloc extends DrillTest { @Test public void testOpMemory() throws Exception { - OperatorFixtureBuilder builder = OperatorFixture.builder(); + OperatorFixture.Builder builder = OperatorFixture.builder(); builder.systemOption(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.7); builder.systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 10); builder.systemOption(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY, 40 * ONE_MB); try (OperatorFixture fixture = builder.build()) { + final OptionManager optionManager = fixture.getOptionManager(); + + optionManager.setLocalOption(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.7); + optionManager.setLocalOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 10); + optionManager.setLocalOption(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY, 40 * ONE_MB); // Enough memory to go above configured minimum. - long opMinMem = MemoryAllocationUtilities.computeOperatorMemory(fixture.options(), 4 * ONE_GB, 2); + long opMinMem = MemoryAllocationUtilities.computeOperatorMemory(optionManager, 4 * ONE_GB, 2); assertEquals(4 * ONE_GB / 10 / 2, opMinMem); // Too little memory per operator. Use configured minimum. - opMinMem = MemoryAllocationUtilities.computeOperatorMemory(fixture.options(), ONE_GB, 100); + opMinMem = MemoryAllocationUtilities.computeOperatorMemory(optionManager, ONE_GB, 100); assertEquals(40 * ONE_MB, opMinMem); } } http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/FragmentStatusReporterTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/FragmentStatusReporterTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/FragmentStatusReporterTest.java index 29cd6a2..7fae7a9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/FragmentStatusReporterTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/FragmentStatusReporterTest.java @@ -19,12 +19,13 @@ package org.apache.drill.exec.work.fragment; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.exec.rpc.control.Controller; import org.apache.drill.exec.rpc.control.WorkEventBus; import org.junit.Before; import org.junit.Test; import org.apache.drill.exec.memory.BufferAllocator; -import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.ops.FragmentContextImpl; import org.apache.drill.exec.ops.FragmentStats; import org.apache.drill.exec.proto.BitControl.FragmentStatus; import org.apache.drill.exec.proto.ExecProtos.FragmentHandle; @@ -47,20 +48,16 @@ import static org.apache.drill.exec.proto.UserBitShared.FragmentState.RUNNING; public class FragmentStatusReporterTest { - //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentStatusReporterTest.class); - private FragmentStatusReporter statusReporter; private ControlTunnel foremanTunnel; - private FragmentContext context; + private FragmentContextImpl context; @Before public void setUp() throws Exception { - context = mock(FragmentContext.class); - when(context.getStats()).thenReturn(mock(FragmentStats.class)); - when(context.getHandle()).thenReturn(FragmentHandle.getDefaultInstance()); - when(context.getAllocator()).thenReturn(mock(BufferAllocator.class)); + context = mock(FragmentContextImpl.class); + Controller controller = mock(Controller.class); // Create 2 different endpoint such that foremanEndpoint is different than // localEndpoint @@ -68,9 +65,14 @@ public class FragmentStatusReporterTest { DrillbitEndpoint foremanEndpoint = DrillbitEndpoint.newBuilder().setAddress("10.0.0.2").build(); foremanTunnel = mock(ControlTunnel.class); + when(context.getEndpoint()).thenReturn(localEndpoint); + when(context.getController()).thenReturn(controller); + when(controller.getTunnel(foremanEndpoint)).thenReturn(foremanTunnel); + + when(context.getStats()).thenReturn(mock(FragmentStats.class)); + when(context.getHandle()).thenReturn(FragmentHandle.getDefaultInstance()); + when(context.getAllocator()).thenReturn(mock(BufferAllocator.class)); when(context.getForemanEndpoint()).thenReturn(foremanEndpoint); - when(context.getIdentity()).thenReturn(localEndpoint); - when(context.getControlTunnel(foremanEndpoint)).thenReturn(foremanTunnel); statusReporter = new FragmentStatusReporter(context); } @@ -132,7 +134,7 @@ public class FragmentStatusReporterTest { DrillbitEndpoint localEndpoint = DrillbitEndpoint.newBuilder().setAddress("10.0.0.1").build(); - when(context.getIdentity()).thenReturn(localEndpoint); + when(context.getEndpoint()).thenReturn(localEndpoint); when(context.getForemanEndpoint()).thenReturn(localEndpoint); when(context.getWorkEventbus()).thenReturn(mock(WorkEventBus.class)); @@ -153,7 +155,7 @@ public class FragmentStatusReporterTest { public void testCloseLocalForeman() throws Exception { DrillbitEndpoint localEndpoint = DrillbitEndpoint.newBuilder().setAddress("10.0.0.1").build(); - when(context.getIdentity()).thenReturn(localEndpoint); + when(context.getEndpoint()).thenReturn(localEndpoint); when(context.getForemanEndpoint()).thenReturn(localEndpoint); when(context.getWorkEventbus()).thenReturn(mock(WorkEventBus.class)); statusReporter = new FragmentStatusReporter(context); http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java index 8ee87c0..9ddcdb7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java @@ -72,7 +72,6 @@ import static org.apache.drill.exec.util.StoragePluginTestUtils.TMP_SCHEMA; * creates the requested Drillbit and client. */ public class ClusterFixture extends BaseFixture implements AutoCloseable { - public static final String ENABLE_FULL_CACHE = "drill.exec.test.use-full-cache"; public static final int MAX_WIDTH_PER_NODE = 2; @SuppressWarnings("serial") http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java index 2871020..cdb5eb0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java @@ -21,11 +21,13 @@ import java.util.Collection; import java.util.Properties; import java.util.Map.Entry; +import com.typesafe.config.ConfigValue; import org.apache.drill.common.config.DrillConfig; import com.typesafe.config.Config; import com.typesafe.config.ConfigValueFactory; import org.apache.drill.common.map.CaseInsensitiveMap; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.server.options.OptionDefinition; import org.apache.drill.exec.server.options.SystemOptionManager; @@ -36,7 +38,7 @@ import org.apache.drill.exec.server.options.SystemOptionManager; public class ConfigBuilder { protected String configResource; - protected Properties configProps; + protected Properties configProps = new Properties(); protected CaseInsensitiveMap<OptionDefinition> definitions = SystemOptionManager.createDefaultOptionDefinitions(); /** @@ -45,17 +47,38 @@ public class ConfigBuilder { * @return this builder * @see {@link #put(String, Object)} */ - public ConfigBuilder configProps(Properties configProps) { if (hasResource()) { // Drill provides no constructor for this use case. throw new IllegalArgumentException( "Cannot provide both a config resource and config properties."); } + if (this.configProps == null) { - this.configProps = configProps; - } else { - this.configProps.putAll(configProps); + this.configProps = createDefaultProperties(); + } + + this.configProps.putAll(configProps); + + return this; + } + + public ConfigBuilder configProps(DrillConfig drillConfig) { + if (hasResource()) { + // Drill provides no constructor for this use case. + throw new IllegalArgumentException( "Cannot provide both a config resource and config properties."); } + + if (configProps == null) { + configProps = createDefaultProperties(); + } + + for (Entry<String, ConfigValue> entry: drillConfig.entrySet()) { + final Object key = entry.getKey(); + final Object value = entry.getValue().unwrapped(); + + configProps.put(key, value); + } + return this; } @@ -102,13 +125,25 @@ public class ConfigBuilder { // Drill provides no constructor for this use case. throw new IllegalArgumentException( "Cannot provide both a config resource and config properties."); } + if (configProps == null) { - configProps = new Properties(); + configProps = createDefaultProperties(); } + configProps.put(key, value.toString()); return this; } + private static Properties createDefaultProperties() + { + Properties properties = new Properties(); + properties.put(ExecConstants.CAST_TO_NULLABLE_NUMERIC, "false"); + properties.put(ExecConstants.USE_DYNAMIC_UDFS_KEY, "false"); + properties.put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false"); + + return properties; + } + public ConfigBuilder putDefinition(OptionDefinition definition) { definitions.put(definition.getValidator().getOptionName(), definition); return this; @@ -157,7 +192,7 @@ public class ConfigBuilder { ConfigValueFactory.fromAnyRef(entry.getValue())); } - return new DrillConfig(drillConfig, true); + return new DrillConfig(drillConfig); } public boolean hasResource() { http://git-wip-us.apache.org/repos/asf/drill/blob/186536d5/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java index 6135b1c..2cd5d02 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java @@ -18,35 +18,51 @@ package org.apache.drill.test; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import io.netty.buffer.DrillBuf; +import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.scanner.persistence.ScanResult; +import org.apache.drill.common.types.TypeProtos; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.compile.CodeCompiler; import org.apache.drill.exec.exception.OutOfMemoryException; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.expr.holders.ValueHolder; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.BaseFragmentContext; import org.apache.drill.exec.ops.BaseOperatorContext; import org.apache.drill.exec.ops.BufferManager; import org.apache.drill.exec.ops.BufferManagerImpl; -import org.apache.drill.exec.ops.FragmentContextInterface; +import org.apache.drill.exec.ops.ContextInformation; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.ops.MetricDef; +import org.apache.drill.exec.ops.OpProfileDef; import org.apache.drill.exec.ops.OperatorContext; +import org.apache.drill.exec.ops.OperatorStatReceiver; import org.apache.drill.exec.ops.OperatorStats; import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; +import org.apache.drill.exec.proto.ExecProtos; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.apache.drill.exec.record.TupleMetadata; import org.apache.drill.exec.record.TupleSchema; import org.apache.drill.exec.record.VectorContainer; import org.apache.drill.exec.record.selection.SelectionVector2; -import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.SystemOptionManager; +import org.apache.drill.exec.store.PartitionExplorer; +import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider; import org.apache.drill.exec.testing.ExecutionControls; import org.apache.drill.test.ClusterFixtureBuilder.RuntimeOption; import org.apache.drill.test.rowSet.DirectRowSet; @@ -83,34 +99,46 @@ import com.google.common.util.concurrent.ListenableFuture; * <li>Multiple threads of execution.</li> * </ul> */ - public class OperatorFixture extends BaseFixture implements AutoCloseable { + public OperatorContext operatorContext(PhysicalOperator config) { + return new MockOperatorContext(context, allocator(), config); + } + /** * Builds an operator fixture based on a set of config options and system/session * options. */ - - public static class OperatorFixtureBuilder + public static class Builder { - protected ConfigBuilder configBuilder = new ConfigBuilder(); - protected List<RuntimeOption> systemOptions; + protected List<RuntimeOption> systemOptions = new ArrayList<>(); protected ExecutionControls controls; + private final ConfigBuilder configBuilder = new ConfigBuilder(); + protected ExecutorService scanExecutor; + protected ExecutorService scanDecoderExecutor; + + public Builder() + { + } public ConfigBuilder configBuilder() { return configBuilder; } - public OperatorFixtureBuilder systemOption(String key, Object value) { - if (systemOptions == null) { - systemOptions = new ArrayList<>(); - } + public Builder systemOption(String key, Object value) { systemOptions.add(new RuntimeOption(key, value)); return this; } - public OperatorFixtureBuilder setControls(ExecutionControls controls) { - this.controls = controls; + public Builder setScanExecutor(final ExecutorService scanExecutor) + { + this.scanExecutor = Preconditions.checkNotNull(scanExecutor); + return this; + } + + public Builder setScanDecoderExecutor(final ExecutorService scanDecoderExecutor) + { + this.scanDecoderExecutor = Preconditions.checkNotNull(scanDecoderExecutor); return this; } @@ -124,21 +152,30 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { * uses the same code generation mechanism as the full Drill, but * provide test-specific versions of various other services. */ - - public static class TestFragmentContext extends BaseFragmentContext { - + public static class MockFragmentContext extends BaseFragmentContext { private final DrillConfig config; private final OptionManager options; private final CodeCompiler compiler; - private ExecutionControls controls; private final BufferManagerImpl bufferManager; private final BufferAllocator allocator; + private final ExecutorService scanExecutorService; + private final ExecutorService scanDecodeExecutorService; + + private ExecutorState executorState = new OperatorFixture.MockExecutorState(); + private ExecutionControls controls; - public TestFragmentContext(DrillConfig config, OptionManager options, BufferAllocator allocator) { + public MockFragmentContext(final DrillConfig config, + final OptionManager options, + final BufferAllocator allocator, + final ExecutorService scanExecutorService, + final ExecutorService scanDecodeExecutorService) { super(newFunctionRegistry(config, options)); - this.config = config; - this.options = options; - this.allocator = allocator; + this.config = Preconditions.checkNotNull(config); + this.options = Preconditions.checkNotNull(options); + this.allocator = Preconditions.checkNotNull(allocator); + this.scanExecutorService = scanExecutorService; + this.scanDecodeExecutorService = scanDecodeExecutorService; + this.controls = new ExecutionControls(options); compiler = new CodeCompiler(config, options); bufferManager = new BufferManagerImpl(allocator); } @@ -149,18 +186,14 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { return new FunctionImplementationRegistry(config, classpathScan, options); } - public void setExecutionControls(ExecutionControls controls) { - this.controls = controls; - } - @Override public OptionManager getOptions() { return options; } @Override - public boolean shouldContinue() { - return true; + public boolean isImpersonationEnabled() { + return false; } @Override @@ -174,35 +207,59 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { } @Override - public DrillbitContext getDrillbitContext() { - throw new UnsupportedOperationException("Drillbit context not available for operator unit tests"); + public ExecutorService getScanDecodeExecutor() { + return scanDecodeExecutorService; } @Override - protected CodeCompiler getCompiler() { - return compiler; + public ExecutorService getScanExecutor() { + return scanExecutorService; } @Override - protected BufferManager getBufferManager() { - return bufferManager; + public ExecutorService getExecutor() { + return null; + } + + @Override + public ExecutorState getExecutorState() { + return executorState; + } + + @Override + public BufferAllocator getNewChildAllocator(String operatorName, int operatorId, + long initialReservation, long maximumReservation) { + return allocator.newChildAllocator( + "op:" + operatorId + ":" + operatorName, + initialReservation, + maximumReservation); + } + + @Override + public ExecProtos.FragmentHandle getHandle() { + return ExecProtos.FragmentHandle.newBuilder().build(); + } + + @Override + public BufferAllocator getAllocator() { + return allocator; } @SuppressWarnings("resource") @Override public OperatorContext newOperatorContext(PhysicalOperator popConfig, - OperatorStats stats) throws OutOfMemoryException { + OperatorStats stats) throws OutOfMemoryException { BufferAllocator childAllocator = allocator.newChildAllocator( - "test:" + popConfig.getClass().getSimpleName(), - popConfig.getInitialAllocation(), - popConfig.getMaxAllocation() - ); - return new TestOperatorContext(this, childAllocator, popConfig); + "test:" + popConfig.getClass().getSimpleName(), + popConfig.getInitialAllocation(), + popConfig.getMaxAllocation() + ); + return new MockOperatorContext(this, childAllocator, popConfig); } @Override public OperatorContext newOperatorContext(PhysicalOperator popConfig) - throws OutOfMemoryException { + throws OutOfMemoryException { return newOperatorContext(popConfig, null); } @@ -210,43 +267,102 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { public String getQueryUserName() { return "fred"; } + @Override + public SchemaPlus getFullRootSchema() { + return null; + } + + @Override + public String getFragIdString() { + return null; + } + + @Override + public CodeCompiler getCompiler() { + return compiler; + } + + @Override + protected BufferManager getBufferManager() { + return bufferManager; + } + + @Override + public void close() { + bufferManager.close(); + } + + @Override + public ContextInformation getContextInformation() { + return null; + } + + @Override + public PartitionExplorer getPartitionExplorer() { + return null; + } + + @Override + public ValueHolder getConstantValueHolder(String value, TypeProtos.MinorType type, Function<DrillBuf, ValueHolder> holderInitializer) { + return null; + } } private final SystemOptionManager options; - private final TestFragmentContext context; + private final MockFragmentContext context; + private LocalPersistentStoreProvider provider; - protected OperatorFixture(OperatorFixtureBuilder builder) { + protected OperatorFixture(Builder builder) { config = builder.configBuilder().build(); allocator = RootAllocatorFactory.newRoot(config); - options = new SystemOptionManager(config); - try { - options.init(); - } catch (Exception e) { - throw new IllegalStateException("Failed to initialize the system option manager", e); - } - if (builder.systemOptions != null) { - applySystemOptions(builder.systemOptions); - } - context = new TestFragmentContext(config, options, allocator); + options = createOptionManager(); + context = new MockFragmentContext(config, options, allocator, builder.scanExecutor, builder.scanDecoderExecutor); + applySystemOptions(builder.systemOptions); } - private void applySystemOptions(List<RuntimeOption> systemOptions) { + private void applySystemOptions(List<RuntimeOption> systemOptions) { for (RuntimeOption option : systemOptions) { options.setLocalOption(option.key, option.value); } } - public SystemOptionManager options() { return options; } - public FragmentContextInterface fragmentContext() { return context; } + public OptionManager getOptionManager() + { + return options; + } + + private SystemOptionManager createOptionManager() + { + try { + provider = new LocalPersistentStoreProvider(config); + provider.start(); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + final SystemOptionManager options = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(config), provider, config); + + try { + options.init(); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + return options; + } + + public FragmentContext getFragmentContext() { return context; } @Override public void close() throws Exception { + provider.close(); + context.close(); allocator.close(); options.close(); } - public static OperatorFixtureBuilder builder() { - OperatorFixtureBuilder builder = new OperatorFixtureBuilder(); + public static Builder builder() { + Builder builder = new Builder(); builder.configBuilder() // Required to avoid Dynamic UDF calls for missing or // ambiguous functions. @@ -287,19 +403,20 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { } } - public static class TestOperatorContext extends BaseOperatorContext { + public static class MockOperatorContext extends BaseOperatorContext { + private final OperatorStats operatorStats; - private final OperatorStats stats; - - public TestOperatorContext(FragmentContextInterface fragContext, - BufferAllocator allocator, - PhysicalOperator config) { + public MockOperatorContext(FragmentContext fragContext, + BufferAllocator allocator, + PhysicalOperator config) { super(fragContext, allocator, config); - stats = new OperatorStats(100, 101, 0, allocator); + this.operatorStats = new OperatorStats(new OpProfileDef(0, 0, 100), allocator); } @Override - public OperatorStats getStats() { return stats; } + public OperatorStats getStats() { + return operatorStats; + } @Override public <RESULT> ListenableFuture<RESULT> runCallableAs( @@ -308,14 +425,38 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable { } } + public static class MockExecutorState implements FragmentContext.ExecutorState + { + @Override + public boolean shouldContinue() { + return true; + } + + @Override + public void fail(Throwable t) { + + } + + @Override + public boolean isFailed() { + return false; + } + + @Override + public Throwable getFailureCause() { + return null; + } + } + @SuppressWarnings("resource") public OperatorContext newOperatorContext(PhysicalOperator popConfig) { BufferAllocator childAllocator = allocator.newChildAllocator( - "test:" + popConfig.getClass().getSimpleName(), - popConfig.getInitialAllocation(), - popConfig.getMaxAllocation() - ); - return new TestOperatorContext(context, childAllocator, popConfig); + "test:" + popConfig.getClass().getSimpleName(), + popConfig.getInitialAllocation(), + popConfig.getMaxAllocation() + ); + + return new MockOperatorContext(context, childAllocator, popConfig); } public RowSet wrap(VectorContainer container, SelectionVector2 sv2) {