Github user hanm commented on a diff in the pull request: https://github.com/apache/zookeeper/pull/184#discussion_r111487159 --- Diff: src/java/test/org/apache/zookeeper/test/QuorumSSLTest.java --- @@ -0,0 +1,668 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.test; + +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.apache.zookeeper.PortAssignment; +import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.common.QuorumX509Util; +import org.apache.zookeeper.server.ServerCnxnFactory; +import org.apache.zookeeper.server.quorum.QuorumPeerTestBase; +import org.bouncycastle.asn1.ocsp.OCSPResponse; +import org.bouncycastle.asn1.ocsp.OCSPResponseStatus; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x500.X500NameBuilder; +import org.bouncycastle.asn1.x500.style.BCStyle; +import org.bouncycastle.asn1.x509.AuthorityInformationAccess; +import org.bouncycastle.asn1.x509.BasicConstraints; +import org.bouncycastle.asn1.x509.CRLDistPoint; +import org.bouncycastle.asn1.x509.CRLNumber; +import org.bouncycastle.asn1.x509.CRLReason; +import org.bouncycastle.asn1.x509.DistributionPoint; +import org.bouncycastle.asn1.x509.DistributionPointName; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.GeneralName; +import org.bouncycastle.asn1.x509.GeneralNames; +import org.bouncycastle.asn1.x509.KeyUsage; +import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo; +import org.bouncycastle.asn1.x509.X509ObjectIdentifiers; +import org.bouncycastle.cert.X509CRLHolder; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.X509ExtensionUtils; +import org.bouncycastle.cert.X509v2CRLBuilder; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.bc.BcX509ExtensionUtils; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils; +import org.bouncycastle.cert.jcajce.JcaX509v2CRLBuilder; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.cert.ocsp.BasicOCSPResp; +import org.bouncycastle.cert.ocsp.BasicOCSPRespBuilder; +import org.bouncycastle.cert.ocsp.CertificateID; +import org.bouncycastle.cert.ocsp.CertificateStatus; +import org.bouncycastle.cert.ocsp.OCSPException; +import org.bouncycastle.cert.ocsp.OCSPReq; +import org.bouncycastle.cert.ocsp.OCSPResp; +import org.bouncycastle.cert.ocsp.OCSPRespBuilder; +import org.bouncycastle.cert.ocsp.Req; +import org.bouncycastle.cert.ocsp.UnknownStatus; +import org.bouncycastle.cert.ocsp.jcajce.JcaBasicOCSPRespBuilder; +import org.bouncycastle.cert.ocsp.jcajce.JcaCertificateID; +import org.bouncycastle.crypto.util.PublicKeyFactory; +import org.bouncycastle.crypto.util.SubjectPublicKeyInfoFactory; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openssl.MiscPEMGenerator; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.DigestCalculator; +import org.bouncycastle.operator.OperatorException; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.bouncycastle.operator.jcajce.JcaDigestCalculatorProviderBuilder; +import org.bouncycastle.util.io.pem.PemWriter; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.net.InetSocketAddress; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.PrivateKey; +import java.security.Security; +import java.security.cert.Certificate; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; +import static org.apache.zookeeper.test.ClientBase.createTmpDir; + +public class QuorumSSLTest extends QuorumPeerTestBase { + + private static final String SSL_QUORUM_ENABLED = "sslQuorum=true\n"; + private static final String PORT_UNIFICATION_ENABLED = "portUnification=true\n"; + private static final String PORT_UNIFICATION_DISABLED = "portUnification=false\n"; + + private static final char[] PASSWORD = "testpass".toCharArray(); + private static final String HOSTNAME = "localhost"; + + private QuorumX509Util quorumX509Util = new QuorumX509Util(); + + private MainThread q1; + private MainThread q2; + private MainThread q3; + + private int clientPortQp1; + private int clientPortQp2; + private int clientPortQp3; + + private String tmpDir; + + private String quorumConfiguration; + private String validKeystorePath; + private String truststorePath; + + private KeyPair rootKeyPair; + private X509Certificate rootCertificate; + + private KeyPair defaultKeyPair; + + private ContentSigner contentSigner; + + @Before + public void setup() throws Exception { + ClientBase.setupTestEnv(); + + tmpDir = createTmpDir().getAbsolutePath(); + + clientPortQp1 = PortAssignment.unique(); + clientPortQp2 = PortAssignment.unique(); + clientPortQp3 = PortAssignment.unique(); + + validKeystorePath = tmpDir + "/valid.jks"; + truststorePath = tmpDir + "/truststore.jks"; + + quorumConfiguration = generateQuorumConfiguration(); + + Security.addProvider(new BouncyCastleProvider()); + + rootKeyPair = createKeyPair(); + contentSigner = new JcaContentSignerBuilder("SHA256WithRSAEncryption").build(rootKeyPair.getPrivate()); + rootCertificate = createSelfSignedCertifcate(rootKeyPair); + + // Write the truststore + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, PASSWORD); + trustStore.setCertificateEntry(rootCertificate.getSubjectDN().toString(), rootCertificate); + FileOutputStream outputStream = new FileOutputStream(truststorePath); + trustStore.store(outputStream, PASSWORD); + outputStream.flush(); + outputStream.close(); + + defaultKeyPair = createKeyPair(); + X509Certificate validCertificate = buildEndEntityCert(defaultKeyPair, rootCertificate, rootKeyPair.getPrivate(), + HOSTNAME, null, null, null); + writeKeystore(validCertificate, defaultKeyPair, validKeystorePath); + + setSSLSystemProperties(); + } + + private void writeKeystore(X509Certificate certificate, KeyPair entityKeyPair, String path) throws Exception { + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null, PASSWORD); + keyStore.setKeyEntry("alias", entityKeyPair.getPrivate(), PASSWORD, new Certificate[] { certificate }); + FileOutputStream outputStream = new FileOutputStream(path); + keyStore.store(outputStream, PASSWORD); + outputStream.flush(); + outputStream.close(); + } + + + private class OCSPHandler implements HttpHandler { + + private X509Certificate revokedCert; + + // Builds an OCSPHandler that responds with a good status for all certificates + // except revokedCert. + public OCSPHandler(X509Certificate revokedCert) { + this.revokedCert = revokedCert; + } + + @Override + public void handle(com.sun.net.httpserver.HttpExchange httpExchange) throws IOException { + byte[] responseBytes; + try { + InputStream request = httpExchange.getRequestBody(); + byte[] requestBytes = new byte[10000]; + request.read(requestBytes); + + OCSPReq ocspRequest = new OCSPReq(requestBytes); + Req[] requestList = ocspRequest.getRequestList(); + + DigestCalculator digestCalculator = new JcaDigestCalculatorProviderBuilder().build().get(CertificateID.HASH_SHA1); + + BasicOCSPRespBuilder responseBuilder = new JcaBasicOCSPRespBuilder(rootKeyPair.getPublic(), digestCalculator); + for ( Req req : requestList ) { + CertificateID certId = req.getCertID(); + CertificateID revokedCertId = new JcaCertificateID(digestCalculator, rootCertificate, revokedCert.getSerialNumber()); + CertificateStatus certificateStatus; + if (revokedCertId.equals(certId)) { + certificateStatus = new UnknownStatus(); + } else { + certificateStatus = CertificateStatus.GOOD; + } + + responseBuilder.addResponse(certId, certificateStatus,null); + } + + X509CertificateHolder[] chain = new X509CertificateHolder[] { new JcaX509CertificateHolder(rootCertificate) }; + ContentSigner signer = new JcaContentSignerBuilder("SHA1withRSA").setProvider("BC").build(rootKeyPair.getPrivate()); + BasicOCSPResp ocspResponse = responseBuilder.build(signer, chain, Calendar.getInstance().getTime() ); + + responseBytes = new OCSPRespBuilder().build(OCSPRespBuilder.SUCCESSFUL, ocspResponse).getEncoded(); + } catch (OperatorException | CertificateEncodingException | OCSPException exception) { + responseBytes = new OCSPResp(new OCSPResponse(new OCSPResponseStatus(OCSPRespBuilder.INTERNAL_ERROR), null)).getEncoded(); + } + + Headers rh = httpExchange.getResponseHeaders(); + rh.set("Content-Type", "application/ocsp-response"); + httpExchange.sendResponseHeaders(200, responseBytes.length); + + OutputStream os = httpExchange.getResponseBody(); + os.write(responseBytes); + os.close(); + } + } + + private X509Certificate createSelfSignedCertifcate(KeyPair keyPair) throws Exception { + X500NameBuilder nameBuilder = new X500NameBuilder(BCStyle.INSTANCE); + nameBuilder.addRDN(BCStyle.CN, HOSTNAME); + Date notBefore = new Date(); // time from which certificate is valid + Calendar cal = Calendar.getInstance(); + cal.setTime(notBefore); + cal.add(Calendar.YEAR, 1); + Date notAfter = cal.getTime(); + BigInteger serialNumber = new BigInteger(128, new Random()); + + X509v3CertificateBuilder certificateBuilder = + new JcaX509v3CertificateBuilder(nameBuilder.build(), serialNumber, notBefore, notAfter, nameBuilder.build(), keyPair.getPublic()) + .addExtension(Extension.basicConstraints, true, new BasicConstraints(0)) + .addExtension(Extension.keyUsage, true, new KeyUsage(KeyUsage.digitalSignature | KeyUsage.keyCertSign | KeyUsage.cRLSign)); + + return new JcaX509CertificateConverter().getCertificate(certificateBuilder.build(contentSigner)); + } + + private void buildCRL(X509Certificate x509Certificate, String crlPath) throws Exception { + X509v2CRLBuilder builder = new JcaX509v2CRLBuilder(x509Certificate.getIssuerX500Principal(), new Date()); + Date notBefore = new Date(); + Calendar cal = Calendar.getInstance(); + cal.setTime(notBefore); + cal.add(Calendar.YEAR, 1); + Date notAfter = cal.getTime(); + builder.setNextUpdate(notAfter); + builder.addCRLEntry(x509Certificate.getSerialNumber(), new Date(), CRLReason.cACompromise); + builder.addExtension(Extension.authorityKeyIdentifier, false, new JcaX509ExtensionUtils().createAuthorityKeyIdentifier(rootCertificate)); + builder.addExtension(Extension.cRLNumber, false, new CRLNumber(new BigInteger("1000"))); + + X509CRLHolder cRLHolder = builder.build(contentSigner); + + PemWriter pemWriter = new PemWriter(new FileWriter(crlPath)); + pemWriter.writeObject(new MiscPEMGenerator(cRLHolder)); + pemWriter.flush(); + pemWriter.close(); + } + + public X509Certificate buildEndEntityCert(KeyPair keyPair, X509Certificate caCert, PrivateKey caPrivateKey, + String hostname, String ipAddress, String crlPath, Integer ocspPort) throws Exception { + X509CertificateHolder holder = new JcaX509CertificateHolder(caCert); + ContentSigner signer =new JcaContentSignerBuilder("SHA256WithRSAEncryption").build(caPrivateKey); + + List<GeneralName> generalNames = new ArrayList<>(); + if (hostname != null) { + generalNames.add(new GeneralName(GeneralName.dNSName, hostname)); + } + + if (ipAddress != null) { + generalNames.add(new GeneralName(GeneralName.iPAddress, ipAddress)); + } + + SubjectPublicKeyInfo entityKeyInfo = + SubjectPublicKeyInfoFactory.createSubjectPublicKeyInfo(PublicKeyFactory.createKey(keyPair.getPublic().getEncoded())); + X509ExtensionUtils extensionUtils = new BcX509ExtensionUtils(); + X509v3CertificateBuilder certificateBuilder = new JcaX509v3CertificateBuilder(holder.getSubject(), new BigInteger(128, new Random()), + new Date(System.currentTimeMillis()), new Date(System.currentTimeMillis() + 100000), + new X500Name("CN=Test End Entity Certificate"), keyPair.getPublic()) + .addExtension(Extension.authorityKeyIdentifier, false, extensionUtils.createAuthorityKeyIdentifier(holder)) + .addExtension(Extension.subjectKeyIdentifier, false, extensionUtils.createSubjectKeyIdentifier(entityKeyInfo)) + .addExtension(Extension.basicConstraints, true, new BasicConstraints(false)) + .addExtension(Extension.keyUsage, true, new KeyUsage(KeyUsage.digitalSignature | KeyUsage.keyEncipherment)); + + if (!generalNames.isEmpty()) { + certificateBuilder.addExtension(Extension.subjectAlternativeName, true, new GeneralNames(generalNames.toArray(new GeneralName[] {}))); + } + + if (crlPath != null) { + DistributionPointName distPointOne = new DistributionPointName(new GeneralNames( + new GeneralName(GeneralName.uniformResourceIdentifier,"file://" + crlPath))); + + certificateBuilder.addExtension(Extension.cRLDistributionPoints, false, + new CRLDistPoint(new DistributionPoint[] { new DistributionPoint(distPointOne, null, null) })); + } + + if (ocspPort != null) { + certificateBuilder.addExtension(Extension.authorityInfoAccess, false, new AuthorityInformationAccess(X509ObjectIdentifiers.ocspAccessMethod, + new GeneralName(GeneralName.uniformResourceIdentifier, "http://" + hostname + ":" + ocspPort))); + } + + return new JcaX509CertificateConverter().getCertificate(certificateBuilder.build(signer)); + } + + + private KeyPair createKeyPair() throws NoSuchProviderException, NoSuchAlgorithmException { + KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA", BouncyCastleProvider.PROVIDER_NAME); + keyPairGenerator.initialize(4096); + KeyPair keyPair = keyPairGenerator.genKeyPair(); + return keyPair; + } + + private String generateQuorumConfiguration() { + int portQp1 = PortAssignment.unique(); + int portQp2 = PortAssignment.unique(); + int portQp3 = PortAssignment.unique(); + + int portLe1 = PortAssignment.unique(); + int portLe2 = PortAssignment.unique(); + int portLe3 = PortAssignment.unique(); + + + + return "server.1=127.0.0.1:" + (portQp1) + ":" + (portLe1) + ";" + clientPortQp1 + "\n" + + "server.2=127.0.0.1:" + (portQp2) + ":" + (portLe2) + ";" + clientPortQp2 + "\n" + + "server.3=127.0.0.1:" + (portQp3) + ":" + (portLe3) + ";" + clientPortQp3; + } + + + public void setSSLSystemProperties() { + System.setProperty(ServerCnxnFactory.ZOOKEEPER_SERVER_CNXN_FACTORY, "org.apache.zookeeper.server.NettyServerCnxnFactory"); + System.setProperty(ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET, "org.apache.zookeeper.ClientCnxnSocketNetty"); + System.setProperty(quorumX509Util.getSslKeystoreLocationProperty(), validKeystorePath); + System.setProperty(quorumX509Util.getSslKeystorePasswdProperty(), "testpass"); + System.setProperty(quorumX509Util.getSslTruststoreLocationProperty(), truststorePath); + System.setProperty(quorumX509Util.getSslTruststorePasswdProperty(), "testpass"); + System.setProperty(quorumX509Util.getSslHostnameVerificationEnabledProperty(), "false"); + } + + @After + public void cleanUp() throws Exception { + clearSSLSystemProperties(); + q1.shutdown(); + q2.shutdown(); + q3.shutdown(); + + Security.removeProvider("BC"); + } + + private void clearSSLSystemProperties() { + System.clearProperty(quorumX509Util.getSslKeystoreLocationProperty()); + System.clearProperty(quorumX509Util.getSslKeystorePasswdProperty()); + System.clearProperty(quorumX509Util.getSslTruststoreLocationProperty()); + System.clearProperty(quorumX509Util.getSslTruststorePasswdProperty()); + System.clearProperty(quorumX509Util.getSslHostnameVerificationEnabledProperty()); + System.clearProperty(quorumX509Util.getSslOcspEnabledProperty()); + System.clearProperty(quorumX509Util.getSslCrlEnabledProperty()); + } + + @Test(timeout = 300000) + public void testQuorumSSL() throws Exception { + q1 = new MainThread(1, clientPortQp1, quorumConfiguration, SSL_QUORUM_ENABLED); + q2 = new MainThread(2, clientPortQp2, quorumConfiguration, SSL_QUORUM_ENABLED); + + + q1.start(); + q2.start(); + + Assert.assertTrue(ClientBase.waitForServerUp("127.0.0.1:" + clientPortQp1, CONNECTION_TIMEOUT)); + Assert.assertTrue(ClientBase.waitForServerUp("127.0.0.1:" + clientPortQp2, CONNECTION_TIMEOUT)); + + clearSSLSystemProperties(); + + // This server should fail to join the quorum as it is not using ssl. + q3 = new MainThread(3, clientPortQp3, quorumConfiguration); + q3.start(); + + Assert.assertFalse(ClientBase.waitForServerUp("127.0.0.1:" + clientPortQp3, CONNECTION_TIMEOUT)); + } + + @Test(timeout = 300000) + public void testRollingUpgrade() throws Exception { --- End diff -- This test is pretty flaky - please check internal Jenkins to diagnose.
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---