Github user hanm commented on a diff in the pull request:
https://github.com/apache/zookeeper/pull/184#discussion_r111487159
--- Diff: src/java/test/org/apache/zookeeper/test/QuorumSSLTest.java ---
@@ -0,0 +1,668 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.test;
+
+import com.sun.net.httpserver.Headers;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.zookeeper.PortAssignment;
+import org.apache.zookeeper.client.ZKClientConfig;
+import org.apache.zookeeper.common.QuorumX509Util;
+import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.quorum.QuorumPeerTestBase;
+import org.bouncycastle.asn1.ocsp.OCSPResponse;
+import org.bouncycastle.asn1.ocsp.OCSPResponseStatus;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x500.X500NameBuilder;
+import org.bouncycastle.asn1.x500.style.BCStyle;
+import org.bouncycastle.asn1.x509.AuthorityInformationAccess;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.CRLDistPoint;
+import org.bouncycastle.asn1.x509.CRLNumber;
+import org.bouncycastle.asn1.x509.CRLReason;
+import org.bouncycastle.asn1.x509.DistributionPoint;
+import org.bouncycastle.asn1.x509.DistributionPointName;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.GeneralName;
+import org.bouncycastle.asn1.x509.GeneralNames;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.asn1.x509.X509ObjectIdentifiers;
+import org.bouncycastle.cert.X509CRLHolder;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.X509ExtensionUtils;
+import org.bouncycastle.cert.X509v2CRLBuilder;
+import org.bouncycastle.cert.X509v3CertificateBuilder;
+import org.bouncycastle.cert.bc.BcX509ExtensionUtils;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils;
+import org.bouncycastle.cert.jcajce.JcaX509v2CRLBuilder;
+import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
+import org.bouncycastle.cert.ocsp.BasicOCSPResp;
+import org.bouncycastle.cert.ocsp.BasicOCSPRespBuilder;
+import org.bouncycastle.cert.ocsp.CertificateID;
+import org.bouncycastle.cert.ocsp.CertificateStatus;
+import org.bouncycastle.cert.ocsp.OCSPException;
+import org.bouncycastle.cert.ocsp.OCSPReq;
+import org.bouncycastle.cert.ocsp.OCSPResp;
+import org.bouncycastle.cert.ocsp.OCSPRespBuilder;
+import org.bouncycastle.cert.ocsp.Req;
+import org.bouncycastle.cert.ocsp.UnknownStatus;
+import org.bouncycastle.cert.ocsp.jcajce.JcaBasicOCSPRespBuilder;
+import org.bouncycastle.cert.ocsp.jcajce.JcaCertificateID;
+import org.bouncycastle.crypto.util.PublicKeyFactory;
+import org.bouncycastle.crypto.util.SubjectPublicKeyInfoFactory;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.openssl.MiscPEMGenerator;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.DigestCalculator;
+import org.bouncycastle.operator.OperatorException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.bouncycastle.operator.jcajce.JcaDigestCalculatorProviderBuilder;
+import org.bouncycastle.util.io.pem.PemWriter;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigInteger;
+import java.net.InetSocketAddress;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Security;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+import static org.apache.zookeeper.test.ClientBase.createTmpDir;
+
+public class QuorumSSLTest extends QuorumPeerTestBase {
+
+ private static final String SSL_QUORUM_ENABLED = "sslQuorum=true\n";
+ private static final String PORT_UNIFICATION_ENABLED =
"portUnification=true\n";
+ private static final String PORT_UNIFICATION_DISABLED =
"portUnification=false\n";
+
+ private static final char[] PASSWORD = "testpass".toCharArray();
+ private static final String HOSTNAME = "localhost";
+
+ private QuorumX509Util quorumX509Util = new QuorumX509Util();
+
+ private MainThread q1;
+ private MainThread q2;
+ private MainThread q3;
+
+ private int clientPortQp1;
+ private int clientPortQp2;
+ private int clientPortQp3;
+
+ private String tmpDir;
+
+ private String quorumConfiguration;
+ private String validKeystorePath;
+ private String truststorePath;
+
+ private KeyPair rootKeyPair;
+ private X509Certificate rootCertificate;
+
+ private KeyPair defaultKeyPair;
+
+ private ContentSigner contentSigner;
+
+ @Before
+ public void setup() throws Exception {
+ ClientBase.setupTestEnv();
+
+ tmpDir = createTmpDir().getAbsolutePath();
+
+ clientPortQp1 = PortAssignment.unique();
+ clientPortQp2 = PortAssignment.unique();
+ clientPortQp3 = PortAssignment.unique();
+
+ validKeystorePath = tmpDir + "/valid.jks";
+ truststorePath = tmpDir + "/truststore.jks";
+
+ quorumConfiguration = generateQuorumConfiguration();
+
+ Security.addProvider(new BouncyCastleProvider());
+
+ rootKeyPair = createKeyPair();
+ contentSigner = new
JcaContentSignerBuilder("SHA256WithRSAEncryption").build(rootKeyPair.getPrivate());
+ rootCertificate = createSelfSignedCertifcate(rootKeyPair);
+
+ // Write the truststore
+ KeyStore trustStore =
KeyStore.getInstance(KeyStore.getDefaultType());
+ trustStore.load(null, PASSWORD);
+
trustStore.setCertificateEntry(rootCertificate.getSubjectDN().toString(),
rootCertificate);
+ FileOutputStream outputStream = new
FileOutputStream(truststorePath);
+ trustStore.store(outputStream, PASSWORD);
+ outputStream.flush();
+ outputStream.close();
+
+ defaultKeyPair = createKeyPair();
+ X509Certificate validCertificate =
buildEndEntityCert(defaultKeyPair, rootCertificate, rootKeyPair.getPrivate(),
+ HOSTNAME, null, null, null);
+ writeKeystore(validCertificate, defaultKeyPair, validKeystorePath);
+
+ setSSLSystemProperties();
+ }
+
+ private void writeKeystore(X509Certificate certificate, KeyPair
entityKeyPair, String path) throws Exception {
+ KeyStore keyStore =
KeyStore.getInstance(KeyStore.getDefaultType());
+ keyStore.load(null, PASSWORD);
+ keyStore.setKeyEntry("alias", entityKeyPair.getPrivate(),
PASSWORD, new Certificate[] { certificate });
+ FileOutputStream outputStream = new FileOutputStream(path);
+ keyStore.store(outputStream, PASSWORD);
+ outputStream.flush();
+ outputStream.close();
+ }
+
+
+ private class OCSPHandler implements HttpHandler {
+
+ private X509Certificate revokedCert;
+
+ // Builds an OCSPHandler that responds with a good status for all
certificates
+ // except revokedCert.
+ public OCSPHandler(X509Certificate revokedCert) {
+ this.revokedCert = revokedCert;
+ }
+
+ @Override
+ public void handle(com.sun.net.httpserver.HttpExchange
httpExchange) throws IOException {
+ byte[] responseBytes;
+ try {
+ InputStream request = httpExchange.getRequestBody();
+ byte[] requestBytes = new byte[10000];
+ request.read(requestBytes);
+
+ OCSPReq ocspRequest = new OCSPReq(requestBytes);
+ Req[] requestList = ocspRequest.getRequestList();
+
+ DigestCalculator digestCalculator = new
JcaDigestCalculatorProviderBuilder().build().get(CertificateID.HASH_SHA1);
+
+ BasicOCSPRespBuilder responseBuilder = new
JcaBasicOCSPRespBuilder(rootKeyPair.getPublic(), digestCalculator);
+ for ( Req req : requestList ) {
+ CertificateID certId = req.getCertID();
+ CertificateID revokedCertId = new
JcaCertificateID(digestCalculator, rootCertificate,
revokedCert.getSerialNumber());
+ CertificateStatus certificateStatus;
+ if (revokedCertId.equals(certId)) {
+ certificateStatus = new UnknownStatus();
+ } else {
+ certificateStatus = CertificateStatus.GOOD;
+ }
+
+ responseBuilder.addResponse(certId,
certificateStatus,null);
+ }
+
+ X509CertificateHolder[] chain = new
X509CertificateHolder[] { new JcaX509CertificateHolder(rootCertificate) };
+ ContentSigner signer = new
JcaContentSignerBuilder("SHA1withRSA").setProvider("BC").build(rootKeyPair.getPrivate());
+ BasicOCSPResp ocspResponse = responseBuilder.build(signer,
chain, Calendar.getInstance().getTime() );
+
+ responseBytes = new
OCSPRespBuilder().build(OCSPRespBuilder.SUCCESSFUL, ocspResponse).getEncoded();
+ } catch (OperatorException | CertificateEncodingException |
OCSPException exception) {
+ responseBytes = new OCSPResp(new OCSPResponse(new
OCSPResponseStatus(OCSPRespBuilder.INTERNAL_ERROR), null)).getEncoded();
+ }
+
+ Headers rh = httpExchange.getResponseHeaders();
+ rh.set("Content-Type", "application/ocsp-response");
+ httpExchange.sendResponseHeaders(200, responseBytes.length);
+
+ OutputStream os = httpExchange.getResponseBody();
+ os.write(responseBytes);
+ os.close();
+ }
+ }
+
+ private X509Certificate createSelfSignedCertifcate(KeyPair keyPair)
throws Exception {
+ X500NameBuilder nameBuilder = new
X500NameBuilder(BCStyle.INSTANCE);
+ nameBuilder.addRDN(BCStyle.CN, HOSTNAME);
+ Date notBefore = new Date(); // time from which
certificate is valid
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(notBefore);
+ cal.add(Calendar.YEAR, 1);
+ Date notAfter = cal.getTime();
+ BigInteger serialNumber = new BigInteger(128, new Random());
+
+ X509v3CertificateBuilder certificateBuilder =
+ new JcaX509v3CertificateBuilder(nameBuilder.build(),
serialNumber, notBefore, notAfter, nameBuilder.build(), keyPair.getPublic())
+ .addExtension(Extension.basicConstraints, true, new
BasicConstraints(0))
+ .addExtension(Extension.keyUsage, true, new
KeyUsage(KeyUsage.digitalSignature | KeyUsage.keyCertSign | KeyUsage.cRLSign));
+
+ return new
JcaX509CertificateConverter().getCertificate(certificateBuilder.build(contentSigner));
+ }
+
+ private void buildCRL(X509Certificate x509Certificate, String crlPath)
throws Exception {
+ X509v2CRLBuilder builder = new
JcaX509v2CRLBuilder(x509Certificate.getIssuerX500Principal(), new Date());
+ Date notBefore = new Date();
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(notBefore);
+ cal.add(Calendar.YEAR, 1);
+ Date notAfter = cal.getTime();
+ builder.setNextUpdate(notAfter);
+ builder.addCRLEntry(x509Certificate.getSerialNumber(), new Date(),
CRLReason.cACompromise);
+ builder.addExtension(Extension.authorityKeyIdentifier, false, new
JcaX509ExtensionUtils().createAuthorityKeyIdentifier(rootCertificate));
+ builder.addExtension(Extension.cRLNumber, false, new CRLNumber(new
BigInteger("1000")));
+
+ X509CRLHolder cRLHolder = builder.build(contentSigner);
+
+ PemWriter pemWriter = new PemWriter(new FileWriter(crlPath));
+ pemWriter.writeObject(new MiscPEMGenerator(cRLHolder));
+ pemWriter.flush();
+ pemWriter.close();
+ }
+
+ public X509Certificate buildEndEntityCert(KeyPair keyPair,
X509Certificate caCert, PrivateKey caPrivateKey,
+ String hostname, String
ipAddress, String crlPath, Integer ocspPort) throws Exception {
+ X509CertificateHolder holder = new
JcaX509CertificateHolder(caCert);
+ ContentSigner signer =new
JcaContentSignerBuilder("SHA256WithRSAEncryption").build(caPrivateKey);
+
+ List<GeneralName> generalNames = new ArrayList<>();
+ if (hostname != null) {
+ generalNames.add(new GeneralName(GeneralName.dNSName,
hostname));
+ }
+
+ if (ipAddress != null) {
+ generalNames.add(new GeneralName(GeneralName.iPAddress,
ipAddress));
+ }
+
+ SubjectPublicKeyInfo entityKeyInfo =
+
SubjectPublicKeyInfoFactory.createSubjectPublicKeyInfo(PublicKeyFactory.createKey(keyPair.getPublic().getEncoded()));
+ X509ExtensionUtils extensionUtils = new BcX509ExtensionUtils();
+ X509v3CertificateBuilder certificateBuilder = new
JcaX509v3CertificateBuilder(holder.getSubject(), new BigInteger(128, new
Random()),
+ new Date(System.currentTimeMillis()), new
Date(System.currentTimeMillis() + 100000),
+ new X500Name("CN=Test End Entity Certificate"),
keyPair.getPublic())
+ .addExtension(Extension.authorityKeyIdentifier, false,
extensionUtils.createAuthorityKeyIdentifier(holder))
+ .addExtension(Extension.subjectKeyIdentifier, false,
extensionUtils.createSubjectKeyIdentifier(entityKeyInfo))
+ .addExtension(Extension.basicConstraints, true, new
BasicConstraints(false))
+ .addExtension(Extension.keyUsage, true, new
KeyUsage(KeyUsage.digitalSignature | KeyUsage.keyEncipherment));
+
+ if (!generalNames.isEmpty()) {
+
certificateBuilder.addExtension(Extension.subjectAlternativeName, true, new
GeneralNames(generalNames.toArray(new GeneralName[] {})));
+ }
+
+ if (crlPath != null) {
+ DistributionPointName distPointOne = new
DistributionPointName(new GeneralNames(
+ new
GeneralName(GeneralName.uniformResourceIdentifier,"file://" + crlPath)));
+
+
certificateBuilder.addExtension(Extension.cRLDistributionPoints, false,
+ new CRLDistPoint(new DistributionPoint[] { new
DistributionPoint(distPointOne, null, null) }));
+ }
+
+ if (ocspPort != null) {
+ certificateBuilder.addExtension(Extension.authorityInfoAccess,
false, new AuthorityInformationAccess(X509ObjectIdentifiers.ocspAccessMethod,
+ new GeneralName(GeneralName.uniformResourceIdentifier,
"http://"; + hostname + ":" + ocspPort)));
+ }
+
+ return new
JcaX509CertificateConverter().getCertificate(certificateBuilder.build(signer));
+ }
+
+
+ private KeyPair createKeyPair() throws NoSuchProviderException,
NoSuchAlgorithmException {
+ KeyPairGenerator keyPairGenerator =
KeyPairGenerator.getInstance("RSA", BouncyCastleProvider.PROVIDER_NAME);
+ keyPairGenerator.initialize(4096);
+ KeyPair keyPair = keyPairGenerator.genKeyPair();
+ return keyPair;
+ }
+
+ private String generateQuorumConfiguration() {
+ int portQp1 = PortAssignment.unique();
+ int portQp2 = PortAssignment.unique();
+ int portQp3 = PortAssignment.unique();
+
+ int portLe1 = PortAssignment.unique();
+ int portLe2 = PortAssignment.unique();
+ int portLe3 = PortAssignment.unique();
+
+
+
+ return "server.1=127.0.0.1:" + (portQp1) + ":" + (portLe1) + ";" +
clientPortQp1 + "\n" +
+ "server.2=127.0.0.1:" + (portQp2) + ":" + (portLe2) + ";" +
clientPortQp2 + "\n" +
+ "server.3=127.0.0.1:" + (portQp3) + ":" + (portLe3) + ";" +
clientPortQp3;
+ }
+
+
+ public void setSSLSystemProperties() {
+
System.setProperty(ServerCnxnFactory.ZOOKEEPER_SERVER_CNXN_FACTORY,
"org.apache.zookeeper.server.NettyServerCnxnFactory");
+ System.setProperty(ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET,
"org.apache.zookeeper.ClientCnxnSocketNetty");
+
System.setProperty(quorumX509Util.getSslKeystoreLocationProperty(),
validKeystorePath);
+ System.setProperty(quorumX509Util.getSslKeystorePasswdProperty(),
"testpass");
+
System.setProperty(quorumX509Util.getSslTruststoreLocationProperty(),
truststorePath);
+
System.setProperty(quorumX509Util.getSslTruststorePasswdProperty(), "testpass");
+
System.setProperty(quorumX509Util.getSslHostnameVerificationEnabledProperty(),
"false");
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ clearSSLSystemProperties();
+ q1.shutdown();
+ q2.shutdown();
+ q3.shutdown();
+
+ Security.removeProvider("BC");
+ }
+
+ private void clearSSLSystemProperties() {
+
System.clearProperty(quorumX509Util.getSslKeystoreLocationProperty());
+
System.clearProperty(quorumX509Util.getSslKeystorePasswdProperty());
+
System.clearProperty(quorumX509Util.getSslTruststoreLocationProperty());
+
System.clearProperty(quorumX509Util.getSslTruststorePasswdProperty());
+
System.clearProperty(quorumX509Util.getSslHostnameVerificationEnabledProperty());
+ System.clearProperty(quorumX509Util.getSslOcspEnabledProperty());
+ System.clearProperty(quorumX509Util.getSslCrlEnabledProperty());
+ }
+
+ @Test(timeout = 300000)
+ public void testQuorumSSL() throws Exception {
+ q1 = new MainThread(1, clientPortQp1, quorumConfiguration,
SSL_QUORUM_ENABLED);
+ q2 = new MainThread(2, clientPortQp2, quorumConfiguration,
SSL_QUORUM_ENABLED);
+
+
+ q1.start();
+ q2.start();
+
+ Assert.assertTrue(ClientBase.waitForServerUp("127.0.0.1:" +
clientPortQp1, CONNECTION_TIMEOUT));
+ Assert.assertTrue(ClientBase.waitForServerUp("127.0.0.1:" +
clientPortQp2, CONNECTION_TIMEOUT));
+
+ clearSSLSystemProperties();
+
+ // This server should fail to join the quorum as it is not using
ssl.
+ q3 = new MainThread(3, clientPortQp3, quorumConfiguration);
+ q3.start();
+
+ Assert.assertFalse(ClientBase.waitForServerUp("127.0.0.1:" +
clientPortQp3, CONNECTION_TIMEOUT));
+ }
+
+ @Test(timeout = 300000)
+ public void testRollingUpgrade() throws Exception {
--- End diff --
This test is pretty flaky - please check internal Jenkins to diagnose.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
