Author: jnioche
Date: Fri May 16 13:32:35 2014
New Revision: 1595193

URL: http://svn.apache.org/r1595193
Log:
NUTCH-1676 Add rudimentary SSL support to protocol-http

Modified:
    nutch/trunk/CHANGES.txt
    
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
    nutch/trunk/src/plugin/protocol-http/plugin.xml
    
nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java

Modified: nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1595193&r1=1595192&r2=1595193&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Fri May 16 13:32:35 2014
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Nutch Current Development
 
+* NUTCH-1676 Add rudimentary SSL support to protocol-http (jnioche, markus)
+
 * NUTCH-1772 Injector does not need merging if no pre-existing crawldb 
(jnioche)
 
 * NUTCH-1752 Cache robots.txt rules per protocol:host:port (snagel)

Modified: 
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=1595193&r1=1595192&r2=1595193&view=diff
==============================================================================
--- 
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
 (original)
+++ 
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
 Fri May 16 13:32:35 2014
@@ -19,6 +19,9 @@ package org.apache.nutch.protocol.http.a
 // JDK imports
 import java.io.IOException;
 import java.net.URL;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
 
 // Logging imports
 import org.slf4j.Logger;
@@ -43,9 +46,6 @@ import org.apache.hadoop.io.Text;
 // crawler-commons imports
 import crawlercommons.robots.BaseRobotRules;
 
-/**
- * @author Jérôme Charron
- */
 public abstract class HttpBase implements Protocol {
   
   public static final Text RESPONSE_TIME = new Text("_rs_");
@@ -103,6 +103,12 @@ public abstract class HttpBase implement
   
   /** Skip page if Crawl-Delay longer than this value. */
   protected long maxCrawlDelay = -1L;
+  
+  /** Which TLS/SSL protocols to support */
+  protected Set<String> tlsPreferredProtocols;
+  
+  /** Which TLS/SSL cipher suites to support */
+  protected Set<String> tlsPreferredCipherSuites;
 
   /** Creates a new instance of HttpBase */
   public HttpBase() {
@@ -133,6 +139,32 @@ public abstract class HttpBase implement
       this.useHttp11 = conf.getBoolean("http.useHttp11", false);
       this.responseTime = conf.getBoolean("http.store.responsetime", true);
       this.robots.setConf(conf);
+      
+      String[] protocols = conf.getStrings("http.tls.supported.protocols", 
"TLSv1.2", "TLSv1.1", "TLSv1", "SSLv3");
+      String[] ciphers = conf.getStrings("http.tls.supported.cipher.suites", 
+          
"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
+          
"TLS_RSA_WITH_AES_256_CBC_SHA256","TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384","TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384",
+          
"TLS_DHE_RSA_WITH_AES_256_CBC_SHA256","TLS_DHE_DSS_WITH_AES_256_CBC_SHA256","TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA",
+          
"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","TLS_RSA_WITH_AES_256_CBC_SHA","TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA",
+          
"TLS_ECDH_RSA_WITH_AES_256_CBC_SHA","TLS_DHE_RSA_WITH_AES_256_CBC_SHA","TLS_DHE_DSS_WITH_AES_256_CBC_SHA",
+          
"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256","TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256","TLS_RSA_WITH_AES_128_CBC_SHA256",
+          
"TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256","TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256","TLS_DHE_RSA_WITH_AES_128_CBC_SHA256",
+          
"TLS_DHE_DSS_WITH_AES_128_CBC_SHA256","TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA","TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
+          
"TLS_RSA_WITH_AES_128_CBC_SHA","TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA","TLS_ECDH_RSA_WITH_AES_128_CBC_SHA",
+          
"TLS_DHE_RSA_WITH_AES_128_CBC_SHA","TLS_DHE_DSS_WITH_AES_128_CBC_SHA","TLS_ECDHE_ECDSA_WITH_RC4_128_SHA",
+          
"TLS_ECDHE_RSA_WITH_RC4_128_SHA","SSL_RSA_WITH_RC4_128_SHA","TLS_ECDH_ECDSA_WITH_RC4_128_SHA",
+          
"TLS_ECDH_RSA_WITH_RC4_128_SHA","TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA","TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA",
+          
"SSL_RSA_WITH_3DES_EDE_CBC_SHA","TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA","TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA",
+          
"SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA","SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA","SSL_RSA_WITH_RC4_128_MD5",
+          
"TLS_EMPTY_RENEGOTIATION_INFO_SCSV","TLS_RSA_WITH_NULL_SHA256","TLS_ECDHE_ECDSA_WITH_NULL_SHA",
+          
"TLS_ECDHE_RSA_WITH_NULL_SHA","SSL_RSA_WITH_NULL_SHA","TLS_ECDH_ECDSA_WITH_NULL_SHA","TLS_ECDH_RSA_WITH_NULL_SHA",
+          
"SSL_RSA_WITH_NULL_MD5","SSL_RSA_WITH_DES_CBC_SHA","SSL_DHE_RSA_WITH_DES_CBC_SHA","SSL_DHE_DSS_WITH_DES_CBC_SHA",
+          
"TLS_KRB5_WITH_RC4_128_SHA","TLS_KRB5_WITH_RC4_128_MD5","TLS_KRB5_WITH_3DES_EDE_CBC_SHA","TLS_KRB5_WITH_3DES_EDE_CBC_MD5",
+          "TLS_KRB5_WITH_DES_CBC_SHA","TLS_KRB5_WITH_DES_CBC_MD5");
+
+      tlsPreferredProtocols = new HashSet<String>(Arrays.asList(protocols));
+      tlsPreferredCipherSuites = new HashSet<String>(Arrays.asList(ciphers));
+
       logConf();
   }
 
@@ -258,6 +290,14 @@ public abstract class HttpBase implement
     return useHttp11;
   }
   
+  public Set<String> getTlsPreferredCipherSuites() {
+    return tlsPreferredCipherSuites;
+  }
+  
+  public Set<String> getTlsPreferredProtocols() {
+    return tlsPreferredProtocols;
+  }
+
   private static String getAgentString(String agentName,
                                        String agentVersion,
                                        String agentDesc,

Modified: nutch/trunk/src/plugin/protocol-http/plugin.xml
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-http/plugin.xml?rev=1595193&r1=1595192&r2=1595193&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-http/plugin.xml (original)
+++ nutch/trunk/src/plugin/protocol-http/plugin.xml Fri May 16 13:32:35 2014
@@ -40,6 +40,11 @@
                       class="org.apache.nutch.protocol.http.Http">
         <parameter name="protocolName" value="http"/>
       </implementation>
+      
+      <implementation id="org.apache.nutch.protocol.http.Http"
+                       class="org.apache.nutch.protocol.http.Http">
+           <parameter name="protocolName" value="https"/>
+      </implementation>
 
    </extension>
 

Modified: 
nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java?rev=1595193&r1=1595192&r2=1595193&view=diff
==============================================================================
--- 
nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
 (original)
+++ 
nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
 Fri May 16 13:32:35 2014
@@ -16,7 +16,6 @@
  */
 package org.apache.nutch.protocol.http;
 
-// JDK imports
 import java.io.BufferedInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.EOFException;
@@ -28,6 +27,13 @@ import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.URL;
 
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+ 
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.SSLSocketFactory;
+
 import org.apache.hadoop.conf.Configuration;
 
 import org.apache.nutch.crawl.CrawlDatum;
@@ -50,6 +56,11 @@ public class HttpResponse implements Res
   private byte[] content;
   private int code;
   private Metadata headers = new SpellCheckedMetadata();
+  
+  protected enum Scheme {
+    HTTP,
+    HTTPS,
+  }
 
   /**
    * Default public constructor.
@@ -66,9 +77,16 @@ public class HttpResponse implements Res
     this.url = url;
     this.orig = url.toString();
     this.base = url.toString();
-
-    if (!"http".equals(url.getProtocol()))
-      throw new HttpException("Not an HTTP url:" + url);
+        
+    Scheme scheme = null;
+ 
+    if ("http".equals(url.getProtocol())) {
+      scheme = Scheme.HTTP;
+    } else if ("https".equals(url.getProtocol())) {
+      scheme = Scheme.HTTPS;
+    } else {
+      throw new HttpException("Unknown scheme (not http/https) for url:" + 
url);
+    }
 
     if (Http.LOG.isTraceEnabled()) {
       Http.LOG.trace("fetching " + url);
@@ -84,7 +102,11 @@ public class HttpResponse implements Res
     int port;
     String portString;
     if (url.getPort() == -1) {
-      port= 80;
+      if (scheme == Scheme.HTTP) {
+        port = 80;
+      } else {
+        port = 443;
+      }
       portString= "";
     } else {
       port= url.getPort();
@@ -102,6 +124,26 @@ public class HttpResponse implements Res
       int sockPort = http.useProxy() ? http.getProxyPort() : port;
       InetSocketAddress sockAddr= new InetSocketAddress(sockHost, sockPort);
       socket.connect(sockAddr, http.getTimeout());
+     
+      if (scheme == Scheme.HTTPS) {
+        SSLSocketFactory factory = 
(SSLSocketFactory)SSLSocketFactory.getDefault();
+        SSLSocket sslsocket = (SSLSocket)factory.createSocket(socket, 
sockHost, sockPort, true);
+        sslsocket.setUseClientMode(true);
+        
+        // Get the protocols and ciphers supported by this JVM    
+        Set<String> protocols = new 
HashSet<String>(Arrays.asList(sslsocket.getSupportedProtocols()));
+        Set<String> ciphers = new 
HashSet<String>(Arrays.asList(sslsocket.getSupportedCipherSuites()));
+        
+        // Intersect with preferred protocols and ciphers
+        protocols.retainAll(http.getTlsPreferredProtocols());
+        ciphers.retainAll(http.getTlsPreferredCipherSuites());
+        
+        sslsocket.setEnabledProtocols(protocols.toArray(new 
String[protocols.size()]));
+        sslsocket.setEnabledCipherSuites(ciphers.toArray(new 
String[ciphers.size()]));
+        
+        sslsocket.startHandshake();
+        socket = sslsocket;
+      }
       
       this.conf = http.getConf();
       if (sockAddr != null


Reply via email to