I fought last night with Ht://Dig 3.1.6 and ssl.9 patch on a FreeBSD
4.6-STABLE. I think I won, so here's info hoping it will help others.
Recent security patches to the OpenSSL broke the Ht://Dig+SSL
implementation. In addition to that, the ssl.9 patch uses sslv2,
which I would not recommend secure Web servers to offer. In my
installation of Apache13+modssl (apache 1.3.27 + mod_ssl 2.8.11),
Ht://Dig would fail with error message "Unable to establish the
connection". The Web log would have:
=> [error] SSL handshake failed (server xxxx:443, client xxx) (OpenSSL library error
follows)
=> [error] OpenSSL: error:1406B0CB:SSL routines:GET_CLIENT_MASTER_KEY:peer error no
cipher
Since the OpenSSL patching, I've seen only one unofficial SSL fix and
it is for VMS by Martin Vorlaender (comp.os.vms). The fix was pretty
straight forward, so I took added ssl.9 to htdig-3.1.6.tar.gz. The
Martin's VMS changed I added by hand to Connection.cc and
Connection.h. I tested it and everything seems to work fine now.
Below is the patch for those interested. It includes ssl.9 changes, so
that is not needed.
Please, review the patch as I am not a programming guru, more of an OS
administrator and security specialist.
Mari
--
-export-a-crypto-system-sig -RSA-3-lines-PERL
#!/bin/perl -sp0777i<X+d*lMLa^*lN%0]dsXx++lMlN/dsM0<j]dsj
$/=unpack('H*',$_);$_=`echo 16dio\U$k"SK$/SM$n\EsN0p[lN*1
lK[d2%Sa2/d0$^Ixp"|dc`;s/\W//g;$_=pack('H*',/((..)*)$/)
---cut---cut---here's the patch---cut---cut---cut---cut
diff -ru htdig-3.1.6/Makefile.config.in htdig-3.1.6_ssl_patched/Makefile.config.in
--- Makefile.config.in Fri Feb 1 01:47:14 2002
+++ htdig-3.1.6_ssl_patched/Makefile.config.in Fri Oct 18 08:27:37 2002
@@ -24,13 +24,13 @@
SENDMAIL= @SENDMAIL@
DEFINES= -DDEFAULT_CONFIG_FILE=\"$(DEFAULT_CONFIG_FILE)\"
-LIBDIRS= -L../htlib -L../htcommon -L../db/dist -L/usr/lib
+LIBDIRS= -L../htlib -L../htcommon -L../db/dist -L/usr/lib -L$(OPENSSL)/lib
INCS= -I$(top_srcdir)/htlib -I$(top_srcdir)/htcommon \
- -I../db/dist -I../include
+ -I../db/dist -I../include -I$(OPENSSL)/include
HTLIBS= ../htcommon/libcommon.a \
../htlib/libht.a \
../db/dist/libdb.a
-LIBS= $(HTLIBS) @LIBS@
+LIBS= $(HTLIBS) @LIBS@ -lssl -lcrypto
DIST= @PACKAGE@-@VERSION@
DISTDIR= $(top_srcdir)/../$(DIST)
diff -ru htdig-3.1.6/configure htdig-3.1.6_ssl_patched/configure
--- configure Fri Feb 1 01:47:14 2002
+++ htdig-3.1.6_ssl_patched/configure Fri Oct 18 08:27:37 2002
@@ -2193,7 +2193,7 @@
int main() {
/* Ultrix mips cc rejects this. */
-typedef int charset[2]; const charset x;
+typedef int charset[2]; const charset x = {0,0};
/* SunOS 4.1.1 cc rejects this. */
char const *const *ccp;
char **p;
diff -ru htdig-3.1.6/htcommon/DocumentDB.cc
htdig-3.1.6_ssl_patched/htcommon/DocumentDB.cc
--- htcommon/DocumentDB.cc Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htcommon/DocumentDB.cc Fri Oct 18 08:27:37 2002
@@ -231,7 +231,7 @@
while ((key = dbf->Get_Next()))
{
dbf->Get(key, data);
- if (strncmp(HtURLCodec::instance()->decode(key), "http:", 5) == 0)
+ if (strncmp(HtURLCodec::instance()->decode(key), "http:", 5) == 0 ||
+strncmp(HtURLCodec::instance()->decode(key), "https:", 6) == 0)
{
ref = new DocumentRef;
ref->Deserialize(data);
@@ -419,7 +419,7 @@
while ((coded_key = dbf->Get_Next()))
{
String key = HtURLCodec::instance()->decode(coded_key);
- if (mystrncasecmp(key, "http:", 5) == 0)
+ if (mystrncasecmp(key, "http:", 5) == 0 || mystrncasecmp(key, "https:", 6) ==
+0)
{
DocumentRef *ref = (*this)[key];
if (ref)
diff -ru htdig-3.1.6/htcommon/defaults.cc htdig-3.1.6_ssl_patched/htcommon/defaults.cc
--- htcommon/defaults.cc Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htcommon/defaults.cc Fri Oct 18 08:27:37 2002
@@ -43,7 +43,7 @@
{"boolean_syntax_errors", "Expected 'a search word' 'at the end'
'instead of' 'end of expression'"},
{"build_select_lists", ""},
{"case_sensitive", "true"},
- {"common_url_parts", "http:// http://www. ftp:// ftp://ftp. /pub/
.html .htm .gif .jpg .jpeg /index.html /index.htm .com/ .com mailto:"},
+ {"common_url_parts", "https:// https://www. http:// http://www.
+ftp:// ftp://ftp. /pub/ .html .htm .gif .jpg .jpeg /index.html /index.htm .com/ .com
+mailto:"},
{"create_image_list", "false"},
{"create_url_list", "false"},
{"compression_level", "0"},
diff -ru htdig-3.1.6/htdig/Document.cc htdig-3.1.6_ssl_patched/htdig/Document.cc
--- htdig/Document.cc Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htdig/Document.cc Fri Oct 18 08:27:34 2002
@@ -474,7 +474,7 @@
return Document_no_host;
}
}
-
+ c.assign_ssl(strcmp(url->service(), "https") == 0);
if (c.connect(1) == NOTOK)
{
if (debug)
diff -ru htdig-3.1.6/htdig/Images.cc htdig-3.1.6_ssl_patched/htdig/Images.cc
--- htdig/Images.cc Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htdig/Images.cc Fri Oct 18 08:27:34 2002
@@ -61,7 +61,7 @@
{
String u = url;
URL Url(url);
- if (strcmp(Url.service(), "http") != 0)
+ if (strcmp(Url.service(), "http") != 0 && strcmp(Url.service(), "https") != 0)
return 0;
u.lowercase();
@@ -81,6 +81,7 @@
return 0;
if (c.assign_server(Url.host()) == NOTOK)
return 0;
+ c.assign_ssl(strcmp(Url.service(), "https") == 0);
if (c.connect(1) == NOTOK)
{
diff -ru htdig-3.1.6/htdig/Retriever.cc htdig-3.1.6_ssl_patched/htdig/Retriever.cc
--- htdig/Retriever.cc Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htdig/Retriever.cc Fri Oct 18 08:27:34 2002
@@ -117,8 +117,7 @@
// from == 2 add url from db.log
// from == 3 urls in db.docs and there was a db.log
//
-void
-Retriever::Initial(char *list, int from)
+void Retriever::Initial(char *list, int from)
{
//
// Split the list of urls up into individual urls.
@@ -133,14 +132,15 @@
URL u(tokens[i]);
server = (Server *) servers[u.signature()];
url = u.get();
+cout << "URL: " << url;
if (debug > 2)
cout << "\t" << from << ":" << (int) log << ":" << url;
if (!server)
{
- String robotsURL = "http://";;
- robotsURL << u.host() << "/robots.txt";
+ String robotsURL = u.service();
+ robotsURL << "://" << u.host() << "/robots.txt";
StringList *localRobotsFiles = GetLocal(robotsURL.get());
- server = new Server(u.host(), u.port(), localRobotsFiles);
+ server = new Server(u.host(), u.port(), strcmp(u.service(), "https") == 0,
+localRobotsFiles);
servers.Add(u.signature(), server);
delete localRobotsFiles;
}
@@ -679,10 +679,10 @@
// Currently, we only deal with HTTP URLs. Gopher and ftp will
// come later... ***FIX***
//
- if (strstr(u, "/../") || strncmp(u, "http://";, 7) != 0)
+ if (strstr(u, "/../") || (strncmp(u, "http://";, 7) != 0 && strncmp(u, "https://";,
+8) != 0))
{
if (debug > 2)
- cout << endl <<" Rejected: Not an http or relative link!";
+ cout << endl <<" Rejected: Not an http, https or relative link!";
return FALSE;
}
@@ -1365,10 +1365,10 @@
//
// Hadn't seen this server, yet. Register it
//
- String robotsURL = "http://";;
- robotsURL << url.host() << "/robots.txt";
+ String robotsURL = url.service();
+ robotsURL << "://" << url.host() << "/robots.txt";
StringList *localRobotsFile = GetLocal(robotsURL.get());
- server = new Server(url.host(), url.port(), localRobotsFile);
+ server = new Server(url.host(), url.port(), (int)
+(strcmp(url.service(), "https") == 0), localRobotsFile);
servers.Add(url.signature(), server);
delete localRobotsFile;
}
@@ -1509,10 +1509,10 @@
//
// Hadn't seen this server, yet. Register it
//
- String robotsURL = "http://";;
- robotsURL << url.host() << "/robots.txt";
+ String robotsURL = url.service();
+ robotsURL << "://" << url.host() << "/robots.txt";
StringList *localRobotsFile = GetLocal(robotsURL.get());
- server = new Server(url.host(), url.port(), localRobotsFile);
+ server = new Server(url.host(), url.port(), strcmp(url.service(),
+"https") == 0, localRobotsFile);
servers.Add(url.signature(), server);
delete localRobotsFile;
}
diff -ru htdig-3.1.6/htdig/Server.cc htdig-3.1.6_ssl_patched/htdig/Server.cc
--- htdig/Server.cc Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htdig/Server.cc Fri Oct 18 08:27:34 2002
@@ -20,9 +20,9 @@
//*****************************************************************************
-// Server::Server(char *host, int port, StringList *local_robots_files)
+// Server::Server(char *host, int port, int ssl, StringList *local_robots_file)
//
-Server::Server(char *host, int port, StringList *local_robots_files)
+Server::Server(char *host, int port, int ssl, StringList *local_robots_files)
{
if (debug > 0)
cout << endl << "New server: " << host << ", " << port << endl;
@@ -40,7 +40,8 @@
//
// Attempt to get a robots.txt file from the specified server
//
- String url = "http://";;
+ String url = "http://";;
+ if (ssl) url = "https://";;
url << host << ':' << port << "/robots.txt";
Document doc(url, 0);
diff -ru htdig-3.1.6/htdig/Server.h htdig-3.1.6_ssl_patched/htdig/Server.h
--- htdig/Server.h Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htdig/Server.h Fri Oct 18 08:27:34 2002
@@ -26,7 +26,7 @@
//
// Construction/Destruction
//
- Server(char *host, int port, StringList *local_robots_files = NULL);
+ Server(char *host, int port, int ssl, StringList *local_robots_file = NULL);
~Server();
//
diff -ru htdig-3.1.6/htlib/Connection.cc htdig-3.1.6_ssl_patched/htlib/Connection.cc
--- htlib/Connection.cc Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htlib/Connection.cc Fri Oct 18 08:27:34 2002
@@ -42,6 +42,10 @@
int rresvport(int *);
}
+SSL_CTX *Connection::ctx = NULL;
+SSL_METHOD *Connection::meth = NULL;
+
+
List all_connections;
Connection::Connection()
@@ -54,8 +58,34 @@
timeout_value = 0;
retry_value = 1;
wait_time = 5; // wait 5 seconds after a failed connection
+ ssl = NULL;
+ m_ssl_on = 0;
}
+void Connection::initSSL()
+{
+ static int init = 1;
+
+ if (init) {
+ init = 0;
+
+ SSL_library_init();
+ SSL_load_error_strings();
+ char buffer[200];
+ RAND_load_file(RAND_file_name(buffer, sizeof buffer), -1);
+ meth = SSLv23_client_method();
+ }
+
+ if (ctx == NULL)
+ {
+ ctx = SSL_CTX_new(meth);
+ if (ctx == NULL)
+ {
+ cout << ssl_error_message() << endl;
+ exit(1);
+ }
+ }
+}
//*************************************************************************
// Connection::Connection(int socket)
@@ -79,6 +109,8 @@
timeout_value = 0;
retry_value = 1;
wait_time = 5;
+ ssl = NULL;
+ m_ssl_on = 0;
}
@@ -91,6 +123,11 @@
this->close();
delete peer;
delete server_name;
+ if ( ctx != NULL )
+ {
+ SSL_CTX_free(ctx);
+ ctx = NULL;
+ }
}
@@ -101,15 +138,15 @@
{
if (priv)
{
- int aport = IPPORT_RESERVED - 1;
+ int aport = IPPORT_RESERVED - 1;
- sock = rresvport(&aport);
+ sock = rresvport(&aport);
}
else
- sock = socket(AF_INET, SOCK_STREAM, 0);
+ sock = socket(AF_INET, SOCK_STREAM, 0);
if (sock == NOTOK)
- return NOTOK;
+ return NOTOK;
int on = 1;
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on));
@@ -164,9 +201,13 @@
connected = 0;
if (sock >= 0)
{
- int ret = ::close(sock);
- sock = -1;
- return ret;
+ if (m_ssl_on)
+ SSL_shutdown(ssl);
+ int ret = ::close(sock);
+ SSL_free(ssl);
+ ssl = NULL;
+ sock = -1;
+ return ret;
}
return NOTOK;
}
@@ -246,14 +287,34 @@
}
//*****************************************************************************
+// int Connection::assign_ssl(int ssl_on)
+//
+int Connection::assign_ssl(int ssl_on)
+{
+ m_ssl_on = ssl_on;
+ return OK;
+}
+
+
+//*****************************************************************************
+// char* Connection::ssl_error_message(void)
+//
+char* Connection::ssl_error_message(void)
+{
+ return ERR_error_string(ERR_get_error(), NULL);
+}
+
+//*****************************************************************************
// int Connection::connect(int allow_EINTR)
//
int Connection::connect(int allow_EINTR)
{
int status;
int retries = retry_value;
+ if (m_ssl_on)
+ initSSL();
- while (retries--)
+ while (retries--)
{
//
// Set an alarm to make sure the connect() call times out
@@ -278,8 +339,24 @@
if (status == 0 || errno == EALREADY || errno == EISCONN)
{
- connected = 1;
- return OK;
+ if (m_ssl_on)
+ {
+ ssl = SSL_new(ctx);
+ if (ssl != NULL)
+ {
+ SSL_set_fd(ssl, sock);
+ if (SSL_connect(ssl) > 0)
+ {
+ connected = 1;
+ return OK;
+ }
+ }
+ }
+ else
+ {
+ connected = 1;
+ return OK;
+ }
}
//
@@ -424,26 +501,32 @@
need_io_stop = 0;
do
{
- errno = 0;
-
- if (timeout_value > 0) {
- fd_set fds;
- FD_ZERO(&fds);
- FD_SET(sock, &fds);
-
- timeval tv;
- tv.tv_sec = timeout_value;
- tv.tv_usec = 0;
-
- int selected = ::select(sock+1, &fds, 0, 0, &tv);
- if (selected <= 0)
- need_io_stop++;
- }
+ errno = 0;
- if (!need_io_stop)
- count = ::read(sock, buffer, maxlength);
- else
- count = -1; // Input timed out
+ if (timeout_value > 0)
+ {
+ fd_set fds;
+ FD_ZERO(&fds);
+ FD_SET(sock, &fds);
+
+ timeval tv;
+ tv.tv_sec = timeout_value;
+ tv.tv_usec = 0;
+
+ int selected = ::select(sock+1, &fds, 0, 0, &tv);
+ if (selected <= 0)
+ need_io_stop++;
+ }
+
+ if (!need_io_stop)
+ {
+ if (ssl != NULL)
+ count = SSL_read(ssl, buffer, maxlength);
+ else
+ count = ::read(sock, buffer, maxlength);
+ }
+ else
+ count = -1; // Input timed out
}
while (count < 0 && errno == EINTR && !need_io_stop);
need_io_stop = 0;
@@ -461,7 +544,10 @@
do
{
- count = ::write(sock, buffer, maxlength);
+ if (ssl != NULL)
+ count = SSL_write(ssl, buffer, maxlength);
+ else
+ count = ::write(sock, buffer, maxlength);
}
while (count < 0 && errno == EINTR && !need_io_stop);
need_io_stop = 0;
diff -ru htdig-3.1.6/htlib/Connection.h htdig-3.1.6_ssl_patched/htlib/Connection.h
--- htlib/Connection.h Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htlib/Connection.h Fri Oct 18 08:27:34 2002
@@ -44,6 +44,14 @@
#include <netinet/in.h>
#include <netdb.h>
+#include <openssl/rsa.h>
+#include <openssl/crypto.h>
+#include <openssl/x509.h>
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <openssl/rand.h>
+
class String;
class Connection : public io
@@ -53,6 +61,7 @@
Connection();
Connection(int socket);
~Connection();
+ void initSSL();
// (De)initialization
int open(int priv = 0);
@@ -73,6 +82,10 @@
int assign_server(unsigned int addr = INADDR_ANY);
char *get_server() {return server_name;}
+ // SSL stuff
+ int assign_ssl(int ssl_on);
+ char *ssl_error_message(void);
+
// Connection establishment
int connect(int allow_EINTR = 0);
Connection *accept(int priv = 0);
@@ -99,6 +112,10 @@
private:
int sock;
+ int m_ssl_on;
+ SSL *ssl;
+ static SSL_CTX *ctx;
+ static SSL_METHOD *meth;
struct sockaddr_in server;
int connected;
char *peer;
diff -ru htdig-3.1.6/htlib/URL.cc htdig-3.1.6_ssl_patched/htlib/URL.cc
--- htlib/URL.cc Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htlib/URL.cc Fri Oct 18 08:27:34 2002
@@ -131,9 +131,15 @@
while (isalpha(*p))
p++;
int hasService = (*p == ':');
- if ((hasService && ((strncmp(ref, "http://";, 7) == 0) ||
- (strncmp(ref, "http:", 5) != 0)))
- || strncmp(ref, "//", 2) == 0)
+
+ if ((hasService &&
+ ( (strncmp(ref, "http://";, 7) == 0) ||
+ (strncmp(ref, "http:", 5) != 0) ||
+ (strncmp(ref, "https://";, 8) == 0) ||
+ (strncmp(ref, "https:", 6) != 0)
+ )
+ )
+ || (strncmp(ref, "//", 2) == 0))
{
//
// No need to look at the parent url since this is a complete url...
@@ -218,7 +224,12 @@
_url << ":";
if (_host.length())
_url << "//" << _host;
- if (_port != 80 && strcmp(_service, "http") == 0)
+
+ // Add on the port if its not one of the standard ones
+ //
+ if ((strcmp(_service, "https") == 0 && _port != 443)||
+ (strcmp(_service, "http" ) == 0 && _port != 80)
+ )
_url << ':' << _port;
_url << _path;
}
@@ -292,13 +303,21 @@
if (p)
_port = atoi(p);
if (!p || _port <= 0)
- _port = 80;
+ {
+ if( strcmp( _service, "https" ) == 0 )
+ _port = 443;
+ else
+ _port = 80;
+ }
}
else
{
_host = strtok(p, "/");
_host.chop(" \t");
- _port = 80;
+ if( strcmp( _service, "https" ) == 0 )
+ _port = 443;
+ else
+ _port = 80;
}
//
@@ -317,7 +336,8 @@
//
_url = _service;
_url << "://" << _host;
- if (_port != 80)
+ if ((strcmp(_service, "https") == 0 && _port != 443)||
+ (strcmp(_service, "http" ) == 0 && _port != 80))
_url << ':' << _port;
_url << _path;
}
@@ -446,7 +466,8 @@
_path.lowercase();
_url = _service;
_url << "://" << _host;
- if (_port != 80)
+ if ((strcmp(_service, "https") == 0 && _port != 443)||
+ (strcmp(_service, "http" ) == 0 && _port != 80))
_url << ':' << _port;
_url << _path;
}
@@ -496,7 +517,8 @@
if (_service.length() == 0 || _normal)
return;
- if (strcmp(_service, "http") != 0)
+ if (strcmp(_service, "http") != 0 &&
+ strcmp(_service, "https") != 0)
return;
removeIndex(_path);
@@ -553,7 +575,8 @@
_url << ":";
if (_host.length())
_url << "//" << _host;
- if (_port != 80 && strcmp(_service, "http") == 0)
+ if ((strcmp(_service, "https") == 0 && _port != 443)||
+ (strcmp(_service, "http" ) == 0 && _port != 80))
_url << ':' << _port;
_url << _path;
_normal = 1;
diff -ru htdig-3.1.6/htlib/URL.h htdig-3.1.6_ssl_patched/htlib/URL.h
--- htlib/URL.h Fri Feb 1 01:47:17 2002
+++ htdig-3.1.6_ssl_patched/htlib/URL.h Fri Oct 18 08:27:34 2002
@@ -63,7 +63,7 @@
void host(char *h) {_host = h;}
int port() {return _port;}
void port(int p) {_port = p;}
- char *service() {return _service;}
+ char *service() {return _service.get();}
void service(char *s) {_service = s;}
char *path() {return _path;}
void path(char *p);
-------------------------------------------------------
This sf.net email is sponsored by:ThinkGeek
Welcome to geek heaven.
http://thinkgeek.com/sf
_______________________________________________
htdig-general mailing list <[EMAIL PROTECTED]>
To unsubscribe, send a message to <[EMAIL PROTECTED]> with a
subject of unsubscribe
FAQ: http://htdig.sourceforge.net/FAQ.html
