Hello community,
here is the log from the commit of package libhtp for openSUSE:Factory checked
in at 2020-09-15 16:28:29
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/libhtp (Old)
and /work/SRC/openSUSE:Factory/.libhtp.new.4249 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "libhtp"
Tue Sep 15 16:28:29 2020 rev:5 rq:834279 version:0.5.34
Changes:
--------
--- /work/SRC/openSUSE:Factory/libhtp/libhtp.changes 2020-05-01
11:12:34.823746607 +0200
+++ /work/SRC/openSUSE:Factory/.libhtp.new.4249/libhtp.changes 2020-09-15
16:28:32.178610927 +0200
@@ -1,0 +2,8 @@
+Sun Sep 13 13:03:31 UTC 2020 - Martin Hauke <mar...@gmx.de>
+
+- Update to version 0.5.34
+ * support data GAP handling
+ * support 100-continue Expect
+ * lzma: give more control over settings
+
+-------------------------------------------------------------------
Old:
----
libhtp-0.5.33.tar.gz
New:
----
libhtp-0.5.34.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ libhtp.spec ++++++
--- /var/tmp/diff_new_pack.GCEVqB/_old 2020-09-15 16:28:32.838611559 +0200
+++ /var/tmp/diff_new_pack.GCEVqB/_new 2020-09-15 16:28:32.842611563 +0200
@@ -19,7 +19,7 @@
%define sover 2
%define lname %{name}%{sover}
Name: libhtp
-Version: 0.5.33
+Version: 0.5.34
Release: 0
Summary: HTTP normalizer and parser
License: BSD-3-Clause
++++++ libhtp-0.5.33.tar.gz -> libhtp-0.5.34.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/.travis.yml
new/libhtp-0.5.34/.travis.yml
--- old/libhtp-0.5.33/.travis.yml 2020-04-27 15:30:04.000000000 +0200
+++ new/libhtp-0.5.34/.travis.yml 2020-09-10 21:47:54.000000000 +0200
@@ -3,7 +3,7 @@
- gcc
- clang
# Change this to your needs
-script: sh autogen.sh && ./configure && make && make check && make distcheck
+script: sh autogen.sh && ./configure && make && (make check || cat
test/test-suite.log) && make distcheck
before_install:
- sudo apt-get update -qq
- sudo apt-get install -y build-essential autoconf automake libtool zlib1g
zlib1g-dev make
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/ChangeLog new/libhtp-0.5.34/ChangeLog
--- old/libhtp-0.5.33/ChangeLog 2020-04-27 15:30:04.000000000 +0200
+++ new/libhtp-0.5.34/ChangeLog 2020-09-10 21:47:54.000000000 +0200
@@ -1,3 +1,12 @@
+0.5.34 (11 September 2020)
+--------------------------
+
+- support data GAP handling
+
+- support 100-continue Expect
+
+- lzma: give more control over settings
+
0.5.33 (27 April 2020)
----------------------
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/VERSION new/libhtp-0.5.34/VERSION
--- old/libhtp-0.5.33/VERSION 2020-04-27 15:30:04.000000000 +0200
+++ new/libhtp-0.5.34/VERSION 2020-09-10 21:47:54.000000000 +0200
@@ -1,2 +1,2 @@
# This file is intended to be sourced by sh
-PKG_VERSION=0.5.33
+PKG_VERSION=0.5.34
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_config.c
new/libhtp-0.5.34/htp/htp_config.c
--- old/libhtp-0.5.33/htp/htp_config.c 2020-04-27 15:30:04.000000000 +0200
+++ new/libhtp-0.5.34/htp/htp_config.c 2020-09-10 21:47:54.000000000 +0200
@@ -159,6 +159,7 @@
cfg->extract_request_files_limit = -1; // Use the parser default.
cfg->response_decompression_layer_limit = 2; // 2 layers seem fairly common
cfg->lzma_memlimit = HTP_LZMA_MEMLIMIT;
+ cfg->response_lzma_layer_limit = 1; // default is only one layer
cfg->compression_bomb_limit = HTP_COMPRESSION_BOMB_LIMIT;
cfg->compression_time_limit = HTP_COMPRESSION_TIME_LIMIT_USEC;
@@ -515,6 +516,11 @@
cfg->lzma_memlimit = memlimit;
}
+void htp_config_set_lzma_layers(htp_cfg_t *cfg, int limit) {
+ if (cfg == NULL) return;
+ cfg->response_lzma_layer_limit = limit;
+}
+
void htp_config_set_compression_bomb_limit(htp_cfg_t *cfg, size_t bomblimit) {
if (cfg == NULL) return;
if (bomblimit > INT32_MAX) {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_config.h
new/libhtp-0.5.34/htp/htp_config.h
--- old/libhtp-0.5.33/htp/htp_config.h 2020-04-27 15:30:04.000000000 +0200
+++ new/libhtp-0.5.34/htp/htp_config.h 2020-09-10 21:47:54.000000000 +0200
@@ -435,6 +435,14 @@
void htp_config_set_lzma_memlimit(htp_cfg_t *cfg, size_t memlimit);
/**
+ * Configures the maximum layers LibHTP will pass to liblzma.
+ *
+ * @param[in] cfg
+ * @param[in] limit
+ */
+void htp_config_set_lzma_layers(htp_cfg_t *cfg, int limit);
+
+/**
* Configures the maximum compression bomb size LibHTP will decompress.
*
* @param[in] cfg
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_config_private.h
new/libhtp-0.5.34/htp/htp_config_private.h
--- old/libhtp-0.5.33/htp/htp_config_private.h 2020-04-27 15:30:04.000000000
+0200
+++ new/libhtp-0.5.34/htp/htp_config_private.h 2020-09-10 21:47:54.000000000
+0200
@@ -351,6 +351,9 @@
/** max time for a decompression bomb. */
int32_t compression_time_limit;
+
+ /** How many layers of compression we will decompress (0 => no lzma). */
+ int response_lzma_layer_limit;
};
#ifdef __cplusplus
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_decompressors.c
new/libhtp-0.5.34/htp/htp_decompressors.c
--- old/libhtp-0.5.33/htp/htp_decompressors.c 2020-04-27 15:30:04.000000000
+0200
+++ new/libhtp-0.5.34/htp/htp_decompressors.c 2020-09-10 21:47:54.000000000
+0200
@@ -439,7 +439,8 @@
switch (format) {
case HTP_COMPRESSION_LZMA:
- if (connp->cfg->lzma_memlimit > 0) {
+ if (connp->cfg->lzma_memlimit > 0 &&
+ connp->cfg->response_lzma_layer_limit > 0) {
LzmaDec_Construct(&drec->state);
} else {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "LZMA
decompression disabled");
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_request.c
new/libhtp-0.5.34/htp/htp_request.c
--- old/libhtp-0.5.33/htp/htp_request.c 2020-04-27 15:30:04.000000000 +0200
+++ new/libhtp-0.5.34/htp/htp_request.c 2020-09-10 21:47:54.000000000 +0200
@@ -775,6 +775,10 @@
#ifdef HTP_DEBUG
fprint_raw_data(stderr, __func__, data, len);
#endif
+ if (len == 0) {
+ htp_connp_req_clear_buffer(connp);
+ return HTP_DATA;
+ }
// Is this a line that should be ignored?
if (htp_connp_is_line_ignorable(connp, data, len)) {
@@ -886,12 +890,18 @@
bstr_free(method);
}
if (methodi == HTP_M_UNKNOWN) {
+ if (connp->in_body_data_left <= 0) {
+ // log only once per transaction
+ htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected
request body");
+ } else {
+ connp->in_body_data_left = 1;
+ }
// Interpret remaining bytes as body data
- htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected
request body");
htp_status_t rc = htp_tx_req_process_body_data_ex(connp->in_tx,
data, len);
htp_connp_req_clear_buffer(connp);
return rc;
} // else continue
+ connp->in_body_data_left = -1;
}
//unread last end of line so that REQ_LINE works
if (connp->in_current_read_offset < (int64_t)len) {
@@ -992,7 +1002,7 @@
// only if the stream has been closed. We do not allow zero-sized
// chunks in the API, but we use them internally to force the parsers
// to finalize parsing.
- if (((data == NULL) || (len == 0)) && (connp->in_status !=
HTP_STREAM_CLOSED)) {
+ if (len == 0 && connp->in_status != HTP_STREAM_CLOSED) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Zero-length data
chunks are not allowed");
#ifdef HTP_DEBUG
@@ -1045,7 +1055,25 @@
// Return if there's been an error or if we've run out of data. We are
relying
// on processors to supply error messages, so we'll keep quiet here.
- htp_status_t rc = connp->in_state(connp);
+
+ htp_status_t rc;
+ //handle gap
+ if (data == NULL && len > 0) {
+ //cannot switch over a function pointer in C
+ if (connp->in_state == htp_connp_REQ_BODY_IDENTITY ||
+ connp->in_state == htp_connp_REQ_IGNORE_DATA_AFTER_HTTP_0_9) {
+ rc = connp->in_state(connp);
+ } else if (connp->in_state == htp_connp_REQ_FINALIZE) {
+ //simple version without probing
+ rc = htp_tx_state_request_complete(connp->in_tx);
+ } else {
+ // go to htp_connp_REQ_CONNECT_PROBE_DATA ?
+ htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Gaps are not
allowed during this state");
+ return HTP_STREAM_CLOSED;
+ }
+ } else {
+ rc = connp->in_state(connp);
+ }
if (rc == HTP_OK) {
if (connp->in_status == HTP_STREAM_TUNNEL) {
#ifdef HTP_DEBUG
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_response.c
new/libhtp-0.5.34/htp/htp_response.c
--- old/libhtp-0.5.33/htp/htp_response.c 2020-04-27 15:30:04.000000000
+0200
+++ new/libhtp-0.5.34/htp/htp_response.c 2020-09-10 21:47:54.000000000
+0200
@@ -608,6 +608,19 @@
return HTP_OK;
}
+ // A request can indicate it waits for headers validation
+ // before sending its body cf
+ // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect
+ if (connp->out_tx->response_status_number >= 400 &&
+ connp->out_tx->response_status_number <= 499 &&
+ connp->in_content_length > 0 &&
+ connp->in_body_data_left == connp->in_content_length) {
+ htp_header_t *exp = htp_table_get_c(connp->out_tx->request_headers,
"expect");
+ if ((exp != NULL) && (bstr_cmp_c_nocase(exp->value, "100-continue") ==
0)) {
+ connp->in_state = htp_connp_REQ_FINALIZE;
+ }
+ }
+
// 1. Any response message which MUST NOT include a message-body
// (such as the 1xx, 204, and 304 responses and any response to a HEAD
// request) is always terminated by the first empty line after the
@@ -1228,7 +1241,7 @@
// only if the stream has been closed. We do not allow zero-sized
// chunks in the API, but we use it internally to force the parsers
// to finalize parsing.
- if (((data == NULL) || (len == 0)) && (connp->out_status !=
HTP_STREAM_CLOSED)) {
+ if (len == 0 && connp->out_status != HTP_STREAM_CLOSED) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Zero-length data
chunks are not allowed");
#ifdef HTP_DEBUG
@@ -1277,7 +1290,22 @@
// or if we've run out of data. We are relying
// on processors to add error messages, so we'll
// keep quiet here.
- htp_status_t rc = connp->out_state(connp);
+ htp_status_t rc;
+
+ //handle gap
+ if (data == NULL && len > 0) {
+ if (connp->out_state == htp_connp_RES_BODY_IDENTITY_CL_KNOWN ||
+ connp->out_state == htp_connp_RES_BODY_IDENTITY_STREAM_CLOSE) {
+ rc = connp->out_state(connp);
+ } else if (connp->out_state == htp_connp_RES_FINALIZE) {
+ rc = htp_tx_state_response_complete_ex(connp->out_tx, 0);
+ } else {
+ htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Gaps are not
allowed during this state");
+ return HTP_STREAM_CLOSED;
+ }
+ } else {
+ rc = connp->out_state(connp);
+ }
if (rc == HTP_OK) {
if (connp->out_status == HTP_STREAM_TUNNEL) {
#ifdef HTP_DEBUG
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_transaction.c
new/libhtp-0.5.34/htp/htp_transaction.c
--- old/libhtp-0.5.33/htp/htp_transaction.c 2020-04-27 15:30:04.000000000
+0200
+++ new/libhtp-0.5.34/htp/htp_transaction.c 2020-09-10 21:47:54.000000000
+0200
@@ -1280,6 +1280,7 @@
} else {
int layers = 0;
htp_decompressor_t *comp = NULL;
+ int nblzma = 0;
uint8_t *tok = NULL;
size_t tok_len = 0;
@@ -1326,6 +1327,12 @@
cetype = HTP_COMPRESSION_DEFLATE;
} else if (bstr_util_cmp_mem(tok, tok_len, "lzma", 4) == 0) {
cetype = HTP_COMPRESSION_LZMA;
+ nblzma++;
+ if (nblzma > tx->connp->cfg->response_lzma_layer_limit) {
+ htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
+ "Compression bomb: double lzma encoding");
+ break;
+ }
} else if (bstr_util_cmp_mem(tok, tok_len, "inflate", 7) == 0)
{
cetype = HTP_COMPRESSION_NONE;
} else {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/htp/htp_util.c
new/libhtp-0.5.34/htp/htp_util.c
--- old/libhtp-0.5.33/htp/htp_util.c 2020-04-27 15:30:04.000000000 +0200
+++ new/libhtp-0.5.34/htp/htp_util.c 2020-09-10 21:47:54.000000000 +0200
@@ -1975,7 +1975,12 @@
*
*/
void fprint_raw_data(FILE *stream, const char *name, const void *data, size_t
len) {
- fprint_raw_data_ex(stream, name, data, 0, len);
+ // may happen for gaps
+ if (data == NULL && len > 0) {
+ fprintf(stream, "\n%s: ptr NULL len %u\n", name, (unsigned int)len);
+ } else {
+ fprint_raw_data_ex(stream, name, data, 0, len);
+ }
}
/**
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/htp.pc.in new/libhtp-0.5.34/htp.pc.in
--- old/libhtp-0.5.33/htp.pc.in 2020-04-27 15:30:04.000000000 +0200
+++ new/libhtp-0.5.34/htp.pc.in 2020-09-10 21:47:54.000000000 +0200
@@ -7,6 +7,6 @@
Description: A security-aware HTTP parser, designed for use in IDS/IPS and WAF
products.
Version: @PACKAGE_VERSION@
Libs: -L${libdir} -lhtp
-Libs.private: @LIBICONV@
+Libs.private: -lz @LIBICONV@
Cflags: -I${includedir} -I${libdir}/htp/include
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/test/files/99-expect-100.t
new/libhtp-0.5.34/test/files/99-expect-100.t
--- old/libhtp-0.5.33/test/files/99-expect-100.t 1970-01-01
01:00:00.000000000 +0100
+++ new/libhtp-0.5.34/test/files/99-expect-100.t 2020-09-10
21:47:54.000000000 +0200
@@ -0,0 +1,26 @@
+>>>
+PUT /forbidden HTTP/1.1
+Content-Length: 14
+Expect: 100-continue
+
+
+<<<
+HTTP/1.0 401 Forbidden
+Content-Length: 0
+
+
+>>>
+POST /ok HTTP/1.1
+Content-Length: 14
+Expect: 100-continue
+
+
+<<<
+HTTP/1.0 100 continue
+
+
+>>>
+Hello People!
+
+<<<
+HTTP/1.0 200 OK
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/libhtp-0.5.33/test/test_main.cpp
new/libhtp-0.5.34/test/test_main.cpp
--- old/libhtp-0.5.33/test/test_main.cpp 2020-04-27 15:30:04.000000000
+0200
+++ new/libhtp-0.5.34/test/test_main.cpp 2020-09-10 21:47:54.000000000
+0200
@@ -2068,3 +2068,23 @@
ASSERT_EQ(200, tx->response_status_number);
ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress);
}
+
+TEST_F(ConnectionParsing, Expect100) {
+ int rc = test_run(home, "99-expect-100.t", cfg, &connp);
+ ASSERT_GE(rc, 0);
+
+ ASSERT_EQ(2, htp_list_size(connp->conn->transactions));
+ htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0);
+ ASSERT_TRUE(tx != NULL);
+ ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "PUT"));
+ ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress);
+ ASSERT_EQ(401, tx->response_status_number);
+ ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress);
+
+ tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 1);
+ ASSERT_TRUE(tx != NULL);
+ ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "POST"));
+ ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress);
+ ASSERT_EQ(200, tx->response_status_number);
+ ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress);
+}
