Integration of libhtp-0.2.3 rev 199

libhtp/Makefile.am

@@ -1,6 +1,8 @@
+
 SUBDIRS= $(GENERIC_LIBRARY_NAME) test
 
 EXTRA_DIST = ChangeLog COPYING LICENSE LIBHTP_LICENSING_EXCEPTION docs/doxygen.conf docs/QUICK_START
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = htp.pc
+

libhtp/docs/QUICK_START

@@ -99,4 +99,3 @@ options:
   2. Associate one opaque structure with a transaction instance, using htp_tx_set_user_data().
      The best place to do this is in a TRANSACTION_START callback. Don't forget to free up
      any resources you allocate on per-transaction basis, before you delete each transaction.
-

libhtp/htp.pc.in

@@ -5,7 +5,7 @@ includedir=${prefix}/include
 
 Name: HTP
 Description: HTTP parser
-Version: 0.2.X
+Version: 0.2.3
 Libs: -L${libdir} -lhtp
 Cflags: -I${includedir}/htp -I${libdir}/htp/include
 

libhtp/htp/htp.h

@@ -41,7 +41,7 @@ typedef struct htp_urldecoder_t htp_urldecoder_t;
 
 // -- Defines -------------------------------------------------------------------------------------
 
-#define HTP_BASE_VERSION_TEXT	"0.2.x"
+#define HTP_BASE_VERSION_TEXT	"0.2.3"
 
 #define HTP_ERROR              -1
 #define HTP_OK                  0
@@ -825,6 +825,17 @@ struct htp_tx_t {
     /** Parsed request headers. */
     table_t *request_headers;
 
+    /** Contains raw request headers. This field is generated on demand, use
+     *  htp_tx_get_request_headers_raw() to get it.
+     */
+    bstr *request_headers_raw;
+
+    /** How many request header lines have been included in the raw
+     *  buffer (above).
+     */
+    size_t request_headers_raw_lines;
+
+
     /** Request transfer coding: IDENTITY or CHUNKED. Only available on requests that have bodies. */
     int request_transfer_coding;
 
@@ -1113,6 +1124,8 @@ char *htp_tx_progress_as_string(htp_tx_t *tx);
 
 bstr *htp_unparse_uri_noencode(htp_uri_t *uri);
 
+bstr *htp_tx_get_request_headers_raw(htp_tx_t *tx);
+
 #endif	/* _HTP_H */
 
 

libhtp/htp/htp_request.c

@@ -431,8 +431,6 @@ int htp_connp_REQ_HEADERS(htp_connp_t *connp) {
                 free(connp->in_header_line);
                 connp->in_line_len = 0;
                 connp->in_header_line = NULL;
-                connp->in_header_line_index = -1;
-                connp->in_header_line_counter = 0;
 
                 // We've seen all request headers
                 if (connp->in_chunk_count != connp->in_chunk_request_index) {
@@ -462,7 +460,7 @@ int htp_connp_REQ_HEADERS(htp_connp_t *connp) {
             }
 
             // Prepare line for consumption
-            htp_chomp(connp->in_line, &connp->in_line_len);
+            int chomp_result = htp_chomp(connp->in_line, &connp->in_line_len);
 
             // Check for header folding
             if (htp_connp_is_line_folded(connp->in_line, connp->in_line_len) == 0) {
@@ -493,7 +491,7 @@ int htp_connp_REQ_HEADERS(htp_connp_t *connp) {
             }
 
             // Add the raw header line to the list
-            connp->in_header_line->line = bstr_memdup((char *) connp->in_line, connp->in_line_len);
+            connp->in_header_line->line = bstr_memdup((char *) connp->in_line, connp->in_line_len + chomp_result);
             list_add(connp->in_tx->request_header_lines, connp->in_header_line);
             connp->in_header_line = NULL;
 

libhtp/htp/htp_request_apache_2_2.c

@@ -65,14 +65,17 @@ int htp_process_request_header_apache_2_2(htp_connp_t *connp) {
 
         for (i = connp->in_header_line_index; i < connp->in_header_line_counter; i++) {
             htp_header_line_t *hl = list_get(connp->in_tx->request_header_lines, i);
-            bstr_add_str_noex(tempstr, hl->line);
+            char *data = bstr_ptr(hl->line);
+            size_t len = bstr_len(hl->line);
+            htp_chomp((unsigned char *)data, &len);
+            bstr_add_mem_noex(tempstr, data, len);
             hl->header = h;
         }
 
         data = (unsigned char *) bstr_ptr(tempstr);
     }
 
-    // Now try to oparse the header
+    // Now try to parse the header
     if (htp_parse_request_header_apache_2_2(connp, h, data, len) != HTP_OK) {
         // Note: downstream responsible for error logging
         if (tempstr != NULL) {
@@ -126,6 +129,8 @@ int htp_parse_request_header_apache_2_2(htp_connp_t *connp, htp_header_t *h, uns
     size_t name_start, name_end;
     size_t value_start, value_end;
 
+    htp_chomp(data, &len);
+
     name_start = 0;
 
     // Look for the colon

libhtp/htp/htp_request_generic.c

@@ -8,7 +8,7 @@
  *
  * In addition, there is a special exception that allows LibHTP to be freely
  * used with any OSI-approved open source licence. Please refer to the file
- * LIBHTP_LICENSING_EXCEPTION for the full text of the exception.
+ * LIBHTP_LICENSING_EXCEPTION fo the full text of the exception.
  *
  */
 
@@ -66,7 +66,10 @@ int htp_process_request_header_generic(htp_connp_t *connp) {
 
         for (i = connp->in_header_line_index; i < connp->in_header_line_counter; i++) {
             htp_header_line_t *hl = list_get(connp->in_tx->request_header_lines, i);
-            bstr_add_str_noex(tempstr, hl->line);
+            char *data = bstr_ptr(hl->line);
+            size_t len = bstr_len(hl->line);
+            htp_chomp((unsigned char *)data, &len);
+            bstr_add_mem_noex(tempstr, data, len);
             hl->header = h;
         }
 
@@ -128,6 +131,8 @@ int htp_parse_request_header_generic(htp_connp_t *connp, htp_header_t *h, unsign
     size_t name_start, name_end;
     size_t value_start, value_end;
 
+    htp_chomp(data, &len);
+
     name_start = 0;
 
     // Look for the colon

libhtp/htp/htp_response.c

@@ -464,8 +464,6 @@ int htp_connp_RES_HEADERS(htp_connp_t *connp) {
                 free(connp->out_header_line);
                 connp->out_line_len = 0;
                 connp->out_header_line = NULL;
-                connp->out_header_line_index = -1;
-                connp->out_header_line_counter = 0;
 
                 // We've seen all response headers
                 if (connp->out_tx->progress == TX_PROGRESS_RES_HEADERS) {

libhtp/htp/htp_transaction.c

@@ -107,6 +107,10 @@ void htp_tx_destroy(htp_tx_t *tx) {
 
     table_destroy(tx->request_headers);
 
+    if (tx->request_headers_raw != NULL) {
+        bstr_free(tx->request_headers_raw);
+    }
+
     bstr_free(tx->response_line);
     bstr_free(tx->response_protocol);
     bstr_free(tx->response_status);

libhtp/htp/htp_util.c

@@ -1819,3 +1819,65 @@ bstr *htp_unparse_uri_noencode(htp_uri_t *uri) {
 
     return r;
 }
+
+/**
+ * Construct a bstr that contains the raw request headers.
+ *
+ * @param tx
+ * @return
+ */
+bstr *htp_tx_generate_request_headers_raw(htp_tx_t *tx) {
+    bstr *request_headers_raw = NULL;
+    size_t i, len = 0;
+
+    for (i = 0; i < list_size(tx->request_header_lines); i++) {
+        htp_header_line_t *hl = list_get(tx->request_header_lines, i);
+        len += bstr_len(hl->line);
+    }
+
+    request_headers_raw = bstr_alloc(len);
+    if (request_headers_raw == NULL) {
+        htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Failed to allocate bstring of %d bytes", len);
+        return NULL;
+    }
+
+    for (i = 0; i < list_size(tx->request_header_lines); i++) {
+        htp_header_line_t *hl = list_get(tx->request_header_lines, i);
+        bstr_add_str_noex(request_headers_raw, hl->line);
+    }
+
+    return request_headers_raw;
+}
+
+/**
+ * Get a bstr that contains the raw request headers. This method will always
+ * return an up-to-date buffer, containing the last known headers. Thus, if
+ * it is called once after REQUEST_HEADERS phase it will return one buffer, but
+ * it may return a different buffer if called after REQUEST_TRAILERS phase (but
+ * only if the request actually contains trailer headers). Do not retain the
+ * bstr pointer, as the buffer may change. If there are no changes to the
+ * request header structure, only one buffer will be contstructed and used. (Multiple
+ * invocations of this method will not cause multiple buffers to be created.)
+ *
+ * @param tx
+ * @return
+ */
+bstr *htp_tx_get_request_headers_raw(htp_tx_t *tx) {
+    // Check that we are not called too early
+    if (tx->progress < TX_PROGRESS_REQ_HEADERS) return NULL;
+
+    if (tx->request_headers_raw == NULL) {
+        tx->request_headers_raw = htp_tx_generate_request_headers_raw(tx);
+        tx->request_headers_raw_lines = list_size(tx->request_header_lines);
+    } else {
+        // Check that the buffer we have is not obsolete
+        if (tx->request_headers_raw_lines < list_size(tx->request_header_lines)) {
+            // Rebuild raw buffer
+            bstr_free(tx->request_headers_raw);
+            tx->request_headers_raw = htp_tx_generate_request_headers_raw(tx);
+            tx->request_headers_raw_lines = list_size(tx->request_header_lines);
+        }
+    }
+
+    return tx->request_headers_raw;
+}

libhtp/test/Makefile.am

@@ -8,5 +8,3 @@ AM_CFLAGS = -g -O2
 
 #check: all
 #	./main
-
-

libhtp/test/files/03-post-urlencoded.t

@@ -31,4 +31,3 @@ Transfer-Encoding: chunked
 1
 9
 0
-

libhtp/test/files/16-connect-extra.t

@@ -29,4 +29,3 @@ Location: https://www.feistyduck.com/
 Vary: Accept-Encoding
 Connection: close
 Content-Type: text/html; charset=iso-8859-1
-

libhtp/test/main.c

@@ -67,6 +67,9 @@ int test_post_urlencoded_chunked(htp_cfg_t *cfg) {
         free(key);
     }
 
+    bstr *raw = htp_tx_get_request_headers_raw(tx);
+    fprint_raw_data(stdout, "REQUEST HEADERS RAW 2", bstr_ptr(raw), bstr_len(raw));
+
     htp_connp_destroy_all(connp);
 
     return 1;
@@ -836,6 +839,7 @@ int main(int argc, char** argv) {
     RUN_TEST(test_connect_extra, cfg);
 
     //RUN_TEST(test_misc, cfg);
+    //RUN_TEST(test_post_urlencoded_chunked, cfg);
 
     printf("Tests: %i\n", tests);
     printf("Failures: %i\n", failures);