From 5a9a23f9bb50313f03cbe4a9521d10622d339525 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 4 Aug 2009 15:23:18 +0200 Subject: [PATCH] Update to the parsers. --- src/app-layer-http.c | 68 ++++++++++++++++++++-- src/app-layer-parser.c | 128 +++++++++++++++++++++++++++++++++++++++-- src/app-layer-parser.h | 13 ++++- src/eidps.c | 1 + 4 files changed, 200 insertions(+), 10 deletions(-) diff --git a/src/app-layer-http.c b/src/app-layer-http.c index f0a82befd5..bfb655e003 100644 --- a/src/app-layer-http.c +++ b/src/app-layer-http.c @@ -29,6 +29,10 @@ enum { HTTP_FIELD_MAX, }; +typedef struct HttpState_ { + +} HttpState; + /** \brief Mapping between HTTP_FIELD_* and AppLayerParsers * * Map the http fields identifiers to the parsers. @@ -53,16 +57,17 @@ int HTTPParseRequest(void *http_state, void *parser_state, u_int8_t *input, u_in http_state, parser_state, input, input_len); char done = FALSE; + u_int32_t offset = 0; AppLayerParserState *pstate = (AppLayerParserState *)parser_state; printf("HTTPParseRequest: pstate->buflen %u\n", pstate->buflen); u_int32_t u32 = 0; - for ( ; u32 < input_len; u32++) { + for ( ; u32 < input_len && pstate->buflen < sizeof(pstate->buf); u32++) { pstate->buf[pstate->buflen] = input[u32]; pstate->buflen++; - if (pstate->buflen > 1 && pstate->buf[pstate->buflen - 2] == '\r' && pstate->buf[pstate->buflen - 1] == '\n') { + if (pstate->buflen >= 2 && pstate->buf[pstate->buflen - 2] == '\r' && pstate->buf[pstate->buflen - 1] == '\n') { printf("HTTPParseRequest: request line done.\n"); done = TRUE; break; @@ -70,11 +75,64 @@ int HTTPParseRequest(void *http_state, void *parser_state, u_int8_t *input, u_in } if (done == TRUE) { + AppLayerParserResultElement *e = AppLayerGetResultElmt(); + if (e == NULL) + return -1; + + e->name_idx = HTTP_FIELD_REQUEST_LINE; + e->data_ptr = input; + e->data_len = pstate->buflen; + output[*output_num] = e; + + (*output_num)++; + printf("HTTPParseRequest: request line:\n"); - PrintRawDataFp(stdout, pstate->buf,pstate->buflen); - pstate->flags |= APP_LAYER_PARSER_DONE; + PrintRawDataFp(stdout, e->data_ptr,e->data_len); + + offset += pstate->buflen; + pstate->buflen = 0; + done = FALSE; + } else { + /* bail with state update */ + return 0; + } + + printf("HTTPParseRequest: u32 %u, pstate->buflen %u\n", u32, pstate->buflen); + for ( ; u32 < input_len && pstate->buflen < sizeof(pstate->buf); u32++) { + pstate->buf[pstate->buflen] = input[u32]; + pstate->buflen++; + + + if (pstate->buflen > 3 && + pstate->buf[pstate->buflen - 4] == '\r' && pstate->buf[pstate->buflen - 3] == '\n' && + pstate->buf[pstate->buflen - 2] == '\r' && pstate->buf[pstate->buflen - 1] == '\n') { + printf("HTTPParseRequest: request headers done @ u32 %u, pstate->buflen %u\n", u32, pstate->buflen); + done = TRUE; + break; + } + } + + if (done == TRUE) { + AppLayerParserResultElement *e = AppLayerGetResultElmt(); + if (e == NULL) + return -1; + + e->name_idx = HTTP_FIELD_REQUEST_HEADERS; + e->data_ptr = input + offset; + e->data_len = pstate->buflen; + output[*output_num] = e; + + (*output_num)++; + + printf("HTTPParseRequest: request headers:\n"); + PrintRawDataFp(stdout, e->data_ptr,e->data_len); + offset += pstate->buflen; pstate->buflen = 0; + done = FALSE; + } else { + /* bail with state update */ + return 0; } return 1; } @@ -88,6 +146,6 @@ void RegisterHTTPParsers(void) { AppLayerRegisterProto("http", ALPROTO_HTTP, STREAM_TOSERVER, HTTPParseRequest); AppLayerRegisterProto("http", ALPROTO_HTTP, STREAM_TOCLIENT, HTTPParseResponse); - AppLayerRegisterParser("http.request_line", HTTPParseRequestLine, "http"); + AppLayerRegisterParser("http.request_line", ALPROTO_HTTP, HTTP_FIELD_REQUEST_LINE, HTTPParseRequestLine, "http"); } diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 8130f338d8..7a8760d8d0 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -14,6 +14,24 @@ #include "app-layer-protos.h" #include "app-layer-parser.h" +static Pool *al_result_pool = NULL; + +void* AppLayerParserResultElementAlloc(void *null) { + AppLayerParserResultElement *e = (AppLayerParserResultElement *)malloc(sizeof(AppLayerParserResultElement)); + if (e == NULL) { + return NULL; + } + + memset(e, 0, sizeof(AppLayerParserResultElement)); + return e; +} +#define AppLayerParserResultElementFree free + +AppLayerParserResultElement *AppLayerGetResultElmt(void) { + AppLayerParserResultElement *e = (AppLayerParserResultElement *)PoolGet(al_result_pool); + return e; +} + static u_int16_t app_layer_sid = 0; static AppLayerProto al_proto_table[ALPROTO_MAX]; @@ -39,12 +57,17 @@ u_int16_t AppLayerParserGetStorageId(void) { * \retval 0 on success * \retval -1 on error */ -int AppLayerRegisterParser(char *name, int (*AppLayerParser)(void *protocol_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num), char *dependency) { +int AppLayerRegisterParser(char *name, u_int16_t proto, u_int16_t parser_id, int (*AppLayerParser)(void *protocol_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num), char *dependency) { al_max_parsers++; al_parser_table[al_max_parsers].name = name; + al_parser_table[al_max_parsers].proto = proto; + al_parser_table[al_max_parsers].parser_local_id = parser_id; al_parser_table[al_max_parsers].AppLayerParser = AppLayerParser; + + printf("AppLayerRegisterParser: registered %p at proto %u, al_proto_table idx %u, storage_id %u, parser_local_id %u\n", + AppLayerParser, proto, al_max_parsers, al_proto_table[proto].storage_id, parser_id); return 0; } @@ -117,12 +140,12 @@ int AppLayerParse(Flow *f, u_int8_t proto, u_int8_t flags, u_int8_t *input, u_in parser_idx = p->to_client; } } else { - printf("L7Parse: using parser %u we stored before\n", parser_state->cur_parser); + printf("AppLayerParse: using parser %u we stored before\n", parser_state->cur_parser); parser_idx = parser_state->cur_parser; } if (parser_idx == 0) { - printf("L7Parse: no parser for protocol %u\n", proto); + printf("AppLayerParse: no parser for protocol %u\n", proto); return 0; } @@ -135,10 +158,31 @@ int AppLayerParse(Flow *f, u_int8_t proto, u_int8_t flags, u_int8_t *input, u_in } } - int r = al_parser_table[parser_idx].AppLayerParser(app_layer_state, parser_state, input, input_len, NULL, NULL); + AppLayerParserResultElement *result_tbl[256]; + memset(&result_tbl,0,sizeof(result_tbl)); + u_int16_t output_num = 0; + int r = al_parser_table[parser_idx].AppLayerParser(app_layer_state, parser_state, input, input_len, result_tbl, &output_num); if (r < 0) return -1; + printf("AppLayerParse: output_num %u\n", output_num); + u_int16_t u = 0; + for (u = 0; u < output_num; u++) { + AppLayerParserResultElement *e = result_tbl[u]; + printf("AppLayerParse: e->name_idx %u, e->data_ptr %p, e->data_len %u, map_size %u\n", e->name_idx, e->data_ptr, e->data_len, al_proto_table[proto].map_size); + + /* no parser defined for this field. */ + if (e->name_idx >= al_proto_table[proto].map_size || al_proto_table[proto].map[e->name_idx] == NULL) { + printf("AppLayerParse: no parser for proto %u, parser_local_id %u\n", proto, e->name_idx); + continue; + } + + parser_idx = al_proto_table[proto].map[e->name_idx]->parser_id; + int r = al_parser_table[parser_idx].AppLayerParser(app_layer_state, parser_state, e->data_ptr, e->data_len, result_tbl, &output_num); + if (r < 0) + return -1; + } + return 0; } @@ -148,5 +192,81 @@ void RegisterAppLayerParsers(void) { memset(&al_parser_table, 0, sizeof(al_parser_table)); app_layer_sid = StreamL7RegisterModule(); + + /** setup result pool + * \todo Per thread pool */ + al_result_pool = PoolInit(100,10,AppLayerParserResultElementAlloc,NULL,AppLayerParserResultElementFree); +} + +void AppLayerParsersInitPostProcess(void) { + printf("AppLayerParsersInitPostProcess: start\n"); + u_int16_t u16 = 0; + + /* build local->global mapping */ + for (u16 = 1; u16 <= al_max_parsers; u16++) { + /* no local parser */ + if (al_parser_table[u16].parser_local_id == 0) + continue; + + if (al_parser_table[u16].parser_local_id > al_proto_table[al_parser_table[u16].proto].map_size) + al_proto_table[al_parser_table[u16].proto].map_size = al_parser_table[u16].parser_local_id; + + printf("AppLayerParsersInitPostProcess: map_size %u\n", al_proto_table[al_parser_table[u16].proto].map_size); + } + + /* for each proto, alloc the map array */ + for (u16 = 0; u16 < ALPROTO_MAX; u16++) { + if (al_proto_table[u16].map_size == 0) + continue; + + al_proto_table[u16].map_size++; + al_proto_table[u16].map = (AppLayerLocalMap **)malloc(al_proto_table[u16].map_size * sizeof(AppLayerLocalMap *)); + if (al_proto_table[u16].map == NULL) { + printf("XXX memory error\n"); + exit(1); + } + memset(al_proto_table[u16].map, 0, al_proto_table[u16].map_size * sizeof(AppLayerLocalMap *)); + + u_int16_t u = 0; + u_int16_t x = 0; + for (u = 1; u <= al_max_parsers; u++) { + /* no local parser */ + if (al_parser_table[u].parser_local_id == 0) + continue; + + if (al_parser_table[u].proto != u16) + continue; + + printf("al_proto_table[%u].map_size %u, x %u, %p %p\n", u16, al_proto_table[u16].map_size, x, al_proto_table[u16].map[x], al_proto_table[u16].map); + u_int16_t parser_local_id = al_parser_table[u].parser_local_id; + printf("parser_local_id: %u\n", parser_local_id); + + if (parser_local_id < al_proto_table[u16].map_size) { + al_proto_table[u16].map[parser_local_id] = malloc(sizeof(AppLayerLocalMap)); + if (al_proto_table[u16].map[parser_local_id] == NULL) { + printf("XXX memory error\n"); + exit(1); + } + + al_proto_table[u16].map[parser_local_id]->parser_id = u; + } + } + } + + for (u16 = 0; u16 < ALPROTO_MAX; u16++) { + if (al_proto_table[u16].map_size == 0) + continue; + + if (al_proto_table[u16].map == NULL) + continue; + + u_int16_t x = 0; + for (x = 0; x < al_proto_table[u16].map_size; x++) { + if (al_proto_table[u16].map[x] == NULL) + continue; + + printf("al_proto_table[%u].map[%u]->parser_id: %u\n", u16, x, al_proto_table[u16].map[x]->parser_id); + } + } } diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index 2864b32484..1744dbb185 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -1,6 +1,12 @@ #ifndef __APP_LAYER_PARSER_H__ #define __APP_LAYER_PARSER_H__ +/** Mapping between local parser id's (e.g. HTTP_FIELD_REQUEST_URI) and + * the dynamically assigned (at registration) global parser id. */ +typedef struct AppLayerLocalMap_ { + u_int16_t parser_id; +} AppLayerLocalMap; + /** \brief Mapping between ALPROTO_* and L7Parsers * * Map the proto to the parsers for the to_client and to_server directions. @@ -9,6 +15,9 @@ typedef struct AppLayerProto_ { u_int16_t to_server; u_int16_t to_client; u_int8_t storage_id; + + AppLayerLocalMap **map; + u_int16_t map_size; } AppLayerProto; typedef struct AppLayerParserResultElement_ { @@ -22,6 +31,8 @@ typedef struct AppLayerParserResultElement_ { typedef struct AppLayerParserTableElement_ { char *name; + u_int16_t proto; + u_int16_t parser_local_id; /** local id of the parser in the parser itself. */ u_int8_t flags; int (*AppLayerParser)(void *protocol_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num); u_int16_t max_outputs; /* rationele is that if we know the max outputs of all parsers, we @@ -38,7 +49,7 @@ typedef struct AppLayerParserState_ { /** \todo this needs to become dynamic */ u_int8_t buf[1024]; - u_int8_t buflen; + u_int32_t buflen; } AppLayerParserState; #endif /* __APP_LAYER_PARSER_H__ */ diff --git a/src/eidps.c b/src/eidps.c index a754d03117..8c9fbf114d 100644 --- a/src/eidps.c +++ b/src/eidps.c @@ -902,6 +902,7 @@ int main(int argc, char **argv) RegisterAppLayerParsers(); RegisterHTTPParsers(); +AppLayerParsersInitPostProcess(); TmModuleReceiveNFQRegister(); TmModuleVerdictNFQRegister();