diff --git a/configure.ac b/configure.ac index f54464b61b..4b8a36f4d1 100644 --- a/configure.ac +++ b/configure.ac @@ -399,6 +399,8 @@ AC_DEFINE([HAVE_DETECT_DISABLED], [1], [Detection is disabled])] ]) + AM_CONDITIONAL([BUILD_PCIE_LOGGING], [test ! -z "$TILERA_ROOT"]) + # libraries AC_MSG_CHECKING([for Mpipe]) @@ -407,7 +409,7 @@ [ AC_MSG_RESULT([yes]) AC_DEFINE([HAVE_MPIPE],[1],[mPIPE support is available]) - LDFLAGS="$LDFLAGS -lgxio -ltmc" + LDFLAGS="$LDFLAGS -lgxpci -lgxio -ltmc" ], [AC_MSG_RESULT([no])]) @@ -1628,7 +1630,7 @@ AC_SUBST(e_localstatedir) AC_DEFINE_UNQUOTED([CONFIG_DIR],["$e_sysconfdir"],[Our CONFIG_DIR]) AC_SUBST(e_magic_file) -AC_OUTPUT(Makefile src/Makefile qa/Makefile qa/coccinelle/Makefile rules/Makefile doc/Makefile contrib/Makefile contrib/file_processor/Makefile contrib/file_processor/Action/Makefile contrib/file_processor/Processor/Makefile suricata.yaml scripts/Makefile scripts/suricatasc/Makefile scripts/suricatasc/suricatasc) +AC_OUTPUT(Makefile src/Makefile qa/Makefile qa/coccinelle/Makefile rules/Makefile doc/Makefile contrib/Makefile contrib/file_processor/Makefile contrib/file_processor/Action/Makefile contrib/file_processor/Processor/Makefile contrib/tile_pcie_logd/Makefile suricata.yaml scripts/Makefile scripts/suricatasc/Makefile scripts/suricatasc/suricatasc) SURICATA_BUILD_CONF="Suricata Configuration: AF_PACKET support: ${enable_af_packet} diff --git a/contrib/Makefile.am b/contrib/Makefile.am index 0f7e35caec..0eb1719ab1 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am @@ -1,3 +1,3 @@ -SUBDIRS = file_processor +SUBDIRS = file_processor tile_pcie_logd EXTRA_DIST = suri-graphite diff --git a/contrib/tile_pcie_logd/LICENSE b/contrib/tile_pcie_logd/LICENSE new file mode 100644 index 0000000000..d511905c16 --- /dev/null +++ b/contrib/tile_pcie_logd/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/contrib/tile_pcie_logd/Makefile.am b/contrib/tile_pcie_logd/Makefile.am new file mode 100644 index 0000000000..81a63fcb2b --- /dev/null +++ b/contrib/tile_pcie_logd/Makefile.am @@ -0,0 +1,14 @@ + +EXTRA_DIST = LICENSE README + +if BUILD_PCIE_LOGGING +bin_PROGRAMS = tile_pcie_logd + +tile_pcie_logd_SOURCE = tile_pcie_logd.c + +AM_CFLAGS = -std=gnu99 -Wall -Werror -g -O2 -I$(TILERA_ROOT)/include \ + -DTILEPCI_HOST + +tile_pcie_logd_LDDADD = -pthread + +endif diff --git a/contrib/tile_pcie_logd/README b/contrib/tile_pcie_logd/README new file mode 100644 index 0000000000..32ce7a31d0 --- /dev/null +++ b/contrib/tile_pcie_logd/README @@ -0,0 +1,38 @@ +Introduction +------------ + +This application allows writing files to an x86 host from a TILEncore-Gx +PCIe card. The file name and data are sent over PCIe using the Tilera +Packet Queue API from an aplication running on the Tilera processor. + +The original purpose is to write log files from Suricata (Intrusion +Dectection System) on the x86 host's file system. + +Running The Logger +------------------ + +To run the application, set the TILERA_ROOT environment variable to +point to a valide Tilera MDE, then do: + + make run + +The application should be started before the application on the Tile +side that will be generating the log data. + +By default, queue number 0 is used. The --queue_index=N command line +argument can be used to change the queue number. + +If more than one TILEncore-Gx PCIe card is installed, the --card=M +argument changes to listening to card M. + +Caveats +------- + +Due to the fact that the host driver allocates 4MB physically +contiguous memory for the packet queue ring buffer, it is possible +that this allocation could fail on a host whose memory has been +considerably fragmented. If the host program exits with the following +error, reboot the host and run the test again. + +Host: Failed to open '/dev/tilegxpci%d/packet_queue/t2h/0': Cannot +allocate memory diff --git a/contrib/tile_pcie_logd/tile_pcie_logd.c b/contrib/tile_pcie_logd/tile_pcie_logd.c new file mode 100644 index 0000000000..eb06f9e492 --- /dev/null +++ b/contrib/tile_pcie_logd/tile_pcie_logd.c @@ -0,0 +1,370 @@ +/* Copyright (C) 2013-2014 Tilera Corporation. + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Ken Steele, Tilera Corporation + * \author Tom DeCanio + * + * Host side of PCIe alert logging from Suricata running on a + * TILEncore-Gx card. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define CHECK_SEQ_NUM 1 + +/* The "/dev/tilegxpci%d" device to be used. */ +unsigned int card_index = 0; + +unsigned int debug = 0; + +/* + * When set, drop all alerts rather than write to file. This is for + * testing performance on a file system that can't write files fast + * enough. + */ +unsigned int drop_alerts = 0; + +/* Packet queue index. */ +unsigned int queue_index; + +/* Prefix added to all file paths sent from PCIe card. */ +char * path_prefix = NULL; + +/* By default, the host ring buffer is 4MB in size and it consists of + * 1024 entries of 4KB buffers. Modify GXPCI_HOST_PQ_RING_ENTRIES in + * to change the size of each buffer. To increase + * the total ring buffer size, re-configure the host kernel using + * CONFIG_FORCE_MAX_ZONEORDER. + */ +#ifdef GXPCI_HOST_PQ_SEGMENT_ENTRIES +/* New definitions for MDE 4.1.5 */ +#define RING_BUF_ELEMS GXPCI_HOST_PQ_SEGMENT_ENTRIES +#define RING_BUF_ELEM_SIZE (HOST_PQ_SEGMENT_MAX_SIZE / GXPCI_HOST_PQ_SEGMENT_ENTRIES) +#else +/* Definitions prior to MDE 4.1.5 */ +#define RING_BUF_ELEMS GXPCI_HOST_PQ_RING_ENTRIES +#define RING_BUF_ELEM_SIZE (HOST_PQ_RING_BUF_MAX_SIZE / GXPCI_HOST_PQ_RING_ENTRIES) +#endif + +/*********************************************************************/ +/* Host-Side Packet Consumer */ +/*********************************************************************/ + +#define TAIL_UPDATE_LIMIT_ENABLE + +#define OP_OPEN 1 +#define OP_WRITE 2 +#define OP_CLOSE 3 + +typedef struct { + uint32_t magic; + uint32_t fileno; + uint32_t op; + uint32_t seq; + volatile uint32_t len; + uint32_t next_offset; + char buf[]; +} TrioMsg; + +typedef struct { + FILE *fd; +} FDesc; + +#define MAX_FDESC 1024 + +static FDesc *fdesc[MAX_FDESC]; + +void run_pcie_logging(void) +{ + char dev_name[40]; + int pq_fd; + unsigned int host_ring_buffer_size = RING_BUF_ELEMS * RING_BUF_ELEM_SIZE; + volatile TrioMsg *p; + + printf("Waiting for PCIe logging data from card %d on queue %d...\n", + card_index, queue_index); + + if (path_prefix) { + printf("PCIe logging into directory: '%s'\n", path_prefix); + fflush(stdout); + } + + /* Open the packet queue file. */ + snprintf(dev_name, sizeof(dev_name), "/dev/tilegxpci%d/packet_queue/t2h/%d", + card_index, queue_index); + do { + pq_fd = open(dev_name, O_RDWR); + if (pq_fd < 0) { + sleep(1); + } + } while (pq_fd < 0); + + /* mmap the register space. */ + struct gxpci_host_pq_regs_app* pq_regs = + (struct gxpci_host_pq_regs_app*) + mmap(0, sizeof(struct gxpci_host_pq_regs_app), + PROT_READ | PROT_WRITE, + MAP_SHARED, pq_fd, TILEPCI_PACKET_QUEUE_INDICES_MMAP_OFFSET); + if (pq_regs == MAP_FAILED) { + fprintf(stderr, "Failed to mmap PCIe control registers.\n"); + exit(EXIT_FAILURE); + } + + /* Configure and allocate the ring buffer for the receive queue. */ + tilepci_packet_queue_info_t buf_info; + + buf_info.buf_size = RING_BUF_ELEM_SIZE; + + int err = ioctl(pq_fd, TILEPCI_IOC_SET_PACKET_QUEUE_BUF, &buf_info); + if (err < 0) { + fprintf(stderr, "Failed TILEPCI_IOC_SET_PACKET_QUEUE_BUF: %s\n", + strerror(errno)); + abort(); + } + + /* On the host side, mmap the receive queue region. */ + void* buffer = + mmap(0, host_ring_buffer_size, PROT_READ | PROT_WRITE, + MAP_SHARED, pq_fd, TILEPCI_PACKET_QUEUE_BUF_MMAP_OFFSET); + assert(buffer != MAP_FAILED); + + /* On the host side, mmap the queue status. */ + struct tlr_pq_status *pq_status = + mmap(0, sizeof(struct tlr_pq_status), PROT_READ | PROT_WRITE, + MAP_SHARED, pq_fd, TILEPCI_PACKET_QUEUE_STS_MMAP_OFFSET); + assert(pq_status != MAP_FAILED); + + pq_regs->consumer_index = 0; + + uint64_t packet_count = 0; + volatile uint32_t write; + uint32_t read = 0; + +#ifdef CHECK_SEQ_NUM + uint32_t expect_seq = 1; +#endif + +#ifdef HOST_INTERRUPT_MODE + volatile uint32_t* producer_index = &(pq_status->drv_consumer_index); +#else + volatile uint32_t* producer_index = &(pq_regs->producer_index); +#endif + + volatile uint32_t* consumer_index = &(pq_regs->consumer_index); + volatile enum gxpci_chan_status_t* status = &(pq_status->status); + + while (1) { + if (*status == GXPCI_CHAN_RESET) { + printf("Tile to Host PCIe logging channel was reset.\n"); + fflush(stdout); + return; + } + + // Get packets off the ring buffer by accessing the receive queue at + // the new write index. + write = *producer_index; + + while (write != read) { + if (*status == GXPCI_CHAN_RESET) { + printf("Tile to Host PCIe logging channel was reset.\n"); + fflush(stdout); + return; + } + + packet_count++; + + p = (TrioMsg *)(buffer + ((read&(RING_BUF_ELEMS-1))*RING_BUF_ELEM_SIZE)); + + if (debug) { + fprintf(stdout, "got a message\n"); + fprintf(stdout, "p->magic: %d\n", p->magic); + fprintf(stdout, "p->fileno: %d\n", p->fileno); +#ifdef CHECK_SEQ_NUM + fprintf(stdout, "p->seq: %d\n", p->seq); +#endif + fprintf(stdout, "p->len: %d\n", p->len); + fprintf(stdout, "p->next_offset: %d\n", p->next_offset); + fprintf(stdout, "p->buf: "); + fwrite(&p->buf, sizeof(char), p->len - offsetof(TrioMsg, buf), stdout); + fprintf(stdout, "\n"); + fflush(stdout); + } + +#ifdef CHECK_SEQ_NUM + if (p->seq != expect_seq) { + /* Check for a reset before reporting a bad sequence + * number to prevent confusing users. */ + if (*status == GXPCI_CHAN_RESET) { + printf("Tile to Host PCIe logging channel was reset.\n"); + fflush(stdout); + return; + } + fprintf(stderr, "BAD sequence expected %d got %d\n", expect_seq, p->seq); + return; + } + expect_seq = p->seq + 1; +#endif + + switch (p->op) { + case OP_OPEN: + if (p->fileno < MAX_FDESC) { + fdesc[p->fileno] = malloc(sizeof(FDesc)); + if (fdesc[p->fileno]) { + char mode[2]; + mode[0] = p->buf[0]; + mode[1] = '\0'; + char *file_name = (char *)&p->buf[1]; + if (path_prefix) { + /* Added path_prefix to the start of the + * file name. Added space for '\0' and '\'. + * By default, no prefix is added. */ + int new_size = strlen(path_prefix) + strlen(file_name) + 1 + 1; + char *new_name = malloc(new_size); + if (!new_name) { + fprintf(stderr, "Failed to allocate memory for %s/%s\n", + path_prefix, file_name); + return; + } + snprintf(new_name, new_size, "%s/%s", + path_prefix, file_name); + file_name = new_name; + } + if ((fdesc[p->fileno]->fd = fopen(file_name, mode)) == NULL) { + fprintf(stderr, "Could not open %s: %s\n", + file_name, strerror(errno)); + } else { + printf("Opened '%s' for logging.\n", file_name); + fflush(stdout); + } + } + } else { + fprintf(stderr, "File number %d exceeds Max of %d\n", p->fileno, MAX_FDESC); + } + break; + case OP_WRITE: + if (drop_alerts) { + /* TODO: Report alert count periodically. */ + } else { + if (fdesc[p->fileno] && fdesc[p->fileno]->fd) { + fwrite(&p->buf, sizeof(char), + p->len - offsetof(TrioMsg, buf), + fdesc[p->fileno]->fd); + fflush(fdesc[p->fileno]->fd); + } + } + break; + case OP_CLOSE: + if (fdesc[p->fileno] && fdesc[p->fileno]->fd) { + fclose( fdesc[p->fileno]->fd); + free(fdesc[p->fileno]); + fdesc[p->fileno] = NULL; + } + break; + } + + read++; + /* Update the read index register to inform the tile side + * that the packet has been read. */ + +#ifdef TAIL_UPDATE_LIMIT_ENABLE + if ((packet_count & 0x3f) == 0) + *consumer_index = read; +#else + *consumer_index = read; +#endif + } + } + return; +} + +/* + * Match argument list option. + * Options ending in '=' take an additional value, which may be + * attached in the same argument or detached in the following + * argument. + * @param arglist Points to remaining argv, updated on match. + * @param option The option to match, ending in '=' if it takes a value. + * @return Value if option matches, NULL otherwise. + */ +char *shift_option(char ***arglist, const char* option) +{ + char** args = *arglist; + char* arg = args[0], **rest = &args[1]; + int optlen = strlen(option); + char* val = arg+optlen; + if (option[optlen - 1] != '=') { + if (strcmp(arg, option)) + return NULL; + } else { + if (strncmp(arg, option, optlen-1)) + return NULL; + if (arg[optlen- 1 ] == '\0') + val = *rest++; + else if (arg[optlen - 1] != '=') + return NULL; + } + *arglist = rest; + return val; +} + +int main(int argc, char** argv) +{ + char **args = &argv[1]; + + /* + * Scan command line options. + */ + while (*args) { + char* opt = NULL; + + if ((opt = shift_option(&args, "--queue_index="))) + queue_index = strtoul(opt, NULL, 0); + else if ((opt = shift_option(&args, "--card="))) + card_index = strtoul(opt, NULL, 0); + else if ((opt = shift_option(&args, "--debug"))) + debug = 1; + else if ((opt = shift_option(&args, "--drop"))) + drop_alerts = 1; + else if ((opt = shift_option(&args, "--prefix="))) + path_prefix = opt; + else { + fprintf(stderr, "Unknown option '%s'.\n", args[0]); + exit(EXIT_FAILURE); + } + } + + run_pcie_logging(); + + return 0; +} diff --git a/src/Makefile.am b/src/Makefile.am index 4723747081..9658aa590b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -308,6 +308,7 @@ util-host-os-info.c util-host-os-info.h \ util-ioctl.h util-ioctl.c \ util-ip.h util-ip.c \ util-logopenfile.h util-logopenfile.c \ +util-logopenfile-tile.h util-logopenfile-tile.c \ util-magic.c util-magic.h \ util-memcmp.c util-memcmp.h \ util-mem.h \ diff --git a/src/alert-fastlog.c b/src/alert-fastlog.c index 6657aed495..d703175341 100644 --- a/src/alert-fastlog.c +++ b/src/alert-fastlog.c @@ -62,7 +62,11 @@ #define MODULE_NAME "AlertFastLog" +/* The largest that size allowed for one alert string. */ #define MAX_FASTLOG_ALERT_SIZE 2048 +/* The largest alert buffer that will be written at one time, possibly + * holding multiple alerts. */ +#define MAX_FASTLOG_BUFFER_SIZE (2 * MAX_FASTLOG_ALERT_SIZE) TmEcode AlertFastLogThreadInit(ThreadVars *, void *, void **); TmEcode AlertFastLogThreadDeinit(ThreadVars *, void *); @@ -98,20 +102,11 @@ int AlertFastLogCondition(ThreadVars *tv, const Packet *p) { static inline void AlertFastLogOutputAlert(AlertFastLogThread *aft, char *buffer, int alert_size) { - FILE *fp = aft->file_ctx->fp; - /* fp should never be NULL, but checking here makes sure the value - * is available before the lock is aquired, rather than possibly - * stalling while holding the lock. - */ - if (unlikely(fp == NULL)) - return; - - /* Output the alert string. Only need to lock here. */ SCMutex *file_lock = &aft->file_ctx->fp_mutex; + /* Output the alert string and count alerts. Only need to lock here. */ SCMutexLock(file_lock); aft->file_ctx->alerts++; - fwrite(buffer, alert_size, 1, fp); - fflush(fp); + aft->file_ctx->Write(buffer, alert_size, aft->file_ctx); SCMutexUnlock(file_lock); } @@ -121,7 +116,6 @@ int AlertFastLogger(ThreadVars *tv, void *data, const Packet *p) int i; char timebuf[64]; char *action = ""; - extern uint8_t engine_mode; int decoder_event = 0; CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); @@ -137,10 +131,13 @@ int AlertFastLogger(ThreadVars *tv, void *data, const Packet *p) decoder_event = 1; } - /* Buffer to store the generated alert string. The buffer is - * reused for each alert. + /* Buffer to store the generated alert strings. The buffer is + * filled with alert strings until it doesn't have room to store + * another full alert, only then is the buffer written. This is + * more efficient for multiple alerts and only slightly slower for + * single alerts. */ - char alert_buffer[MAX_FASTLOG_ALERT_SIZE]; + char alert_buffer[MAX_FASTLOG_BUFFER_SIZE]; for (i = 0; i < p->alerts.cnt; i++) { const PacketAlert *pa = &p->alerts.alerts[i]; @@ -148,12 +145,6 @@ int AlertFastLogger(ThreadVars *tv, void *data, const Packet *p) continue; } - if ((pa->action & ACTION_DROP) && IS_ENGINE_MODE_IPS(engine_mode)) { - action = "[Drop] "; - } else if (pa->action & ACTION_DROP) { - action = "[wDrop] "; - } - char proto[16] = ""; if (likely(decoder_event == 0)) { if (SCProtoNameValid(IP_GET_IPPROTO(p)) == TRUE) { @@ -178,7 +169,8 @@ int AlertFastLogger(ThreadVars *tv, void *data, const Packet *p) ":%" PRIu32 "] %s [**] [Classification: %s] [Priority: " "%" PRIu32 "] [**] [Raw pkt: ", timebuf, action, pa->s->gid, pa->s->id, pa->s->rev, pa->s->msg, pa->s->class_msg, pa->s->prio); - PrintBufferRawLineHex(alert_buffer, &size, GET_PKT_DATA(p), GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32); + PrintBufferRawLineHex(alert_buffer, &size, MAX_FASTLOG_ALERT_SIZE, + GET_PKT_DATA(p), GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32); if (p->pcap_cnt != 0) { PrintBufferData(alert_buffer, &size, MAX_FASTLOG_ALERT_SIZE, "] [pcap file packet: %"PRIu64"]\n", p->pcap_cnt); diff --git a/src/alert-pcapinfo.c b/src/alert-pcapinfo.c index a03a43b536..121b191d48 100644 --- a/src/alert-pcapinfo.c +++ b/src/alert-pcapinfo.c @@ -57,6 +57,7 @@ #include "util-print.h" #include "util-proto-name.h" #include "util-optimize.h" +#include "util-logopenfile.h" #define DEFAULT_LOG_FILENAME "alert-pcapinfo.log" /* We need a new file for each pcap */ diff --git a/src/alert-syslog.c b/src/alert-syslog.c index de1af0bdaf..4ad4b964c6 100644 --- a/src/alert-syslog.c +++ b/src/alert-syslog.c @@ -48,6 +48,7 @@ #include "util-proto-name.h" #include "util-syslog.h" #include "util-optimize.h" +#include "util-logopenfile.h" #ifndef OS_WIN32 diff --git a/src/alert-unified2-alert.c b/src/alert-unified2-alert.c index f483a5d073..71d7b2bdf6 100644 --- a/src/alert-unified2-alert.c +++ b/src/alert-unified2-alert.c @@ -53,6 +53,7 @@ #include "util-time.h" #include "util-byte.h" #include "util-misc.h" +#include "util-logopenfile.h" #include "app-layer-parser.h" #include "app-layer-htp.h" diff --git a/src/output-json.h b/src/output-json.h index dda4e82020..4a42f1e532 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -30,6 +30,7 @@ void TmModuleOutputJsonRegister (void); #include "suricata-common.h" #include "util-buffer.h" +#include "util-logopenfile.h" json_t *CreateJSONHeader(Packet *p, int direction_sensative, char *event_type); TmEcode OutputJSON(json_t *js, void *data, uint64_t *count); diff --git a/src/tm-modules.c b/src/tm-modules.c index 26e7c6ca37..a8ef7b6f44 100644 --- a/src/tm-modules.c +++ b/src/tm-modules.c @@ -28,6 +28,7 @@ #include "tm-threads.h" #include "util-debug.h" #include "threads.h" +#include "util-logopenfile.h" void TmModuleDebugList(void) { TmModule *t; @@ -118,53 +119,6 @@ int TmModuleGetIDForTM(TmModule *tm) return -1; } -/** \brief LogFileNewCtx() Get a new LogFileCtx - * \retval LogFileCtx * pointer if succesful, NULL if error - * */ -LogFileCtx *LogFileNewCtx() -{ - LogFileCtx* lf_ctx; - lf_ctx=(LogFileCtx*)SCMalloc(sizeof(LogFileCtx)); - - if(lf_ctx == NULL) - return NULL; - memset(lf_ctx, 0, sizeof(LogFileCtx)); - - SCMutexInit(&lf_ctx->fp_mutex,NULL); - - return lf_ctx; -} - -/** \brief LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory) - * \param motcx pointer to the OutputCtx - * \retval int 1 if succesful, 0 if error - * */ -int LogFileFreeCtx(LogFileCtx *lf_ctx) -{ - if (lf_ctx == NULL) { - SCReturnInt(0); - } - - if (lf_ctx->fp != NULL) - { - SCMutexLock(&lf_ctx->fp_mutex); - fflush(lf_ctx->fp); - fclose(lf_ctx->fp); - SCMutexUnlock(&lf_ctx->fp_mutex); - } - - SCMutexDestroy(&lf_ctx->fp_mutex); - - if (lf_ctx->prefix != NULL) - SCFree(lf_ctx->prefix); - - if(lf_ctx->filename != NULL) - SCFree(lf_ctx->filename); - - SCFree(lf_ctx); - - SCReturnInt(1); -} void TmModuleRunInit(void) { TmModule *t; @@ -278,6 +232,8 @@ const char * TmModuleTmmIdToString(TmmId id) CASE_CODE (TMM_DECODEERFFILE); CASE_CODE (TMM_RECEIVEERFDAG); CASE_CODE (TMM_DECODEERFDAG); + CASE_CODE (TMM_RECEIVEMPIPE); + CASE_CODE (TMM_DECODEMPIPE); CASE_CODE (TMM_RECEIVENAPATECH); CASE_CODE (TMM_DECODENAPATECH); CASE_CODE (TMM_RECEIVEAFP); diff --git a/src/tm-modules.h b/src/tm-modules.h index 1737eceffa..d646560e45 100644 --- a/src/tm-modules.h +++ b/src/tm-modules.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2014 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -61,35 +61,6 @@ typedef struct TmModule_ { TmModule tmm_modules[TMM_SIZE]; -/** Global structure for Output Context */ -typedef struct LogFileCtx_ { - FILE *fp; - /** It will be locked if the log/alert - * record cannot be written to the file in one call */ - SCMutex fp_mutex; - - /** The name of the file */ - char *filename; - - /**< Used by some alert loggers like the unified ones that append - * the date onto the end of files. */ - char *prefix; - - /** Generic size_limit and size_current - * They must be common to the threads accesing the same file */ - uint64_t size_limit; /**< file size limit */ - uint64_t size_current; /**< file current size */ - - /* Alerts on the module (not on the file) */ - uint64_t alerts; - /* flag to avoid multiple threads printing the same stats */ - uint8_t flags; -} LogFileCtx; - -/* flags for LogFileCtx */ -#define LOGFILE_HEADER_WRITTEN 0x01 -#define LOGFILE_ALERTS_PRINTED 0x02 - /** * Structure that output modules use to maintain private data. */ @@ -102,9 +73,6 @@ typedef struct OutputCtx_ { void (*DeInit)(struct OutputCtx_ *); } OutputCtx; -LogFileCtx *LogFileNewCtx(); -int LogFileFreeCtx(LogFileCtx *); - TmModule *TmModuleGetByName(const char *name); TmModule *TmModuleGetById(int id); int TmModuleGetIdByName(const char *name); diff --git a/src/tm-threads-common.h b/src/tm-threads-common.h index 1336d77c0b..c78cc1d856 100644 --- a/src/tm-threads-common.h +++ b/src/tm-threads-common.h @@ -74,10 +74,8 @@ typedef enum { TMM_RECEIVEAFP, TMM_DECODEAFP, TMM_ALERTPCAPINFO, -#ifdef HAVE_MPIPE TMM_RECEIVEMPIPE, TMM_DECODEMPIPE, -#endif TMM_RECEIVENAPATECH, TMM_DECODENAPATECH, TMM_PACKETLOGGER, diff --git a/src/util-error.c b/src/util-error.c index 3b15f30c19..7c5c30425e 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -283,6 +283,7 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_CONF_YAML_ERROR); CASE_CODE (SC_ERR_CONF_NAME_TOO_LONG); CASE_CODE (SC_ERR_APP_LAYER_PROTOCOL_DETECTION); + CASE_CODE (SC_ERR_PCIE_INIT_FAILED); } return "UNKNOWN_ERROR"; diff --git a/src/util-error.h b/src/util-error.h index 852bccf091..224885a39c 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2014 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -272,6 +272,7 @@ typedef enum { SC_ERR_CONF_YAML_ERROR, SC_ERR_CONF_NAME_TOO_LONG, SC_ERR_APP_LAYER_PROTOCOL_DETECTION, + SC_ERR_PCIE_INIT_FAILED, } SCError; const char *SCErrorToString(SCError); diff --git a/src/util-logopenfile-tile.c b/src/util-logopenfile-tile.c new file mode 100644 index 0000000000..c2414fc49b --- /dev/null +++ b/src/util-logopenfile-tile.c @@ -0,0 +1,370 @@ +/* Copyright (C) 2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Tom DeCanio + * \author Ken Steele, Tilera Corporation + * + * File-like output for logging on Tilera PCIe cards (TILEncore-Gx) + * add the option to send logs across PCIe and then write the output + * files on the host system. + * + */ +#include + +#include "suricata-common.h" /* errno.h, string.h, etc. */ +#include "tm-modules.h" /* LogFileCtx */ +#include "conf.h" /* ConfNode, etc. */ +#include "util-atomic.h" +#include "util-logopenfile-tile.h" + +#ifdef __tile__ +#include +#include + +#if MDE_VERSION_CODE >= MDE_VERSION(4,1,0) +#include +#else +#include +#endif + +/* + * Tilera trio (PCIe) configuration. + */ +static gxio_trio_context_t trio_context_body; +static gxio_trio_context_t* trio_context = &trio_context_body; +/* + * gxpci contexts used for log relay + */ +static gxpci_context_t gxpci_context_body; +static gxpci_context_t *gxpci_context = &gxpci_context_body; +/* The TRIO index. */ +static int trio_index = 0; + +/* The queue index of a packet queue. */ +static unsigned int queue_index = 0; + +/* The local PCIe MAC index. */ +static int loc_mac; + +/* + * Code for writing files over PCIe to host on Tilera TILEncore PCIe cards. + */ + +#define OP_OPEN 1 +#define OP_WRITE 2 +#define OP_CLOSE 3 + +/** Maximum number of commands in one PCIe function call */ +#define MAX_CMDS_BATCH 64 + +typedef struct { + uint32_t magic; + uint32_t fileno; + uint32_t op; + uint32_t seq; + uint32_t len; + uint32_t next_offset; + char buf[]; +} __attribute__((__packed__)) PcieMsg; + +static int gxpci_fileno = 0; +static int pcie_initialized = 0; +/* Allocate a Huge page of memory, registered with Trio, into which + data to be sent over PCIe is written. Each write starts at wc_pos. +*/ +static char *log_mem = NULL; +static uint64_t wr_pos; /* write position within log_mem */ + +static SCMutex raw_mutex __attribute__((aligned(64))); +static SCMutex pcie_mutex __attribute__((aligned(64))); +#define CHECK_SEQ_NUM 1 +#ifdef CHECK_SEQ_NUM +static uint32_t raw_seq = 0; +#endif +static uint32_t comps_rcvd = 0; +/* Block of memory registered with PCIe DMA engine as a source for + * PCIe data transfers. Must be <= Huge Page size (16 MB). + * Must be large enough that it can't wrap before first PCIe transfer + * has completed. + */ +#define PCIE_MEMORY_BLOCK_SIZE (4 * 1024 * 1024) + +/* Send a buffer over PCIe to Host memory. + * len must be smaller than one Packet Queue transfer block. + * TODO: Check errors + */ +static void TilePcieDMABuf(void *buf, uint32_t len) +{ + gxpci_comp_t comp[MAX_CMDS_BATCH]; + gxpci_cmd_t cmd; + int result; + int credits; + + SCMutexLock(&pcie_mutex); + +#ifdef CHECK_SEQ_NUM + ((PcieMsg *)buf)->seq = ++raw_seq; + __insn_mf(); +#endif + + /* Wait for credits to be available for more PCIe writes. */ + do { + result = gxpci_get_comps(gxpci_context, comp, 0, MAX_CMDS_BATCH); + if (result) { + if (unlikely(result == GXPCI_ERESET)) { + SCLogInfo("gxpci channel is reset"); + return; + } else { + __sync_fetch_and_add(&comps_rcvd, result); + } + } + + credits = gxpci_get_cmd_credits(gxpci_context); + if (unlikely(credits == GXPCI_ERESET)) { + SCLogInfo("gxpci channel is reset"); + return; + } + } while (credits == 0); + + cmd.buffer = buf; + /* Round transfer size up to next host cache-line. This will + * transfer more data, but is more efficient. + */ + cmd.size = (len + (CLS - 1)) & ~(CLS - 1); + + __insn_mf(); + + /* Loop until the command is sent. */ + do { + /* Send PCIe command to packet queue from tile to host. */ + result = gxpci_pq_t2h_cmd(gxpci_context, &cmd); + if (result == 0) + break; + if (result == GXPCI_ERESET) { + SCLogInfo("gxpci channel is reset"); + break; + } + /* Not enough credits to send command? */ + if (result == GXPCI_ECREDITS) + continue; + } while (1); + + SCMutexUnlock(&pcie_mutex); +} + +/* Allocate a buffer for data that can be sent over PCIe. Reserves + * space at the beginning for the Pcie msg. The buffer is allocated + * from a 4MB pool on one huge page. The allocation simply walks + * throught the buffer sequentially. This removes the need to free + * the buffers, as they simply age out. + */ +static PcieMsg *TilePcieAllocateBuffer(size_t size) +{ + size += sizeof(PcieMsg); + /* Round up to cache-line size */ + size = (size + (CLS - 1)) & ~(CLS - 1); + + PcieMsg *pmsg; + SCMutexLock(&raw_mutex); + pmsg = (PcieMsg *)&log_mem[wr_pos]; + wr_pos += size; + if (wr_pos > PCIE_MEMORY_BLOCK_SIZE) { + /* Don't have enough space at the end of the memory block, so + * wrap to the start. + */ + pmsg = (PcieMsg *)&log_mem[0]; + wr_pos = size; + + } + SCMutexUnlock(&raw_mutex); + + return pmsg; +} + +static void PcieWriteOpen(PcieFile *fp, const char *path, const char append) +{ + /* Need space for file name, file mode character and string termination */ + const int buffer_size = strlen(path) + 2; + + /* Allocate space in the PCIe output buffer */ + PcieMsg *p = TilePcieAllocateBuffer(buffer_size); + + p->magic = 5555; + p->fileno = fp->fileno; + p->op = OP_OPEN; + p->len = offsetof(PcieMsg, buf); + /* Format is one character Mode, followed by file path. */ + p->len += snprintf(p->buf, buffer_size, "%c%s", append, path); + + TilePcieDMABuf(p, p->len); +} + +static int TilePcieWrite(const char *buffer, int buffer_len, LogFileCtx *log_ctx) +{ + PcieFile *fp = log_ctx->pcie_fp; + /* Allocate space in the PCIe output buffer */ + PcieMsg *p = TilePcieAllocateBuffer(buffer_len); + + p->magic = 5555; + p->fileno = fp->fileno; + p->op = OP_WRITE; + p->len = offsetof(PcieMsg, buf); + p->len += buffer_len; + p->next_offset = 0; + + /* Can remove the need for this memcpy later. */ + memcpy(p->buf, buffer, buffer_len); + + TilePcieDMABuf(p, p->len); + + return buffer_len; +} + +static PcieFile *TileOpenPcieFpInternal(const char *path, const char append_char) +{ + int result; + PcieFile *fp; + + /* Only initialize once */ + if (SCAtomicCompareAndSwap(&pcie_initialized, 0, 1)) { + SCMutexInit(&raw_mutex, NULL); + SCMutexInit(&pcie_mutex, NULL); + + SCLogInfo("Initializing Tile-Gx PCIe index %d / %d, queue: %d", + trio_index, loc_mac, queue_index); + + result = gxio_trio_init(trio_context, trio_index); + if (result < 0) { + pcie_initialized = 0; + SCLogError(SC_ERR_PCIE_INIT_FAILED, + "gxio_trio_init() failed: %d: %s", + result, gxio_strerror(result)); + return NULL; + } + + result = gxpci_init(trio_context, gxpci_context, trio_index, loc_mac); + if (result < 0) { + pcie_initialized = 0; + SCLogError(SC_ERR_PCIE_INIT_FAILED, + "gxpci_init() failed: %d: %s", + result, gxpci_strerror(result)); + return NULL; + } + + /* + * This indicates that we need to allocate an ASID ourselves, + * instead of using one that is allocated somewhere else. + */ + int asid = GXIO_ASID_NULL; + + result = gxpci_open_queue(gxpci_context, asid, GXPCI_PQ_T2H, 0, + queue_index, 0, 0); + if (result < 0) { + pcie_initialized = 0; + SCLogError(SC_ERR_PCIE_INIT_FAILED, + "gxpci_open_queue() failed: %d: %s", + result, gxpci_strerror(result)); + return NULL; + } + + /* + * Allocate and register data buffer + */ + size_t hugepagesz = tmc_alloc_get_huge_pagesize(); + tmc_alloc_t alloc = TMC_ALLOC_INIT; + tmc_alloc_set_huge(&alloc); + tmc_alloc_set_home(&alloc, TMC_ALLOC_HOME_HASH); + tmc_alloc_set_pagesize_exact(&alloc, hugepagesz); + log_mem = tmc_alloc_map(&alloc, hugepagesz); + BUG_ON(PCIE_MEMORY_BLOCK_SIZE > hugepagesz); + + result = gxpci_iomem_register(gxpci_context, log_mem, hugepagesz); + if (result < 0) { + pcie_initialized = 0; + SCLogError(SC_ERR_PCIE_INIT_FAILED, + "gxpci_iomem_register() failed: %d: %s", + result, gxpci_strerror(result)); + return NULL; + } + } + fp = SCMalloc(sizeof(PcieFile)); + if (fp == NULL) { + SCLogError(SC_ERR_PCIE_INIT_FAILED, + "Failed to Allocate memory for PCIe file pointer"); + + return NULL; + } + + /* Sequentially allocate File descriptor numbers. Not currently ever freed */ + fp->fileno = SCAtomicFetchAndAdd(&gxpci_fileno, 1); + PcieWriteOpen(fp, path, append_char); + + return fp; +} + +/** \brief Close a PCIe file + * \param PCIe file desriptor + */ +static void TileClosePcieFp(LogFileCtx *log_ctx) +{ + SCLogInfo("Closing Tile-Gx PCIe: %s", log_ctx->filename); + + /* TODO: Need to count open files and close when reaches zero. */ + SCMutexLock(&pcie_mutex); + + if (gxpci_context) { + gxpci_destroy(gxpci_context); + gxpci_context = NULL; + } + + SCMutexUnlock(&pcie_mutex); + + free(log_ctx->pcie_fp); +} + +/** \brief open the indicated file remotely over PCIe to a host + * \param path filesystem path to open + * \param append_setting open file with O_APPEND: "yes" or "no" + * \retval FILE* on success + * \retval NULL on error + */ +PcieFile *TileOpenPcieFp(LogFileCtx *log_ctx, const char *path, + const char *append_setting) +{ + PcieFile *ret = NULL; + if (strcasecmp(append_setting, "yes") == 0) { + ret = TileOpenPcieFpInternal(path, 'a'); + } else { + ret = TileOpenPcieFpInternal(path, 'w'); + } + + /* Override the default Write and Close functions + * with PCIe Write and Close functions. + */ + log_ctx->Write = TilePcieWrite; + log_ctx->Close = TileClosePcieFp; + + if (ret == NULL) + SCLogError(SC_ERR_FOPEN, "Error opening PCIe file: \"%s\": %s", + path, strerror(errno)); + return ret; +} + +#endif /* __tilegx__ */ diff --git a/src/util-logopenfile-tile.h b/src/util-logopenfile-tile.h new file mode 100644 index 0000000000..4d1ac6576e --- /dev/null +++ b/src/util-logopenfile-tile.h @@ -0,0 +1,32 @@ +/* Copyright (C) 2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Ken Steele, Tilera Corporation + */ + +#ifndef __UTIL_LOGOPENFILE_TILE_H__ +#define __UTIL_LOGOPENFILE_TILE_H__ + +#include "util-logopenfile.h" /* LogFileCtx */ + +PcieFile *TileOpenPcieFp(LogFileCtx *log_ctx, const char *path, + const char *append_setting); + +#endif /* __UTIL_LOGOPENFILE_TILE_H__ */ diff --git a/src/util-logopenfile.c b/src/util-logopenfile.c index e35f049421..743e24c9ea 100644 --- a/src/util-logopenfile.c +++ b/src/util-logopenfile.c @@ -1,5 +1,5 @@ /* vi: set et ts=4: */ -/* Copyright (C) 2007-2011 Open Information Security Foundation +/* Copyright (C) 2007-2014 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -31,6 +31,8 @@ #include "tm-modules.h" /* LogFileCtx */ #include "conf.h" /* ConfNode, etc. */ #include "output.h" /* DEFAULT_LOG_* */ +#include "util-logopenfile.h" +#include "util-logopenfile-tile.h" /** \brief connect to the indicated local stream socket, logging any errors * \param path filesystem path to connect to @@ -71,6 +73,20 @@ err: return NULL; } +static int SCLogFileWrite(const char *buffer, int buffer_len, LogFileCtx *log_ctx) +{ + int ret = fwrite(buffer, buffer_len, 1, log_ctx->fp); + fflush(log_ctx->fp); + + return ret; +} + +static void SCLogFileClose(LogFileCtx *log_ctx) +{ + if (log_ctx->fp) + fclose(log_ctx->fp); +} + /** \brief open the indicated file, logging any errors * \param path filesystem path to open * \param append_setting open file with O_APPEND: "yes" or "no" @@ -94,6 +110,24 @@ SCLogOpenFileFp(const char *path, const char *append_setting) return ret; } +/** \brief open the indicated file remotely over PCIe to a host + * \param path filesystem path to open + * \param append_setting open file with O_APPEND: "yes" or "no" + * \retval FILE* on success + * \retval NULL on error + */ +static PcieFile *SCLogOpenPcieFp(LogFileCtx *log_ctx, const char *path, + const char *append_setting) +{ +#ifndef __tile__ + SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, + "PCIe logging only supported on Tile-Gx Architecture."); + return NULL; +#else + return TileOpenPcieFp(log_ctx, path, append_setting); +#endif +} + /** \brief open a generic output "log file", which may be a regular file or a socket * \param conf ConfNode structure for the output section in question * \param log_ctx Log file context allocated by caller @@ -142,30 +176,87 @@ SCConfLogOpenGeneric(ConfNode *conf, if (filetype == NULL) filetype = DEFAULT_LOG_FILETYPE; + const char *append = ConfNodeLookupChildValue(conf, "append"); + if (append == NULL) + append = DEFAULT_LOG_MODE_APPEND; + // Now, what have we been asked to open? if (strcasecmp(filetype, "unix_stream") == 0) { log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_STREAM); + if (log_ctx->fp == NULL) + return -1; // Error already logged by Open...Fp routine } else if (strcasecmp(filetype, "unix_dgram") == 0) { log_ctx->fp = SCLogOpenUnixSocketFp(log_path, SOCK_DGRAM); + if (log_ctx->fp == NULL) + return -1; // Error already logged by Open...Fp routine } else if (strcasecmp(filetype, DEFAULT_LOG_FILETYPE) == 0) { - const char *append; - - append = ConfNodeLookupChildValue(conf, "append"); - if (append == NULL) - append = DEFAULT_LOG_MODE_APPEND; log_ctx->fp = SCLogOpenFileFp(log_path, append); + if (log_ctx->fp == NULL) + return -1; // Error already logged by Open...Fp routine + } else if (strcasecmp(filetype, "pcie") == 0) { + log_ctx->pcie_fp = SCLogOpenPcieFp(log_ctx, log_path, append); + if (log_ctx->pcie_fp == NULL) + return -1; // Error already logged by Open...Fp routine } else { SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry for " - "%s.type. Expected \"regular\" (default), \"unix_stream\" " + "%s.type. Expected \"regular\" (default), \"unix_stream\", " + "\"pcie\" " "or \"unix_dgram\"", conf->name); } - if (log_ctx->fp == NULL) - return -1; // Error already logged by Open...Fp routine - SCLogInfo("%s output device (%s) initialized: %s", conf->name, filetype, filename); return 0; } + +/** \brief LogFileNewCtx() Get a new LogFileCtx + * \retval LogFileCtx * pointer if succesful, NULL if error + * */ +LogFileCtx *LogFileNewCtx(void) +{ + LogFileCtx* lf_ctx; + lf_ctx = (LogFileCtx*)SCMalloc(sizeof(LogFileCtx)); + + if (lf_ctx == NULL) + return NULL; + memset(lf_ctx, 0, sizeof(LogFileCtx)); + + SCMutexInit(&lf_ctx->fp_mutex,NULL); + + // Default Write and Close functions + lf_ctx->Write = SCLogFileWrite; + lf_ctx->Close = SCLogFileClose; + + return lf_ctx; +} + +/** \brief LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory) + * \param motcx pointer to the OutputCtx + * \retval int 1 if succesful, 0 if error + * */ +int LogFileFreeCtx(LogFileCtx *lf_ctx) +{ + if (lf_ctx == NULL) { + SCReturnInt(0); + } + + if (lf_ctx->fp != NULL) { + SCMutexLock(&lf_ctx->fp_mutex); + lf_ctx->Close(lf_ctx); + SCMutexUnlock(&lf_ctx->fp_mutex); + } + + SCMutexDestroy(&lf_ctx->fp_mutex); + + if (lf_ctx->prefix != NULL) + SCFree(lf_ctx->prefix); + + if(lf_ctx->filename != NULL) + SCFree(lf_ctx->filename); + + SCFree(lf_ctx); + + SCReturnInt(1); +} diff --git a/src/util-logopenfile.h b/src/util-logopenfile.h index 92aa345fee..ef15736fab 100644 --- a/src/util-logopenfile.h +++ b/src/util-logopenfile.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2011 Open Information Security Foundation +/* Copyright (C) 2007-2014 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -27,6 +27,49 @@ #include "conf.h" /* ConfNode */ #include "tm-modules.h" /* LogFileCtx */ +typedef struct { + uint16_t fileno; +} PcieFile; + +/** Global structure for Output Context */ +typedef struct LogFileCtx_ { + union { + FILE *fp; + PcieFile *pcie_fp; + }; + + int (*Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp); + void (*Close)(struct LogFileCtx_ *fp); + + /** It will be locked if the log/alert + * record cannot be written to the file in one call */ + SCMutex fp_mutex; + + /** The name of the file */ + char *filename; + + /**< Used by some alert loggers like the unified ones that append + * the date onto the end of files. */ + char *prefix; + + /** Generic size_limit and size_current + * They must be common to the threads accesing the same file */ + uint64_t size_limit; /**< file size limit */ + uint64_t size_current; /**< file current size */ + + /* Alerts on the module (not on the file) */ + uint64_t alerts; + /* flag to avoid multiple threads printing the same stats */ + uint8_t flags; +} LogFileCtx; + +/* flags for LogFileCtx */ +#define LOGFILE_HEADER_WRITTEN 0x01 +#define LOGFILE_ALERTS_PRINTED 0x02 + +LogFileCtx *LogFileNewCtx(void); +int LogFileFreeCtx(LogFileCtx *); + int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *, const char *); #endif /* __UTIL_LOGOPENFILE_H__ */