From 8a88ed0fedcc6a3f32045a98d2e51fd360259e36 Mon Sep 17 00:00:00 2001 From: max197616 Date: Tue, 30 Jan 2018 14:26:05 +0300 Subject: [PATCH] Version 0.95 --- configure.ac | 284 +++++++++++- include/Makefile.am | 2 +- include/acl.h | 18 + include/arr.h | 208 +++++++++ include/cfg.h | 41 ++ include/cmdlinetask.h | 18 + include/dpdk.h | 19 + include/dpi.h | 24 - include/dtypes.h | 19 + include/flow.h | 343 ++++++++++++-- include/http.h | 70 +++ include/main.h | 56 +-- include/notification.h | 19 + include/params.h | 88 ++++ include/sender.h | 248 ++++++++++- include/stats.h | 84 +++- include/tcp.h | 19 + include/tries.h | 153 +++++++ include/utils.h | 24 + include/worker.h | 115 ++--- src/Makefile.am | 6 +- src/acl.cpp | 27 +- src/cmdlinetask.cpp | 304 ++++++++++++- src/flow.cpp | 191 +++++++- src/http.cpp | 132 ++++++ src/main.cpp | 508 ++++++++++----------- src/notification.cpp | 21 +- src/reloadtask.cpp | 62 +-- src/sender.cpp | 326 ++++++++------ src/statistictask.cpp | 60 +-- src/tries.cpp | 379 ++++++++++++++++ src/utils.cpp | 49 ++ src/worker.cpp | 988 +++++++++++++++++------------------------ 33 files changed, 3577 insertions(+), 1328 deletions(-) create mode 100644 include/arr.h create mode 100644 include/cfg.h delete mode 100644 include/dpi.h create mode 100644 include/http.h create mode 100644 include/params.h create mode 100644 include/tries.h create mode 100644 include/utils.h create mode 100644 src/http.cpp create mode 100644 src/tries.cpp create mode 100644 src/utils.cpp diff --git a/configure.ac b/configure.ac index 887323f..896dd8a 100644 --- a/configure.ac +++ b/configure.ac @@ -2,13 +2,15 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.69]) -AC_INIT(extFilter, 0.89, max1976@mail.ru) +AC_INIT(extFilter, 0.95, max1976@mail.ru) DPDK_HOME= DPDK_TARGET= PEAFOWL_HOME=./peafowl +MARISA_HOME=./marisa + AM_INIT_AUTOMAKE() m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(no)]) @@ -47,9 +49,9 @@ AC_MSG_CHECKING([for debug enabled]) if test x"$debug" = x"true"; then - CXXFLAGS="$CXXFLAGS -std=c++11 -O0 -g -Wall -pthread -msse -msse2 -msse3 -mssse3" + CXXFLAGS="$CXXFLAGS -std=c++11 -O0 -g -Wall -pthread" else - CXXFLAGS="$CXXFLAGS -std=c++11 -O3 -Wall -fno-stack-protector -pthread -msse -msse2 -msse3 -mssse3 -march=native" + CXXFLAGS="$CXXFLAGS -std=c++11 -O3 -Wall -fno-stack-protector -pthread" fi AC_COMPILE_IFELSE([AC_LANG_SOURCE( @@ -70,6 +72,243 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE( AC_MSG_FAILURE(['$CXX $CXXFLAGS' does not accept ISO C++11])) +# Macros for SSE availability check. +AC_DEFUN([EXTFILTER_ENABLE_SSE2], + [AC_EGREP_CPP([yes], [ +#ifdef __SSE2__ +yes +#endif + ], [enable_sse2="yes"], [enable_sse2="no"])]) +AC_DEFUN([EXTFILTER_ENABLE_SSE3], + [AC_EGREP_CPP([yes], [ +#ifdef __SSE3__ +yes +#endif + ], [enable_sse3="yes"], [enable_sse3="no"])]) +AC_DEFUN([EXTFILTER_ENABLE_SSSE3], + [AC_EGREP_CPP([yes], [ +#ifdef __SSSE3__ +yes +#endif + ], [enable_ssse3="yes"], [enable_ssse3="no"])]) +AC_DEFUN([EXTFILTER_ENABLE_SSE4_1], + [AC_EGREP_CPP([yes], [ +#ifdef __SSE4_1__ +yes +#endif + ], [enable_sse4_1="yes"], [enable_sse4_1="no"])]) +AC_DEFUN([EXTFILTER_ENABLE_SSE4_2], + [AC_EGREP_CPP([yes], [ +#ifdef __SSE4_2__ +yes +#endif + ], [enable_sse4_2="yes"], [enable_sse4_2="no"])]) +AC_DEFUN([EXTFILTER_ENABLE_SSE4], + [AC_EGREP_CPP([yes], [ +#if defined(__POPCNT__) && defined(__SSE4_2__) +yes +#endif + ], [enable_sse4="yes"], [enable_sse4="no"])]) +AC_DEFUN([EXTFILTER_ENABLE_SSE4A], + [AC_EGREP_CPP([yes], [ +#ifdef __SSE4A__ +yes +#endif + ], [enable_sse4a="yes"], [enable_sse4a="no"])]) +AC_DEFUN([EXTFILTER_ENABLE_AVX], + [AC_EGREP_CPP([yes], [ +#ifdef __AVX__ +yes +#endif + ], [enable_avx="yes"], [enable_avx="no"])]) +AC_DEFUN([EXTFILTER_ENABLE_AVX2], + [AC_EGREP_CPP([yes], [ +#ifdef __AVX2__ +yes +#endif + ], [enable_avx2="yes"], [enable_avx2="no"])]) + + +AC_DEFUN([EXTFILTER_ENABLE_POPCNT], + [AC_EGREP_CPP([yes], [ +#ifdef __POPCNT__ +yes +#endif + ], [enable_popcnt="yes"], [enable_popcnt="no"])]) + +# Enable native cpu instructions. +AC_MSG_CHECKING([whether to enable optimization for native cpu]) +AC_ARG_ENABLE([native-code], + [AS_HELP_STRING([--enable-native-code], + [generate instructions for native cpu [default=no]])], + [], + [enable_native_code="no"]) +AS_IF([test "x${enable_native_code}" != "xno"], [ + CFLAGS="$CPPFLAGS -march=native -mtune=native" + CPPFLAGS="$CPPFLAGS -march=native -mtune=native" + CXXFLAGS="$CXXFLAGS -march=native -mtune=native" + EXTFILTER_ENABLE_SSE2 + EXTFILTER_ENABLE_SSE3 + EXTFILTER_ENABLE_SSE4_1 + EXTFILTER_ENABLE_SSE4_2 + EXTFILTER_ENABLE_SSE4 + EXTFILTER_ENABLE_SSE4A + EXTFILTER_ENABLE_AVX + EXTFILTER_ENABLE_AVX2 + EXTFILTER_ENABLE_POPCNT +]) +AC_MSG_RESULT([${enable_native_code}]) + +# Checks for SSE availability. +AC_MSG_CHECKING([whether to use SSE2]) +AC_ARG_ENABLE([sse2], + [AS_HELP_STRING([--enable-sse2], + [use SSE2 [default=no]])], + [], + [enable_sse2="no"]) +AS_IF([test "x${enable_sse2}" != "xno"], [EXTFILTER_ENABLE_SSE2]) +AC_MSG_RESULT([${enable_sse2}]) + +AC_MSG_CHECKING([whether to use SSE3]) +AC_ARG_ENABLE([sse3], + [AS_HELP_STRING([--enable-sse3], + [use SSE3 [default=no]])], + [], + [enable_sse3="no"]) +AS_IF([test "x${enable_sse3}" != "xno"], [EXTFILTER_ENABLE_SSE3]) +AC_MSG_RESULT([${enable_sse3}]) + +AC_MSG_CHECKING([whether to use SSSE3]) +AC_ARG_ENABLE([ssse3], + [AS_HELP_STRING([--enable-ssse3], + [use SSSE3 [default=no]])], + [], + [enable_ssse3="no"]) +AS_IF([test "x${enable_ssse3}" != "xno"], [EXTFILTER_ENABLE_SSSE3]) +AC_MSG_RESULT([${enable_ssse3}]) + +AC_MSG_CHECKING([whether to use SSE4.1]) +AC_ARG_ENABLE([sse4.1], + [AS_HELP_STRING([--enable-sse4.1], + [use SSE4.1 [default=no]])], + [], + [enable_sse4_1="no"]) +AS_IF([test "x${enable_sse4_1}" != "xno"], [EXTFILTER_ENABLE_SSE4_1]) +AC_MSG_RESULT([${enable_sse4_1}]) + +AC_MSG_CHECKING([whether to use SSE4.2]) +AC_ARG_ENABLE([sse4.2], + [AS_HELP_STRING([--enable-sse4.2], + [use SSE4.2 [default=no]])], + [], + [enable_sse4_2="no"]) +AS_IF([test "x${enable_sse4_2}" != "xno"], [EXTFILTER_ENABLE_SSE4_2]) +AC_MSG_RESULT([${enable_sse4_2}]) + +AC_MSG_CHECKING([whether to use SSE4]) +AC_ARG_ENABLE([sse4], + [AS_HELP_STRING([--enable-sse4], + [use SSE4 [default=no]])], + [], + [enable_sse4="no"]) +AS_IF([test "x${enable_sse4}" != "xno"], [EXTFILTER_ENABLE_SSE4]) +AC_MSG_RESULT([${enable_sse4}]) + +AC_MSG_CHECKING([whether to use SSE4a]) +AC_ARG_ENABLE([sse4a], + [AS_HELP_STRING([--enable-sse4a], + [use SSE4a [default=no]])], + [], + [enable_sse4a="no"]) +AS_IF([test "x${enable_sse4a}" != "xno"], [EXTFILTER_ENABLE_SSE4A]) +AC_MSG_RESULT([${enable_sse4a}]) + +AC_MSG_CHECKING([whether to use AVX]) +AC_ARG_ENABLE([avx], + [AS_HELP_STRING([--enable-avx], + [use AVX [default=no]])], + [], + [enable_avx="no"]) +AS_IF([test "x${enable_avx}" != "xno"], [EXTFILTER_ENABLE_AVX]) +AC_MSG_RESULT([${enable_avx}]) + +AC_MSG_CHECKING([whether to use AVX2]) +AC_ARG_ENABLE([avx2], + [AS_HELP_STRING([--enable-avx2], + [use AVX2 [default=no]])], + [], + [enable_avx2="no"]) +AS_IF([test "x${enable_avx2}" != "xno"], [EXTFILTER_ENABLE_AVX2]) +AC_MSG_RESULT([${enable_avx2}]) + +AC_MSG_CHECKING([whether to use popcnt]) +AC_ARG_ENABLE([popcnt], + [AS_HELP_STRING([--enable-popcnt], + [use POPCNT [default=no]])], + [], + [enable_popcnt="no"]) +AS_IF([test "x${enable_popcnt}" != "xno"], [EXTFILTER_ENABLE_POPCNT]) +AC_MSG_RESULT([${enable_popcnt}]) + + + +AS_IF([test "x${enable_popcnt}" != "xno"], [ + enable_sse3="yes" +]) + +AS_IF([test "x${enable_avx2}" != "xno"], [ + enable_popcnt="yes" + enable_avx2="yes" +]) +AS_IF([test "x${enable_avx}" != "xno"], [ + enable_popcnt="yes" + enable_avx="yes" +]) +AS_IF([test "x${enable_sse4a}" != "xno"], [ + enable_popcnt="yes" + enable_sse3="yes" +]) +AS_IF([test "x${enable_sse4}" != "xno"], [ + enable_popcnt="yes" + enable_sse4_2="yes" +]) +AS_IF([test "x${enable_sse4_2}" != "xno"], [ + enable_popcnt="yes" + enable_sse4_1="yes" +]) +AS_IF([test "x${enable_sse4_1}" != "xno"], [ + enable_ssse3="yes" +]) +AS_IF([test "x${enable_ssse3}" != "xno"], [ + enable_sse3="yes" +]) +AS_IF([test "x${enable_sse3}" != "xno"], [ + enable_sse2="yes" +]) + +AS_IF([test "x${enable_popcnt}" != "xno"], [ + CXXFLAGS="$CXXFLAGS -mpopcnt" +]) + +if test "x${enable_avx2}" != "xno"; then + CXXFLAGS="$CXXFLAGS -mavx2" +elif test "x${enable_avx}" != "xno"; then + CXXFLAGS="$CXXFLAGS -mavx" +elif test "x${enable_sse4a}" != "xno"; then + CXXFLAGS="$CXXFLAGS -msse4a" +elif test "x${enable_sse4}" != "xno"; then + CXXFLAGS="$CXXFLAGS -msse4" +elif test "x${enable_sse4_2}" != "xno"; then + CXXFLAGS="$CXXFLAGS -msse4.2" +elif test "x${enable_sse4_1}" != "xno"; then + CXXFLAGS="$CXXFLAGS -msse4.1" +elif test "x${enable_ssse3}" != "xno"; then + CXXFLAGS="$CXXFLAGS -mssse3" +elif test "x${enable_sse3}" != "xno"; then + CXXFLAGS="$CXXFLAGS -msse3" +elif test "x${enable_sse2}" != "xno"; then + CXXFLAGS="$CXXFLAGS -msse2" +fi # Checks for libraries. @@ -196,4 +435,43 @@ AC_MSG_RESULT(yes) AC_SUBST(PEAFOWL_INCLUDE, $PEAFOWL_HOME/src) AC_SUBST(PEAFOWL_LIB, $PEAFOWL_LIB) +MARISA_LIB=$MARISA_HOME/lib/marisa/.libs/libmarisa.a +AC_MSG_CHECKING(for $MARISA_LIB) +if test -f "$MARISA_LIB" ; then : + +else + AC_MSG_RESULT([not found, compiling...]) + cd $MARISA_HOME; autoreconf -i; ./configure --enable-native-code; make; cd - + if test -f "$MARISA_LIB" ; then : + + else + AC_MSG_ERROR([Marisa-trie library not found!]) + fi +fi +AC_MSG_RESULT(yes) + +AC_SUBST(MARISA_INCLUDE, $MARISA_HOME/include) +AC_SUBST(MARISA_LIB, $MARISA_LIB) + AC_OUTPUT(Makefile src/Makefile include/Makefile) + +AS_ECHO([]) +AS_ECHO(["${PACKAGE_NAME} ${PACKAGE_VERSION} configuration:"]) +AS_ECHO(["-------------------------------"]) +AS_ECHO([" HOST: ${HOST}"]) +AS_ECHO([" CXX: ${CXX}"]) +AS_ECHO([" CXXFLAGS: ${CXXFLAGS}"]) +AS_ECHO([" LDFLAGS: ${LDFLAGS}"]) +AS_ECHO([" PREFIX: ${prefix}"]) +AS_ECHO([]) +AS_ECHO([" NATIVE: ${enable_native_code}"]) +AS_ECHO([" SSE2: ${enable_sse2}"]) +AS_ECHO([" SSE3: ${enable_sse3}"]) +AS_ECHO([" SSSE3: ${enable_ssse3}"]) +AS_ECHO([" SSE4.1: ${enable_sse4_1}"]) +AS_ECHO([" SSE4.2: ${enable_sse4_2}"]) +AS_ECHO([" SSE4a: ${enable_sse4a}"]) +AS_ECHO([" AVX: ${enable_avx}"]) +AS_ECHO([" AVX2: ${enable_avx2}"]) +AS_ECHO([" POPCNT: ${enable_popcnt}"]) +AS_ECHO([]) diff --git a/include/Makefile.am b/include/Makefile.am index 82f910d..60fc3cf 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -1,2 +1,2 @@ -noinst_HEADERS = main.h worker.h AhoCorasickPlus.h actypes.h ahocorasick.h node.h statistictask.h sender.h sendertask.h stats.h reloadtask.h flow.h dtypes.h replace.h mpool.h dpdk.h acl.h cmdlinetask.h notification.h dpi.h +noinst_HEADERS = main.h worker.h statistictask.h sender.h sendertask.h stats.h reloadtask.h flow.h dtypes.h dpdk.h acl.h cmdlinetask.h notification.h tries.h cfg.h utils.h http.h params.h arr.h diff --git a/include/acl.h b/include/acl.h index 0c387f5..8f1c360 100644 --- a/include/acl.h +++ b/include/acl.h @@ -1,3 +1,21 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ #pragma once #include diff --git a/include/arr.h b/include/arr.h new file mode 100644 index 0000000..6024bb6 --- /dev/null +++ b/include/arr.h @@ -0,0 +1,208 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#pragma once + +#define DPDK_MEMALLOC 1 + +#ifdef DPDK_MEMALLOC +#include +#include +#endif + +enum en_alfs_type_t +{ + en_alfs_short, + en_alfs_long +}; + +// массив с самыми старыми записями в начале и самыми свежими в конце. +// в голове массива всегда самая старая запись +template +class ArrayListFixedSize +{ +private: + struct node + { + T *p_flow; + uint32_t next; + uint32_t prev; + } __rte_cache_aligned; + + uint64_t lifetime; + uint32_t cnt; + node *p_nodes_arr; + uint32_t head; + uint32_t tail; + uint8_t worker_id; + en_alfs_type_t alfs_type; + uint32_t max_view_list; + uint32_t allocated; + +public: + struct cntrs + { + uint64_t err_alloc; + uint64_t alien_rec; + uint64_t oldest; + } counters; + + ArrayListFixedSize() + { + counters = { 0 }; + cnt = 0; + max_view_list = 0; + worker_id = 0; + head = -1; + tail = -1; + } + + ~ArrayListFixedSize() + { +#ifdef DPDK_MEMALLOC + rte_free(p_nodes_arr); +#else + delete p_nodes_arr; +#endif + } + + inline uint32_t getAllocated() + { + return allocated; + } + + inline int init(u_int32_t n_entries, int worker_id_, en_alfs_type_t alfs_type_, uint64_t lifetime_, u_int32_t max_view_list_) + { + alfs_type = alfs_type_; + max_view_list = max_view_list_; + worker_id = worker_id_; + lifetime = lifetime_; +#ifdef DPDK_MEMALLOC + p_nodes_arr = (ArrayListFixedSize::node *)rte_zmalloc(NULL, sizeof(struct node) * n_entries, RTE_CACHE_LINE_SIZE); +#else + p_nodes_arr = new (std::nothrow) node[n_entries]; +#endif + if(p_nodes_arr != nullptr) + { + memset(p_nodes_arr, 0, sizeof(node) * n_entries); + allocated = n_entries; + return 0; + } + return -1; + } + + inline void moveBackRec(uint32_t rec_id) + { + if(rec_id != tail) + { + node *need_node = &p_nodes_arr[rec_id]; + if(rec_id == head) + { + uint32_t next_idx = need_node->next; + head = next_idx; + need_node->next = -1; + p_nodes_arr[next_idx].prev = -1; + need_node->prev = tail; + p_nodes_arr[tail].next = rec_id; + } else { + p_nodes_arr[need_node->prev].next = need_node->next; + p_nodes_arr[need_node->next].prev = need_node->prev; + need_node->prev = tail; + p_nodes_arr[tail].next = rec_id; + need_node->next = -1; + } + tail = rec_id; + } + } + + // передвигает записи с номером idx_alfs в конец массива. обновляется всемя последней модификации записи... + inline void moveBack(T *r, uint64_t timestamp, ArrayListFixedSize *other_alfs) + { + uint32_t idx_alfs = r->cmn.idx_alfs; + r->cmn.last_timestamp = timestamp; + moveBackRec(idx_alfs); + other_alfs->moveBackRec(idx_alfs); + } + + inline T *getOldestMoveBack(uint64_t timestamp, ArrayListFixedSize *p_bind_alfs_) + { + if(tail == -1 || max_view_list == 0) + return nullptr; + node *oldest_node = &p_nodes_arr[head]; // at the head always oldest node... + T *result = oldest_node->p_flow; + uint32_t next_node_idx = oldest_node->next; + if(oldest_node->p_flow->cmn.alfs_type == alfs_type) + { + if(lifetime <= (timestamp - result->cmn.last_timestamp)) // expired + { + moveBack(result, timestamp, p_bind_alfs_); + counters.oldest++; + return result; + } + return nullptr; + } + uint64_t old_aliens = counters.alien_rec; + while(next_node_idx != -1 && max_view_list > (counters.alien_rec - old_aliens)) + { + counters.alien_rec++; + moveBackRec(result->cmn.idx_alfs); + result = p_nodes_arr[next_node_idx].p_flow; + next_node_idx = p_nodes_arr[next_node_idx].next; + if(result->cmn.alfs_type == alfs_type) + { + if(lifetime <= (timestamp - result->cmn.last_timestamp)) // expired + { + moveBack(result, timestamp, p_bind_alfs_); + counters.oldest++; + return result; + } + return nullptr; + } + } + return nullptr; + } + + // добавляет запись в конец массива, т.к. она самая свежая + inline void add_rec(T *p_) + { + p_->cmn.idx_alfs = cnt; + node *nodes = p_nodes_arr; + nodes[cnt].next = -1; + nodes[cnt].p_flow = p_; + if(unlikely(head == -1)) // first element + { + head = cnt; + nodes[cnt].prev = -1; + nodes[cnt].next = -1; + } else { + nodes[tail].next = cnt; + nodes[cnt].prev = tail; + } + tail = cnt; + cnt++; + } + + inline bool can_add_rec() + { + if(cnt < allocated) + return true; + counters.err_alloc++; + return false; + } +}; diff --git a/include/cfg.h b/include/cfg.h new file mode 100644 index 0000000..226069b --- /dev/null +++ b/include/cfg.h @@ -0,0 +1,41 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#pragma once + +#include "params.h" + +// maximum active threads +#define MAX_WORKER_THREADS 10 + +#define MAX_REDIRECT_URL_SIZE 1189 + + +const char r_line1[] = "HTTP/1.1 302 Moved Temporarily\r\n"; +const char r_line2[] = "Location: "; +const char r_line3[] = "\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; +const char f_lines[] = "HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n"; + +const char uri_p[] = "uri=http%3A%2F%2F"; + +#define OUR_REDIR_SIZE (sizeof(r_line1) + sizeof(r_line2) + sizeof(r_line3) - 3) +#define OUR_PAYLOAD_SIZE 1400 + +extern const global_params_t *global_prm; +extern worker_params_t worker_params[MAX_WORKER_THREADS]; diff --git a/include/cmdlinetask.h b/include/cmdlinetask.h index 8441e69..9acbadc 100644 --- a/include/cmdlinetask.h +++ b/include/cmdlinetask.h @@ -1,3 +1,21 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ #pragma once #include diff --git a/include/dpdk.h b/include/dpdk.h index 3b228e6..5eba305 100644 --- a/include/dpdk.h +++ b/include/dpdk.h @@ -1,3 +1,22 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #pragma once #include diff --git a/include/dpi.h b/include/dpi.h deleted file mode 100644 index 6081cc8..0000000 --- a/include/dpi.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include - -struct dpi_flow_info -{ - char *url; - uint16_t url_size; - bool use_pool; - struct rte_mempool *mempool; - struct rte_mempool *dpi_mempool; - void free_mem() - { - if(!use_pool) - { - if(url) - free(url); - } else { - rte_mempool_put(mempool, url); - } - } -}; - diff --git a/include/dtypes.h b/include/dtypes.h index 727e190..07b6bf9 100644 --- a/include/dtypes.h +++ b/include/dtypes.h @@ -1,3 +1,22 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #pragma once #include diff --git a/include/flow.h b/include/flow.h index 4aa8f82..0624e6b 100644 --- a/include/flow.h +++ b/include/flow.h @@ -1,3 +1,21 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ #pragma once @@ -7,6 +25,8 @@ #include #include #include +#include + #include #include #include @@ -14,36 +34,40 @@ #include #include +#include "params.h" +#include "arr.h" + //#define _SIMPLE_HASH 1 extern "C" void dpi_reordering_tcp_delete_all_fragments(dpi_tracking_informations_t *victim); +void initFlowStorages(); + #define IPV6_ADDR_LEN 16 -struct ext_dpi_flow_info +struct flow_common_data_t +{ + uint64_t last_timestamp; + uint8_t last_worker_id; + uint8_t owner_worker_id; + en_alfs_type_t alfs_type; + uint32_t idx_alfs; + int32_t hash_idx; + bool blocked; +}; + +struct flow_base_t { - u_int16_t srcport; u_int16_t dstport; u_int8_t l4prot; - - union src_addr{ /** Addresses mantained in network byte order. **/ - struct in6_addr ipv6_srcaddr; - u_int32_t ipv4_srcaddr; - } src_addr_t; - union dst_addr{ - struct in6_addr ipv6_dstaddr; - u_int32_t ipv4_dstaddr; - } dst_addr_t; - dpi_flow_infos_t infos; uint64_t last_timestamp; -// u_int64_t bytes; -// u_int32_t packets; + flow_common_data_t cmn; inline void free_mem(dpi_flow_cleaner_callback* flow_cleaner_callback) { - if(flow_cleaner_callback != nullptr) + if(flow_cleaner_callback != nullptr && infos.tracking.flow_specific_user_data != nullptr) (*(flow_cleaner_callback))(infos.tracking.flow_specific_user_data); if(infos.tracking.http_informations[0].temp_buffer != nullptr) free(infos.tracking.http_informations[0].temp_buffer); @@ -53,10 +77,60 @@ struct ext_dpi_flow_info free(infos.tracking.ssl_information[0].pkt_buffer); if(infos.tracking.ssl_information[1].pkt_buffer != nullptr) free(infos.tracking.ssl_information[1].pkt_buffer); + infos.tracking.flow_specific_user_data = nullptr; + infos.tracking.http_informations[0].temp_buffer = nullptr; + infos.tracking.http_informations[1].temp_buffer = nullptr; + infos.tracking.ssl_information[0].pkt_buffer = nullptr; + infos.tracking.ssl_information[1].pkt_buffer = nullptr; dpi_reordering_tcp_delete_all_fragments(&(infos.tracking)); + infos.tracking.segments[0] = nullptr; + infos.tracking.segments[1] = nullptr; } + + inline void init_b(uint64_t tm, uint8_t owner_worker_id, uint8_t worker_id, uint32_t idx_alfs, int32_t hash_idx) + { + memset(this, 0, sizeof(struct flow_base_t)); + cmn.last_timestamp = tm; + cmn.last_worker_id = worker_id; + cmn.owner_worker_id = owner_worker_id; + cmn.idx_alfs = idx_alfs; + cmn.hash_idx = hash_idx; + } + }; +struct ext_dpi_flow_info_ipv4 : flow_base_t +{ + union src_addr{ /** Addresses mantained in network byte order. **/ + u_int32_t ipv4_srcaddr; + } src_addr_t; + union dst_addr{ + u_int32_t ipv4_dstaddr; + } dst_addr_t; + + inline void init(uint64_t tm, uint8_t owner_worker_id, uint8_t worker_id, uint32_t idx_alfs, int32_t hash_idx) + { + init_b(tm, owner_worker_id, worker_id, idx_alfs, hash_idx); +// src_addr_t.ipv4_srcaddr = 0; +// dst_addr_t.ipv4_dstaddr = 0; + } +} __rte_cache_aligned; + +struct ext_dpi_flow_info_ipv6 : flow_base_t +{ + union src_addr{ /** Addresses mantained in network byte order. **/ + struct in6_addr ipv6_srcaddr; + } src_addr_t; + union dst_addr{ + struct in6_addr ipv6_dstaddr; + } dst_addr_t; + + inline void init(uint64_t tm, uint8_t owner_worker_id, uint8_t worker_id, uint32_t idx_alfs, int32_t hash_idx) + { + init_b(tm, owner_worker_id, worker_id, idx_alfs, hash_idx); + } + +} __rte_cache_aligned; union ipv4_5tuple_host { struct { @@ -98,17 +172,7 @@ struct packet_info uint8_t *l3; uint64_t timestamp; // packet timestamp (ticks) uint32_t acl_res; // result of acl_classify -}; - -struct ip_5tuple -{ - uint8_t ip_dst[IPV6_ADDR_LEN]; - uint8_t ip_src[IPV6_ADDR_LEN]; - uint16_t port_dst; - uint16_t port_src; - uint8_t proto; -} __attribute__((__packed__)); - +} __rte_cache_aligned; #ifdef __SIMPLE_HASH static inline uint32_t ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len, uint32_t init_val) @@ -181,35 +245,230 @@ static inline uint32_t ipv6_hash_crc(const void *data, __rte_unused uint32_t dat #endif -/// rte_hash holder -class flowHash +/// base class + +class FlowStorage +{ +public: + struct cntrs + { + uint64_t alloc; + uint64_t free; + uint64_t reuse; // количество повторно использованных записей + } counters; + + rte_mempool *mempool; + rte_hash *hash; + FlowStorage(rte_mempool *mempool_) + { + counters = { 0 }; + mempool = mempool_; + hash = nullptr; + } + ~FlowStorage() + { + } + // initialize arrays + virtual int init(flow_storage_params_t *prm) = 0; + +}; + +class __rte_cache_aligned FlowStorageIPV4 : public FlowStorage { +public: + FlowStorageIPV4(flow_storage_params_t *prm); + int init(flow_storage_params_t *prm); + ~FlowStorageIPV4(); + + inline ext_dpi_flow_info_ipv4 *searchFlow(uint8_t *key, uint32_t sig, dpi_pkt_infos_t *pkt_infos, int32_t *idx) + { + int32_t ret = rte_hash_lookup_with_hash(hash, key, sig); + if(ret >= 0) + { + *idx = ret; + return data[ret]; + } + return nullptr; + } + + inline int removeFlow(int32_t idx) + { + void *key_ptr; + if(unlikely(data[idx] == nullptr)) + { + _logger.error("Data in the ipv4 hash at pos %d is null!", (int) idx); + return -1; + } + int fr=rte_hash_get_key_with_position(hash, idx, &key_ptr); + if(unlikely(fr < 0)) + { + _logger.error("Key not found in the ipv4 hash for the position %d", (int) idx); + return -1; + } else { + int32_t delr=rte_hash_del_key(hash, key_ptr); + if(unlikely(delr < 0)) + { + _logger.error("Error (%d) occured while delete data from the ipv4 flow hash table", (int)delr); + return -1; + } + data[idx] = nullptr; + } + return 0; + } + + inline int reuseFlow(uint8_t *key, uint32_t sig, ext_dpi_flow_info_ipv4 *node) + { + int32_t ret = rte_hash_add_key_with_hash(hash, key, sig); + if(unlikely(ret == -EINVAL)) + { + _logger.fatal("Bad parameters in hash add"); + return -1; + } + if(unlikely(ret == -ENOSPC)) + { + _logger.fatal("There is no space in the ipv4 flow hash"); + return -1; + } + node->cmn.hash_idx = ret; + data[ret] = node; + counters.reuse++; + return 0; + } + + inline ext_dpi_flow_info_ipv4 *newFlow() + { + struct ext_dpi_flow_info_ipv4 *newflow; + if(unlikely(rte_mempool_get(mempool, (void **)&newflow) != 0)) + { + return nullptr; + } + counters.alloc++; + return newflow; + } + + inline int addFlow(uint8_t *key, uint32_t sig, ext_dpi_flow_info_ipv4 *node) + { + int32_t ret = rte_hash_add_key_with_hash(hash, key, sig); + if(unlikely(ret == -EINVAL)) + { + _logger.fatal("Bad parameters in hash add"); + return -1; + } + if(unlikely(ret == -ENOSPC)) + { + _logger.fatal("There is no space in the ipv4 flow hash"); + return -1; + } + node->cmn.hash_idx = ret; + data[ret] = node; + return 0; + } + + ext_dpi_flow_info_ipv4 **data; + ArrayListFixedSize short_alfs; + ArrayListFixedSize long_alfs; private: Poco::Logger& _logger; - struct rte_hash *ipv4_FlowHash; - struct rte_hash *ipv6_FlowHash; - uint32_t _flowHashSizeIPv4; - uint32_t _flowHashSizeIPv6; +}; + + +class __rte_cache_aligned FlowStorageIPV6 : public FlowStorage +{ public: - flowHash(int socket_id, int thread_id, uint32_t flowHashSizeIPv4, uint32_t flowHashSizeIPv6); - ~flowHash(); - inline struct rte_hash *getIPv4Hash() + FlowStorageIPV6(flow_storage_params_t *prm); + int init(flow_storage_params_t *prm); + ~FlowStorageIPV6(); + + inline ext_dpi_flow_info_ipv6 *searchFlow(uint8_t *key, uint32_t sig, dpi_pkt_infos_t *pkt_infos, int32_t *idx) { - return ipv4_FlowHash; + int32_t ret = rte_hash_lookup_with_hash(hash, key, sig); + if(unlikely(ret >= 0)) + { + *idx = ret; + return data[ret]; + } + return nullptr; } - inline struct rte_hash *getIPv6Hash() + + inline int removeFlow(int32_t idx) { - return ipv6_FlowHash; + void *key_ptr; + if(unlikely(data[idx] == nullptr)) + { + _logger.error("Data in the ipv6 hash at pos %d is null!", (int) idx); + return -1; + } + int fr=rte_hash_get_key_with_position(hash, idx, &key_ptr); + if(unlikely(fr < 0)) + { + _logger.error("Key not found in the ipv6 hash for the position %d", (int) idx); + return -1; + } else { + int32_t delr=rte_hash_del_key(hash, key_ptr); + if(unlikely(delr < 0)) + { + _logger.error("Error (%d) occured while delete data from the ipv6 flow hash table", (int)delr); + return -1; + } + data[idx] = nullptr; + } + return 0; } - inline uint32_t getHashSizeIPv4() + + inline int reuseFlow(uint8_t *key, uint32_t sig, ext_dpi_flow_info_ipv6 *node) { - return _flowHashSizeIPv4; + int32_t ret = rte_hash_add_key_with_hash(hash, key, sig); + if(unlikely(ret == -EINVAL)) + { + _logger.fatal("Bad parameters in hash add"); + return -1; + } + if(unlikely(ret == -ENOSPC)) + { + _logger.fatal("There is no space in the ipv6 flow hash"); + return -1; + } + node->cmn.hash_idx = ret; + data[ret] = node; + counters.reuse++; + return 0; } - inline uint32_t getHashSizeIPv6() + + inline ext_dpi_flow_info_ipv6 *newFlow() { - return _flowHashSizeIPv6; + struct ext_dpi_flow_info_ipv6 *newflow; + if(unlikely(rte_mempool_get(mempool, (void **)&newflow) != 0)) + { + return nullptr; + } + counters.alloc++; + return newflow; } + inline int addFlow(uint8_t *key, uint32_t sig, ext_dpi_flow_info_ipv6 *node) + { + int32_t ret = rte_hash_add_key_with_hash(hash, key, sig); + if(unlikely(ret == -EINVAL)) + { + _logger.fatal("Bad parameters in hash add"); + return -1; + } + if(unlikely(ret == -ENOSPC)) + { + _logger.fatal("There is no space in the ipv6 flow hash"); + return -1; + } + node->cmn.hash_idx = ret; + data[ret] = node; + return 0; + } + + ext_dpi_flow_info_ipv6 **data; + ArrayListFixedSize short_alfs; + ArrayListFixedSize long_alfs; +private: + Poco::Logger& _logger; + }; diff --git a/include/http.h b/include/http.h new file mode 100644 index 0000000..81ce302 --- /dev/null +++ b/include/http.h @@ -0,0 +1,70 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#pragma once + +#include +#include + +namespace http { + +enum header_state +{ + hstate_nothing = 0, + hstate_host +}; + +struct http_rec +{ + char *buf; // указатель на буфер + uint16_t buf_size; // размер буфера. + uint16_t length; // длина того, что находится в буфере. +}; + +struct http_req_buf +{ + char url_buf[600]; + char host[255]; + struct http_rec uri; + struct http_rec host_r; + header_state h_state; + char h_prev_char; + struct rte_mempool *mempool; + inline void init() + { + uri.buf = &url_buf[0]; + uri.buf_size = sizeof(url_buf); + uri.length = 0; + url_buf[0] = 0; + host[0] = 0; + host_r.buf_size = sizeof(host); + host_r.buf = &host[0]; + host_r.length = 0; + h_state = hstate_nothing; + mempool = nullptr; + } +} __rte_cache_aligned; + +int on_url_ext (http_parser *p, const char* at, size_t length, dpi_pkt_infos_t* pkt_informations, void** flow_specific_user_data, void* user_data); + +int on_header_field_ext(http_parser *p, const char *at, size_t length, dpi_pkt_infos_t* pkt_informations, void** flow_specific_user_data, void* user_data); + +int on_header_value_ext(http_parser *p, const char *at, size_t length, dpi_pkt_infos_t* pkt_informations, void** flow_specific_user_data, void* user_data); + +} diff --git a/include/main.h b/include/main.h index b800784..b174ea1 100644 --- a/include/main.h +++ b/include/main.h @@ -1,13 +1,32 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #pragma once #include #include #include #include -#include "dtypes.h" #include "sender.h" #include "worker.h" #include "notification.h" +#include "tries.h" #define DEFAULT_MBUF_POOL_SIZE 8191 #define MAX_RX_QUEUE_PER_LCORE 16 @@ -69,20 +88,13 @@ class extFilter: public Poco::Util::ServerApplication void handleHelp(const std::string& name,const std::string& value); void displayHelp(); + void initParams(); + void initFlowsIPv4(); + /// Print DPDK ports void printDPDKPorts(const std::string& name,const std::string& value); int main(const ArgVec& args); - /** - Load domains for blocking. - **/ - void loadDomains(std::string &fn, AhoCorasickPlus *_dm_atm); - - /** - Load domains and urls into one database. - **/ - void loadDomainsURLs(std::string &domains, std::string &urls, AhoCorasickPlus *dm_atm); - std::string &getSSLFile() { return _sslFile; @@ -177,6 +189,11 @@ class extFilter: public Poco::Util::ServerApplication return ne; } + inline TriesManager *getTriesManager() + { + return &_tries; + } + static rte_mempool *packet_info_pool[NB_SOCKETS]; static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; @@ -203,27 +220,13 @@ class extFilter: public Poco::Util::ServerApplication std::string _protocolsFile; std::string _statisticsFile; - bool _lower_host; - bool _match_url_exactly; bool _block_ssl_no_sni; - bool _http_redirect; - bool _url_normalization; - bool _remove_dot; int _statistic_interval; - enum ADD_P_TYPES _add_p_type; + struct CSender::params _sender_params; static uint64_t _tsc_hz; - // DPI - uint32_t _dpi_max_active_flows_ipv4; - uint32_t _dpi_max_active_flows_ipv6; - bool _dpi_fragmentation_ipv6_state; - bool _dpi_fragmentation_ipv4_state; - uint16_t _dpi_fragmentation_ipv4_table_size; - uint16_t _dpi_fragmentation_ipv6_table_size; - bool _dpi_tcp_reordering; - uint16_t _dpi_maximum_url_size; int _num_of_senders; @@ -251,6 +254,7 @@ class extFilter: public Poco::Util::ServerApplication int _cmdline_port; Poco::Net::IPAddress _cmdline_ip; uint8_t _dpdk_send_port; + TriesManager _tries; }; diff --git a/include/notification.h b/include/notification.h index bb5afb6..2dacb7c 100644 --- a/include/notification.h +++ b/include/notification.h @@ -1,3 +1,22 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #pragma once #include diff --git a/include/params.h b/include/params.h new file mode 100644 index 0000000..faaad09 --- /dev/null +++ b/include/params.h @@ -0,0 +1,88 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#pragma once + +class FlowStorage; +struct rte_mempool; +class NotifyManager; + +struct flow_storage_params_t +{ + int worker_id; + const uint64_t *p_lifetime; + rte_mempool *mempool; + int part_no; + uint32_t recs_number; +}; + +struct memory_config_t +{ + uint8_t parts_of_flow; // количество частей, на которое разбивается таблица (для каждого worker'a) + uint32_t mask_parts_flow; // маска для выбора части = flow_hash & mask_parts_flow + uint32_t flows_number; // общее количество flows в системе + uint32_t recs_number; // количество элементов в хэше и массиве для каждого worker'а +}; + +struct memory_configs_t +{ + memory_config_t ipv4; + memory_config_t ipv6; + uint32_t http_entries; +}; + + +struct fragmentation_config_t +{ + bool state; + int table_size; +}; + +struct fragmentation_configs_t +{ + fragmentation_config_t ipv4; + fragmentation_config_t ipv6; +}; + +struct global_params_t +{ + memory_configs_t memory_configs; + fragmentation_configs_t frag_configs; + bool tcp_reordering; + uint8_t workers_number; + uint64_t flow_lifetime[2]; // [0] для flows, который завершены или установлены без данных, [1] для flows с данными + uint8_t answer_duplication; +}; + +struct flow_storage_t +{ + FlowStorage **flows; // количество из memory_configs.[ipv4|ipv6].parts_of_flow +}; + +struct worker_params_t +{ + flow_storage_t flows_ipv4; + flow_storage_t flows_ipv6; + bool notify_enabled; + NotifyManager *nm; + uint8_t sender_port; + uint16_t tx_queue_id; +} __rte_cache_aligned; + + diff --git a/include/sender.h b/include/sender.h index 49a7852..73878a3 100644 --- a/include/sender.h +++ b/include/sender.h @@ -19,6 +19,8 @@ #ifndef __SENDER_H #define __SENDER_H +#define __STDC_FORMAT_MACROS +#include #include #include #include @@ -31,10 +33,15 @@ #include #include #include +#include #include #include +#include +#include +#include #include #include +#include "cfg.h" class BSender { @@ -45,23 +52,238 @@ class BSender std::string code; bool send_rst_to_server; int ttl; - int ip6_hops; + int ipv6_hops; int mtu; - params() : code("302 Moved Temporarily"), send_rst_to_server(false), ttl(250), ip6_hops(250), mtu(1500) { } + params() : code("302 Moved Temporarily"), send_rst_to_server(false), ttl(250), ipv6_hops(250), mtu(1500) { } }; BSender(const char *, struct params &prm); virtual ~BSender(); - void Redirect(int user_port, int dst_port, void *ip_from, void *ip_to, int ip_ver, uint32_t acknum, uint32_t seqnum, int f_psh, const char *add_prm); - virtual void sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, std::string &dt, int f_reset, int f_psh); + void HTTPRedirect(int user_port, int dst_port, void *ip_from, void *ip_to, int ip_ver, uint32_t acknum, uint32_t seqnum, int f_psh, const char *redir_url, size_t p_len); + void HTTPForbidden(int user_port, int dst_port, void *ip_from, void *ip_to, int ip_ver, uint32_t acknum, uint32_t seqnum, int f_psh); + virtual void sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, int f_reset, int f_psh); void SendRST(int user_port, int dst_port, void *ip_from, void *ip_to, int ip_ver, uint32_t acknum, uint32_t seqnum, int f_psh); - int makePacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, std::string &dt, int f_reset, int f_psh, uint8_t *buffer); + + inline int makePacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, int f_reset, int f_psh, uint8_t *buffer) + { + int pkt_len; + pkt_id++; + + // IP header + struct iphdr *iph = (struct iphdr *) buffer; + struct ip6_hdr *iph6 = (struct ip6_hdr *) buffer; + + // TCP header + struct tcphdr *tcph = (struct tcphdr *) (buffer + (ip_ver == 4 ? sizeof(struct iphdr) : sizeof(struct ip6_hdr))); + + // Data part + uint8_t *data = (uint8_t *)tcph + sizeof(struct tcphdr); + + if(dt_buf != nullptr && dt_len != 0) + rte_memcpy(data, dt_buf, dt_len); + + if(_logger.getLevel() == Poco::Message::PRIO_DEBUG) + { + Poco::Net::IPAddress ipa(ip_to, ip_ver == 4 ? sizeof(in_addr) : sizeof(in6_addr)); + _logger.debug("Trying to send packet to %s port %d", ipa.toString(), port_to); + } + if(ip_ver == 4) + { + // Fill the IPv4 header + iph->ihl = 5; + iph->version = 4; + iph->tos=0; + iph->tot_len = rte_cpu_to_be_16(sizeof(struct iphdr) + sizeof(struct tcphdr) + dt_len); + iph->id = rte_cpu_to_be_16(pkt_id); + iph->frag_off = 0; + iph->ttl = _parameters.ttl; + iph->protocol = IPPROTO_TCP; + iph->check = 0; + iph->saddr = ((in_addr *)ip_from)->s_addr; + iph->daddr = ((in_addr *)ip_to)->s_addr;; + pkt_len = sizeof(struct iphdr) + sizeof(struct tcphdr) + dt_len; + } else { + // IPv6 version (4 bits), Traffic class (8 bits), Flow label (20 bits) + iph6->ip6_flow = htonl ((6 << 28) | (0 << 20) | 0); + // Payload length (16 bits): TCP header + TCP data + iph6->ip6_plen = rte_cpu_to_be_16 (sizeof(struct tcphdr) + dt_len); + // Next header (8 bits): 6 for TCP + iph6->ip6_nxt = IPPROTO_TCP; + // Hop limit (8 bits): default to maximum value + iph6->ip6_hops = 250; + rte_mov16((uint8_t *)&iph6->ip6_src, (uint8_t *)ip_from); + rte_mov16((uint8_t *)&iph6->ip6_dst, (uint8_t *)ip_to); + pkt_len = (sizeof(struct ip6_hdr) + sizeof(struct tcphdr) + dt_len); + } + + // TCP Header + tcph->source = port_from; + tcph->dest = port_to; + tcph->seq = acknum; + tcph->doff = 5; + tcph->syn = 0; + tcph->res1 = 0; + tcph->res2 = 0; + tcph->rst = f_reset; + tcph->psh = f_psh; + if(f_reset) + { + tcph->ack = 0; + tcph->ack_seq = 0; + tcph->fin = 0; + tcph->window = rte_cpu_to_be_16(0xEF); + } else { + tcph->ack_seq = seqnum; + tcph->ack = 1; + tcph->fin = 1; + tcph->window = rte_cpu_to_be_16(5885); // TODO get from original packet... + } + tcph->urg = 0; + tcph->check = 0; + tcph->urg_ptr = 0; + + if(ip_ver == 4) + tcph->check = rte_ipv4_udptcp_cksum((const ipv4_hdr*)iph, tcph); + else + tcph->check = rte_ipv6_udptcp_cksum((const ipv6_hdr*)iph6, tcph); + return pkt_len; + } + + inline int makeSwapPacketIPv4(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, bool f_reset, bool f_psh, uint8_t *buffer) + { + int pkt_len; + + struct ipv4_hdr *ipv4_header = (struct ipv4_hdr *)pkt; + struct tcphdr *tcph_orig = (struct tcphdr *)(pkt + sizeof(struct ipv4_hdr)); + + // IP header + struct iphdr *iph = (struct iphdr *) buffer; + + // TCP header + struct tcphdr *tcph = (struct tcphdr *) (buffer + sizeof(struct iphdr)); + + // Data part + uint8_t *data = (uint8_t *)tcph + sizeof(struct tcphdr); + + if(dt_buf != nullptr && dt_len != 0) + rte_memcpy(data, dt_buf, dt_len); + + if(_logger.getLevel() == Poco::Message::PRIO_DEBUG) + { + Poco::Net::IPAddress ipa(&ipv4_header->src_addr, sizeof(in_addr) ); + _logger.debug("Trying to send packet to %s port %d", ipa.toString(), (int) rte_be_to_cpu_16(tcph_orig->source)); + } + // Fill the IPv4 header + iph->ihl = 5; + iph->version = 4; + iph->tos = 0; + iph->tot_len = rte_cpu_to_be_16(sizeof(struct iphdr) + sizeof(struct tcphdr) + dt_len); + iph->id = rte_cpu_to_be_16(pkt_id++); + iph->frag_off = 0; + iph->ttl = _parameters.ttl; + iph->protocol = IPPROTO_TCP; + iph->check = 0; + iph->saddr = ipv4_header->dst_addr; + iph->daddr = ipv4_header->src_addr; + pkt_len = sizeof(struct iphdr) + sizeof(struct tcphdr) + dt_len; + + // TCP Header + tcph->source = tcph_orig->dest; + tcph->dest = tcph_orig->source; + tcph->seq = acknum; + tcph->doff = 5; + tcph->syn = 0; + tcph->res1 = 0; + tcph->res2 = 0; + tcph->rst = f_reset; + tcph->psh = f_psh; + if(f_reset) + { + tcph->ack = 0; + tcph->ack_seq = 0; + tcph->fin = 0; + tcph->window = 0; + } else { + tcph->window = tcph_orig->window; + tcph->ack_seq = seqnum; + tcph->ack = 1; + tcph->fin = 0; + } + tcph->urg = 0; + tcph->check = 0; + tcph->urg_ptr = 0; + + tcph->check = rte_ipv4_udptcp_cksum((const ipv4_hdr*)iph, tcph); + return pkt_len; + } + + inline int makeSwapPacketIPv6(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, bool f_reset, bool f_psh, uint8_t *buffer) + { + int pkt_len; + + struct ipv6_hdr *ipv6_header = (struct ipv6_hdr *)pkt; + struct tcphdr *tcph_orig = (struct tcphdr *)(pkt + sizeof(struct ipv6_hdr)); + + // IP header + struct ip6_hdr *iph6 = (struct ip6_hdr *) buffer; + + // TCP header + struct tcphdr *tcph = (struct tcphdr *) (buffer + sizeof(struct ipv6_hdr)); + + // Data part + uint8_t *data = (uint8_t *)tcph + sizeof(struct tcphdr); + + if(dt_buf != nullptr && dt_len != 0) + rte_memcpy(data, dt_buf, dt_len); + + if(_logger.getLevel() == Poco::Message::PRIO_DEBUG) + { + Poco::Net::IPAddress ipa(&ipv6_header->src_addr, sizeof(in6_addr) ); + _logger.debug("Trying to send packet to %s port %d", ipa.toString(), (int) rte_be_to_cpu_16(tcph_orig->source)); + } + // IPv6 version (4 bits), Traffic class (8 bits), Flow label (20 bits) + iph6->ip6_flow = htonl ((6 << 28) | (0 << 20) | 0); + // Payload length (16 bits): TCP header + TCP data + iph6->ip6_plen = rte_cpu_to_be_16 (sizeof(struct tcphdr) + dt_len); + // Next header (8 bits): 6 for TCP + iph6->ip6_nxt = IPPROTO_TCP; + // Hop limit (8 bits): default to maximum value + iph6->ip6_hops = 250; + rte_mov16((uint8_t *)&iph6->ip6_src, (uint8_t *)&ipv6_header->dst_addr); + rte_mov16((uint8_t *)&iph6->ip6_dst, (uint8_t *)&ipv6_header->src_addr); + pkt_len = (sizeof(struct ip6_hdr) + sizeof(struct tcphdr) + dt_len); + // TCP Header + tcph->source = tcph_orig->dest; + tcph->dest = tcph_orig->source; + tcph->seq = acknum; + tcph->doff = 5; + tcph->syn = 0; + tcph->res1 = 0; + tcph->res2 = 0; + tcph->rst = f_reset; + tcph->psh = f_psh; + tcph->window = tcph_orig->window; + if(f_reset) + { + tcph->ack = 0; + tcph->ack_seq = 0; + tcph->fin = 0; + } else { + tcph->ack_seq = seqnum; + tcph->ack = 1; + tcph->fin = 1; + } + tcph->urg = 0; + tcph->check = 0; + tcph->urg_ptr = 0; + tcph->check = rte_ipv6_udptcp_cksum((const ipv6_hdr*)iph6, tcph); + return pkt_len; + } virtual int Send(uint8_t *buffer, int size, void *addr, int addr_size) = 0; Poco::Logger& _logger; - std::string rHeader; struct params _parameters; uint16_t pkt_id; }; @@ -102,11 +324,23 @@ class ESender : public BSender }; ESender(struct nparams ¶ms, uint8_t port, struct rte_mempool *mp, WorkerThread *wt); ~ESender(); - void sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, std::string &dt, int f_reset, int f_psh); + + void sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, int f_reset, int f_psh); + int Send(uint8_t *buffer, int size, void *addr, int addr_size) { return size; } + + void sendPacketIPv4(const uint8_t *l3_pkt, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, bool f_reset, bool f_psh); + void sendPacketIPv6(const uint8_t *l3_pkt, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, bool f_reset, bool f_psh); + void HTTPRedirectIPv4(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, bool f_psh, const char *redir_url, size_t r_len); + void HTTPRedirectIPv6(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, bool f_psh, const char *redir_url, size_t r_len); + void SendRSTIPv4(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum); + void SendRSTIPv6(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum); + void HTTPForbiddenIPv4(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, bool f_psh); + void HTTPForbiddenIPv6(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, bool f_psh); + private: uint8_t _port; struct ether_hdr _eth_hdr; diff --git a/include/stats.h b/include/stats.h index ee1d718..1bd8717 100644 --- a/include/stats.h +++ b/include/stats.h @@ -1,3 +1,22 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #pragma once struct LatencyCounters @@ -8,44 +27,69 @@ struct LatencyCounters uint64_t blocked_pkts; }; -class ThreadStats +struct ThreadStats { -public: - uint64_t redirected_domains; - uint64_t redirected_urls; - uint64_t sended_rst; uint64_t ip_packets; uint64_t total_bytes; - uint64_t matched_ssl; + uint64_t matched_ssl_sni; uint64_t matched_ssl_ip; uint64_t matched_ip_port; uint64_t total_packets; uint64_t analyzed_packets; - uint64_t matched_domains; - uint64_t matched_urls; uint64_t ipv4_packets; uint64_t ipv6_packets; - uint64_t ndpi_flows_count; - uint64_t ndpi_ipv4_flows_count; - uint64_t ndpi_ipv6_flows_count; - uint64_t max_ipv4_flows; - uint64_t max_ipv6_flows; - uint64_t ndpi_flows_deleted; uint64_t missed_packets; - uint64_t enqueued_packets; uint64_t ipv4_short_packets; uint64_t ipv4_fragments; uint64_t ipv6_fragments; - uint64_t already_detected_blocked; uint64_t reassembled_flows; struct LatencyCounters latency_counters; - uint64_t ndpi_flows_expired; - uint64_t dpi_use_url_malloc; uint64_t dpi_no_mempool_http; - ThreadStats() : redirected_domains(0), redirected_urls(0), sended_rst(0), ip_packets(0), total_bytes(0), matched_ssl(0), matched_ssl_ip(0), matched_ip_port(0),total_packets(0), analyzed_packets(0), matched_domains(0), matched_urls(0), ipv4_packets(0), ipv6_packets(0), ndpi_flows_count(0), ndpi_ipv4_flows_count(0), ndpi_ipv6_flows_count(0), max_ipv4_flows(0), max_ipv6_flows(0), ndpi_flows_deleted(0), missed_packets(0), enqueued_packets(0), ipv4_short_packets(0), ipv4_fragments(0), ipv6_fragments(0), already_detected_blocked(0), reassembled_flows(0), latency_counters{0,0,0,0}, ndpi_flows_expired(0), dpi_use_url_malloc(0), dpi_no_mempool_http(0) {} - void clear() { redirected_domains = 0; redirected_urls = 0; sended_rst = 0; ip_packets = 0; total_bytes = 0; matched_ssl = 0; matched_ssl_ip = 0; matched_ip_port = 0; total_packets = 0; analyzed_packets = 0; matched_domains = 0; matched_urls = 0; ipv4_packets = 0; ipv6_packets = 0; ndpi_flows_count = 0; ndpi_flows_deleted = 0; missed_packets = 0; enqueued_packets = 0; ipv4_short_packets = 0; ipv4_fragments = 0; ipv6_fragments = 0; ndpi_ipv4_flows_count = 0; ndpi_ipv6_flows_count = 0; max_ipv4_flows = 0; max_ipv6_flows = 0; already_detected_blocked = 0; reassembled_flows = 0; latency_counters = {0, 0, 0, 0}; ndpi_flows_expired = 0; dpi_use_url_malloc = 0; dpi_no_mempool_http = 0; } + uint64_t ssl_packets; + uint64_t http_packets; + + // ipv4 + uint64_t recycling_flow; + uint64_t no_create_flow; + uint64_t new_flow; + uint64_t error_alloc_flow; + uint64_t close_flow; + uint64_t alfs_fail_flow; + uint64_t reuse_flow; + uint64_t hash_add_fail_flow; + uint64_t seen_already_blocked_http_ipv4; + uint64_t seen_already_blocked_ssl_ipv4; + uint64_t sended_rst_ipv4; + uint64_t sended_forbidden_ipv4; + uint64_t matched_http_bl_ipv4; + uint64_t redirected_http_bl_ipv4; + + // ipv6 + uint64_t recycling_flow_ipv6; + uint64_t no_create_flow_ipv6; + uint64_t new_flow_ipv6; + uint64_t error_alloc_flow_ipv6; + uint64_t close_flow_ipv6; + uint64_t alfs_fail_flow_ipv6; + uint64_t reuse_flow_ipv6; + uint64_t hash_add_fail_flow_ipv6; + uint64_t seen_already_blocked_http_ipv6; + uint64_t seen_already_blocked_ssl_ipv6; + uint64_t sended_rst_ipv6; + uint64_t sended_forbidden_ipv6; + uint64_t matched_http_bl_ipv6; + uint64_t redirected_http_bl_ipv6; + + ThreadStats() + { + memset(this, 0, sizeof(ThreadStats)); + } + void clear() + { + memset(this, 0, sizeof(ThreadStats)); + } }; diff --git a/include/tcp.h b/include/tcp.h index d2119a6..5ab84c2 100644 --- a/include/tcp.h +++ b/include/tcp.h @@ -1,3 +1,22 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #pragma once namespace TCP diff --git a/include/tries.h b/include/tries.h new file mode 100644 index 0000000..92f38ac --- /dev/null +++ b/include/tries.h @@ -0,0 +1,153 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#pragma once + +#include +#include +#include "cfg.h" + +#define SEARCH_AGENTS MAX_WORKER_THREADS + + +class Tries +{ +public: + Tries() : _is_mdomains_ready(false), _is_urls_ready(false) {}; + ~Tries() {}; + bool search_prefix(marisa::Agent *agent, char *rhost, std::size_t rhost_size, char *url, std::size_t url_size); + bool lookup(marisa::Agent *agent, char *rhost, std::size_t rhost_size, char *url, std::size_t url_size); + inline bool isMDomainsReady() + { + return _is_mdomains_ready; + } + inline bool isURLsReady() + { + return _is_urls_ready; + } + void setMDomainsReady() + { + _is_mdomains_ready = true; + } + void setURLsReady() + { + _is_urls_ready = true; + } +public: + // домены, содержащие маску + marisa::Trie mdomains_trie; + // url и домены без маски + marisa::Trie urls_trie; + bool _is_mdomains_ready; + bool _is_urls_ready; +}; + +class TriesControl +{ +public: + TriesControl(); + ~TriesControl() {}; + bool load(std::string &domains, std::string &urls); + + inline Tries *getActiveTrie() + { + return &tries[active_trie]; + } + + Tries tries[2]; + uint8_t active_trie; +private: + Poco::Logger& _logger; + bool _first_load; + time_t _domains_ch_time; + time_t _urls_ch_time; + time_t load_time; +}; + + +class BlacklistsManager +{ +public: + struct bl_service_profile + { + std::string domains_file; + std::string urls_file; + std::string sni_file; + char redir_url[OUR_PAYLOAD_SIZE - OUR_REDIR_SIZE]; + size_t redir_url_size; + bool need_add_url; + }; + BlacklistsManager() : _active_profile(0) + { + _sp[0].redir_url[0] = 0; + _sp[1].redir_url[0] = 0; + _sp[0].need_add_url = false; + _sp[1].need_add_url = false; + _sp[1].redir_url_size = 0; + _sp[0].redir_url_size = 0; + } + ~BlacklistsManager() {}; + inline TriesControl *getHttpBlacklist() + { + return &_http_bl; + } + inline TriesControl *getSNIBlacklist() + { + return &_sni_bl; + } + inline bl_service_profile *getActiveSP() + { + return &_sp[_active_profile]; + } + inline void changeProfile() + { + _active_profile = _active_profile == 0 ? 1 : 0; + } + /// set active profile to 0, store file names in it, load blacklists + bool init(std::string &_domains_file, std::string &_urls_file, std::string &_sni_file, const char *redir_url = nullptr, size_t url_length = 0); + /// reload blacklists from the current active profile + bool update(); + void fillRedirURL(uint8_t profile, const char *redir_url, size_t url_length); + void fillProfile(uint8_t profile, std::string &_domains_file, std::string &_urls_file, std::string &_sni_file, const char *redir_url, size_t url_length); +private: + uint8_t _active_profile; + TriesControl _http_bl; + TriesControl _sni_bl; + struct bl_service_profile _sp[2]; +}; + +class TriesManager +{ +public: + TriesManager() : _logger(Poco::Logger::get("TriesManager")) + {}; + ~TriesManager() {}; + + int checkURLBlocked(int thread_id, const char *hostname, uint32_t host_len, const char *uri, uint32_t uri_len, char **redir_url); + int checkSNIBlocked(int thread_id, const char *sni, uint32_t sni_len); + inline BlacklistsManager *getBLManager() + { + return &_bl_manager; + } +private: + BlacklistsManager _bl_manager; + marisa::Agent _agents[SEARCH_AGENTS]; + char _url[MAX_WORKER_THREADS][4096]; + Poco::Logger& _logger; +}; diff --git a/include/utils.h b/include/utils.h new file mode 100644 index 0000000..1bb6534 --- /dev/null +++ b/include/utils.h @@ -0,0 +1,24 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#pragma once + +#include + +std::size_t url_encode(char *buf, const char *from, std::size_t len, std::size_t buf_size); diff --git a/include/worker.h b/include/worker.h index 2a7b695..31dc9aa 100644 --- a/include/worker.h +++ b/include/worker.h @@ -1,3 +1,22 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #pragma once #include @@ -8,25 +27,22 @@ #include #include #include -#include #include #include -#include "dtypes.h" -#include "AhoCorasickPlus.h" #include "flow.h" #include "stats.h" #include "dpdk.h" #include "sender.h" +#include "http.h" -#define EXTF_GC_INTERVAL 1000 // us -#define EXTF_ALL_GC_INTERVAL 1 // seconds +//#define EXTF_GC_INTERVAL 1000 // us +//#define EXTF_ALL_GC_INTERVAL 1 // seconds -#define EXT_DPI_FLOW_TABLE_MAX_IDLE_TIME 30 /** In seconds. **/ +//#define EXT_DPI_FLOW_TABLE_MAX_IDLE_TIME 30 /** In seconds. **/ #define EXTFILTER_CAPTURE_BURST_SIZE 32 #define EXTFILTER_WORKER_BURST_SIZE 32 -#define URI_RESERVATION_SIZE 4096 #define CERT_RESERVATION_SIZE 1024 /* Configure how many packets ahead to prefetch, when reading packets */ @@ -37,45 +53,16 @@ class ESender; struct WorkerConfig { - uint32_t CoreId; - - uint8_t port; - AhoCorasickPlus *atm; - AhoCorasickPlus *atm_new; - AhoCorasickPlus *atmSSLDomains; - AhoCorasickPlus *atmSSLDomains_new; - - bool match_url_exactly; - bool lower_host; bool block_ssl_no_sni; - bool http_redirect; - enum ADD_P_TYPES add_p_type; - - bool url_normalization; - bool remove_dot; - bool notify_enabled; NotifyManager *nm; uint8_t sender_port; uint16_t tx_queue_id; - uint16_t maximum_url_size; - WorkerConfig() { - CoreId = RTE_MAX_LCORE+1; - atm = NULL; - atmSSLDomains = NULL; - atm_new = NULL; - atmSSLDomains_new = NULL; - match_url_exactly = false; - lower_host = false; block_ssl_no_sni = false; - http_redirect = true; - add_p_type = A_TYPE_NONE; - url_normalization = true; - remove_dot = true; notify_enabled = false; nm = nullptr; } @@ -95,49 +82,33 @@ class WorkerThread : public DpdkWorkerThread dpi_library_state_t *dpi_state; - std::string uri; - std::string certificate; - bool analyzePacket(struct rte_mbuf* mBuf, uint64_t timestamp); - ext_dpi_flow_info *getFlow(uint8_t *host_key, uint64_t timestamp, int32_t *idx, uint32_t sig, dpi_pkt_infos_t *pkt_infos); + +// bool analyzePacketIPv4(struct rte_mbuf* mBuf, uint64_t timestamp); + dpi_identification_result_t getAppProtocol(uint8_t *host_key, uint64_t timestamp, uint32_t sig, dpi_pkt_infos_t *pkt_infos); dpi_identification_result_t identifyAppProtocol(const unsigned char* pkt, u_int32_t length, u_int32_t current_time, uint8_t *host_key, uint32_t sig); bool checkSSL(); std::string _name; bool _need_block; - uint16_t _partition_id; - - struct ext_dpi_flow_info **ipv4_flows; - struct ext_dpi_flow_info **ipv6_flows; - struct rte_mempool *flows_pool; - - flowHash *m_FlowHash; /// for sender through dpdk int _n_send_pkts; struct rte_mbuf* _sender_buf[EXTFILTER_WORKER_BURST_SIZE]; ESender *_snd; - struct rte_mempool *_url_mempool; - struct rte_mempool *_dpi_mempool; - Poco::URI *uri_p; + struct rte_mempool *_dpi_http_mempool; + + uint8_t _worker_id; + uint32_t ipv4_flow_mask; + uint32_t ipv6_flow_mask; public: - WorkerThread(const std::string& name, WorkerConfig &workerConfig, dpi_library_state_t* state, int socketid, flowHash *fh, struct ESender::nparams &sp, struct rte_mempool *mp, struct rte_mempool *url_mempool, struct rte_mempool *dpi_mempool); + WorkerThread(uint8_t worker_id, const std::string& name, WorkerConfig &workerConfig, dpi_library_state_t* state, int socketid, struct ESender::nparams &sp, struct rte_mempool *mp, struct rte_mempool *dpi_http_mempool); ~WorkerThread(); - bool checkHTTP(std::string &uri, dpi_pkt_infos_t *pkt); - bool checkSSL(std::string &certificate, dpi_pkt_infos_t *pkt); - - inline std::string &getUri() - { - return uri; - } - - inline std::string &getCert() - { - return certificate; - } + bool checkURLBlocked(const char *host, size_t host_len, const char *uri, size_t uri_len, dpi_pkt_infos_t* pkt); + bool checkSNIBlocked(const char *sni, size_t sni_len, dpi_pkt_infos_t* pkt); inline void setNeedBlock(bool b) { @@ -178,14 +149,22 @@ class WorkerThread : public DpdkWorkerThread m_ThreadStats.clear(); } - inline struct rte_mempool *getUrlMempool() + inline struct http::http_req_buf *allocateHTTPBuf() { - return _url_mempool; + struct http::http_req_buf *res; + if(rte_mempool_get(_dpi_http_mempool, (void **)&res) != 0) + { + _logger.error("Unable to allocate memory for the http buffer"); + return nullptr; + } + res->init(); + res->mempool = _dpi_http_mempool; + return res; } - inline struct rte_mempool *getDPIMempool() + + inline uint8_t getWorkerID() { - return _dpi_mempool; + return _worker_id; } - }; diff --git a/src/Makefile.am b/src/Makefile.am index 507d921..e2007ef 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,10 +1,10 @@ -AM_CPPFLAGS = -I$(DPDK_INCLUDE) -I$(top_srcdir)/$(PEAFOWL_INCLUDE) +AM_CPPFLAGS = -I$(DPDK_INCLUDE) -I$(top_srcdir)/$(PEAFOWL_INCLUDE) -I$(MARISA_INCLUDE) -LDADD =-lpcap -L $(DPDK_LIB) -lrt -lm -ldl $(top_srcdir)/$(PEAFOWL_LIB) +LDADD =-lpcap -L $(DPDK_LIB) -lrt -lm -ldl $(top_srcdir)/$(PEAFOWL_LIB) $(MARISA_LIB) bin_PROGRAMS = extFilter extFilter_LDFLAGS = -Wl,--whole-archive -lrte_acl -lrte_pmd_bond -lrte_pmd_vmxnet3_uio -lrte_pmd_virtio -lrte_pmd_enic -lrte_pmd_i40e -lrte_pmd_fm10k -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -lrte_pmd_af_packet -lrte_ethdev -lrte_eal -lrte_mbuf -lrte_mempool -lrte_mempool_ring -lrte_mempool_stack -lrte_ring -lrte_kvargs -lrte_hash -lrte_cmdline -lrte_net -Wl,--no-whole-archive -extFilter_SOURCES = main.cpp worker.cpp AhoCorasickPlus.cpp ahocorasick.cpp node.cpp mpool.cpp replace.cpp sender.cpp sendertask.cpp statistictask.cpp reloadtask.cpp flow.cpp acl.cpp cmdlinetask.cpp notification.cpp +extFilter_SOURCES = main.cpp worker.cpp sender.cpp statistictask.cpp reloadtask.cpp flow.cpp acl.cpp cmdlinetask.cpp notification.cpp tries.cpp utils.cpp http.cpp diff --git a/src/acl.cpp b/src/acl.cpp index 78e6fad..d2410c8 100644 --- a/src/acl.cpp +++ b/src/acl.cpp @@ -1,3 +1,24 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#define __STDC_FORMAT_MACROS +#include #include #include "acl.h" @@ -61,7 +82,7 @@ rte_acl_ctx* ACL::_setup_acl(struct rte_acl_rule* acl_base, unsigned int acl_num acl_build_param.num_categories = DEFAULT_MAX_CATEGORIES; acl_build_param.num_fields = dim; - memcpy(&acl_build_param.defs, ipv6 ? ipv6_defs : ipv4_defs, + rte_memcpy(&acl_build_param.defs, ipv6 ? ipv6_defs : ipv4_defs, ipv6 ? sizeof(ipv6_defs) : sizeof(ipv4_defs)); if (rte_acl_build(context, &acl_build_param) != 0) @@ -261,7 +282,7 @@ int ACL::initACL(std::map &fns, int _numa_on, std::set &fns, int _numa_on, std::set +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #include #include "cmdlinetask.h" #include @@ -14,6 +33,7 @@ #include #include "notification.h" #include "main.h" +#include "cfg.h" #include #include @@ -341,11 +361,22 @@ struct cmd_showworker_result cmdline_fixed_string_t option; }; +extern "C" uint32_t ssl_max_packet_size; +extern "C" uint64_t ssl_mallocs; +extern "C" uint64_t ssl_reallocs; + static void display_worker_stats(struct cmdline* cl,const ThreadStats &stats) { cmdline_printf(cl, " Seen packets: %" PRIu64 "\n", stats.total_packets); cmdline_printf(cl, " IP packets: %" PRIu64 " (IPv4 packets: %" PRIu64 ", IPv6 packets: %" PRIu64 ")\n", stats.ip_packets, stats.ipv4_packets, stats.ipv6_packets); cmdline_printf(cl, " Total bytes: %" PRIu64 "\n", stats.total_bytes); + cmdline_printf(cl, " HTTP packets: %" PRIu64 "\n", stats.http_packets); + cmdline_printf(cl, " SSL/TLS packets: %" PRIu64 "\n", stats.ssl_packets); + + cmdline_printf(cl, " SSL/TLS max packet size: %" PRIu32 "\n", ssl_max_packet_size); + cmdline_printf(cl, " SSL/TLS mallocs: %" PRIu64 "\n", ssl_mallocs); + cmdline_printf(cl, " SSL/TLS reallocs: %" PRIu64 "\n", ssl_reallocs); + if(stats.ip_packets && stats.total_bytes) { uint32_t avg_pkt_size = (unsigned int)(stats.total_bytes/stats.ip_packets); @@ -357,25 +388,205 @@ static void display_worker_stats(struct cmdline* cl,const ThreadStats &stats) cmdline_printf(cl, " IPv4 short packets: %" PRIu64 "\n", stats.ipv4_short_packets); cmdline_printf(cl, " Matched by:\n"); cmdline_printf(cl, " ACL ip/port: %" PRIu64 "\n", stats.matched_ip_port); - cmdline_printf(cl, " SSL: %" PRIu64 "\n", stats.matched_ssl); + cmdline_printf(cl, " SSL SNI: %" PRIu64 "\n", stats.matched_ssl_sni); cmdline_printf(cl, " ACL SSL: %" PRIu64 "\n", stats.matched_ssl_ip); - cmdline_printf(cl, " domain: %" PRIu64 "\n", stats.matched_domains); - cmdline_printf(cl, " URL: %" PRIu64 "\n", stats.matched_urls); + cmdline_printf(cl, " HTTP: %" PRIu64 "\n", stats.matched_http_bl_ipv4 + stats.matched_http_bl_ipv6); + cmdline_printf(cl, " IPv4: %" PRIu64 "\n", stats.matched_http_bl_ipv4); + cmdline_printf(cl, " IPv6: %" PRIu64 "\n", stats.matched_http_bl_ipv6); cmdline_printf(cl, " Redirected:\n"); - cmdline_printf(cl, " domains: %" PRIu64 "\n", stats.redirected_domains); - cmdline_printf(cl, " URLs: %" PRIu64 "\n",stats.redirected_urls); - cmdline_printf(cl, " Sended rst: %" PRIu64 "\n",stats.sended_rst); - cmdline_printf(cl, " Maximum active flows:\n"); - cmdline_printf(cl, " IPv4: %" PRIu64 "\n", stats.max_ipv4_flows); - cmdline_printf(cl, " IPv6: %" PRIu64 "\n", stats.max_ipv6_flows); - cmdline_printf(cl, " Active flows: %" PRIu64 "\n", stats.ndpi_flows_count); - cmdline_printf(cl, " IPv4: %" PRIu64 "\n", stats.ndpi_ipv4_flows_count); - cmdline_printf(cl, " IPv6: %" PRIu64 "\n", stats.ndpi_ipv6_flows_count); - cmdline_printf(cl, " Flows deleted: %" PRIu64 "\n", stats.ndpi_flows_deleted); - cmdline_printf(cl, " Flows expired: %" PRIu64 "\n", stats.ndpi_flows_expired); - cmdline_printf(cl, " Reassembled flows: %" PRIu64 "\n", stats.reassembled_flows); - cmdline_printf(cl, " Use malloc for url: %" PRIu64 "\n", stats.dpi_use_url_malloc); - cmdline_printf(cl, " Bad alloc for URL: %" PRIu64 "\n", stats.dpi_no_mempool_http); + cmdline_printf(cl, " HTTP blacklisted: %" PRIu64 "\n", stats.redirected_http_bl_ipv4 + stats.redirected_http_bl_ipv6); + cmdline_printf(cl, " IPv4: %" PRIu64 "\n", stats.redirected_http_bl_ipv4); + cmdline_printf(cl, " IPv6: %" PRIu64 "\n", stats.redirected_http_bl_ipv6); + cmdline_printf(cl, " Sended rst:\n"); + cmdline_printf(cl, " IPv4: %" PRIu64 "\n",stats.sended_rst_ipv4); + cmdline_printf(cl, " IPv6: %" PRIu64 "\n",stats.sended_rst_ipv6); + cmdline_printf(cl, " Sended forbidden:\n"); + cmdline_printf(cl, " IPv4: %" PRIu64 "\n",stats.sended_forbidden_ipv4); + cmdline_printf(cl, " IPv6: %" PRIu64 "\n",stats.sended_forbidden_ipv6); + cmdline_printf(cl, " Already blocked:\n"); + cmdline_printf(cl, " IPv4:\n"); + cmdline_printf(cl, " HTTP : %" PRIu64 "\n", stats.seen_already_blocked_http_ipv4); + cmdline_printf(cl, " SSL : %" PRIu64 "\n", stats.seen_already_blocked_ssl_ipv4); + cmdline_printf(cl, " IPv6:\n"); + cmdline_printf(cl, " HTTP : %" PRIu64 "\n", stats.seen_already_blocked_http_ipv6); + cmdline_printf(cl, " SSL : %" PRIu64 "\n", stats.seen_already_blocked_ssl_ipv6); + cmdline_printf(cl, " DPI errors:\n"); + cmdline_printf(cl, " No memory http: %" PRIu64 "\n",stats.dpi_no_mempool_http); + cmdline_printf(cl, " Flows:\n"); + cmdline_printf(cl, " IPv4:\n"); + cmdline_printf(cl, " New: %" PRIu64 "\n", stats.new_flow); + cmdline_printf(cl, " Recycling: %" PRIu64 "\n", stats.recycling_flow); + cmdline_printf(cl, " Reuse: %" PRIu64 "\n", stats.reuse_flow); + cmdline_printf(cl, " Close: %" PRIu64 "\n", stats.close_flow); + cmdline_printf(cl, " No create: %" PRIu64 "\n", stats.no_create_flow); + cmdline_printf(cl, " No memory: %" PRIu64 "\n", stats.error_alloc_flow); + cmdline_printf(cl, " Alfs fail: %" PRIu64 "\n", stats.alfs_fail_flow); + cmdline_printf(cl, " IPv6:\n"); + cmdline_printf(cl, " New: %" PRIu64 "\n", stats.new_flow_ipv6); + cmdline_printf(cl, " Recycling: %" PRIu64 "\n", stats.recycling_flow_ipv6); + cmdline_printf(cl, " Reuse: %" PRIu64 "\n", stats.reuse_flow_ipv6); + cmdline_printf(cl, " Close: %" PRIu64 "\n", stats.close_flow_ipv6); + cmdline_printf(cl, " No create: %" PRIu64 "\n", stats.no_create_flow_ipv6); + cmdline_printf(cl, " No memory: %" PRIu64 "\n", stats.error_alloc_flow_ipv6); + cmdline_printf(cl, " Alfs fail: %" PRIu64 "\n", stats.alfs_fail_flow_ipv6); + +} + +static uint32_t total_allocated_flows_ipv4 = 0; +static uint64_t total_reused_flows_ipv4 = 0; +static uint32_t total_allocated_flows_ipv6 = 0; +static uint64_t total_reused_flows_ipv6 = 0; + +static uint32_t alfs_short_allocated_ipv4 = 0; +static uint64_t alfs_short_oldest_ipv4 = 0; +static uint64_t alfs_short_alien_ipv4 = 0; +static uint64_t alfs_short_error_ipv4 = 0; + +static uint32_t alfs_long_allocated_ipv4 = 0; +static uint64_t alfs_long_oldest_ipv4 = 0; +static uint64_t alfs_long_alien_ipv4 = 0; +static uint64_t alfs_long_error_ipv4 = 0; + +static uint32_t alfs_short_allocated_ipv6 = 0; +static uint64_t alfs_short_oldest_ipv6 = 0; +static uint64_t alfs_short_alien_ipv6 = 0; +static uint64_t alfs_short_error_ipv6 = 0; + +static uint32_t alfs_long_allocated_ipv6 = 0; +static uint64_t alfs_long_oldest_ipv6 = 0; +static uint64_t alfs_long_alien_ipv6 = 0; +static uint64_t alfs_long_error_ipv6 = 0; + +static void display_worker_memory_stats(struct cmdline *cl, uint8_t worker_id) +{ + cmdline_printf(cl, " Flows memory usage:\n"); + cmdline_printf(cl, " IPv4:\n"); + cmdline_printf(cl, " Parameters:\n"); + cmdline_printf(cl, " Total: %" PRIu32 "\n", global_prm->memory_configs.ipv4.flows_number); + cmdline_printf(cl, " Parts: %d\n", (int) global_prm->memory_configs.ipv4.parts_of_flow); + cmdline_printf(cl, " Parts mask: %" PRIu32 "\n", global_prm->memory_configs.ipv4.mask_parts_flow); + cmdline_printf(cl, " Cache size: %" PRIu32 "\n", global_prm->memory_configs.ipv4.recs_number); + cmdline_printf(cl, " Current usage:\n"); + + cmdline_printf(cl, " Per part:\n"); + uint32_t total_allocated = 0; + uint64_t total_reused = 0; + for(int i=0; i < global_prm->memory_configs.ipv4.parts_of_flow; i++) + { + cmdline_printf(cl, " Part %d:\n", i); + FlowStorageIPV4 *fs_ipv4 = (FlowStorageIPV4 *) worker_params[worker_id].flows_ipv4.flows[i]; + cmdline_printf(cl, " Allocated: %" PRIu32 "\n", fs_ipv4->counters.alloc); + cmdline_printf(cl, " Reused: %" PRIu64 "\n", fs_ipv4->counters.reuse); + total_allocated += fs_ipv4->counters.alloc; + total_reused += fs_ipv4->counters.reuse; + cmdline_printf(cl, " Alfs:\n"); + cmdline_printf(cl, " Short:\n"); + cmdline_printf(cl, " Allctd: %" PRIu32 "\n", fs_ipv4->short_alfs.getAllocated()); + cmdline_printf(cl, " Oldest: %" PRIu64 "\n", fs_ipv4->short_alfs.counters.oldest); + cmdline_printf(cl, " Alien: %" PRIu64 "\n", fs_ipv4->short_alfs.counters.alien_rec); + cmdline_printf(cl, " No mem: %" PRIu64 "\n", fs_ipv4->short_alfs.counters.err_alloc); + cmdline_printf(cl, " Long:\n"); + cmdline_printf(cl, " Allctd: %" PRIu32 "\n", fs_ipv4->long_alfs.getAllocated()); + cmdline_printf(cl, " Oldest: %" PRIu64 "\n", fs_ipv4->long_alfs.counters.oldest); + cmdline_printf(cl, " Alien: %" PRIu64 "\n", fs_ipv4->long_alfs.counters.alien_rec); + cmdline_printf(cl, " No mem: %" PRIu64 "\n", fs_ipv4->long_alfs.counters.err_alloc); + alfs_short_allocated_ipv4 += fs_ipv4->short_alfs.getAllocated(); + alfs_short_oldest_ipv4 += fs_ipv4->short_alfs.counters.oldest; + alfs_short_alien_ipv4 += fs_ipv4->short_alfs.counters.alien_rec; + alfs_short_error_ipv4 += fs_ipv4->short_alfs.counters.err_alloc; + + alfs_long_allocated_ipv4 += fs_ipv4->long_alfs.getAllocated(); + alfs_long_oldest_ipv4 += fs_ipv4->long_alfs.counters.oldest; + alfs_long_alien_ipv4 += fs_ipv4->long_alfs.counters.alien_rec; + alfs_long_error_ipv4 += fs_ipv4->long_alfs.counters.err_alloc; + + } + total_allocated_flows_ipv4 += total_allocated; + total_reused_flows_ipv4 += total_reused; + cmdline_printf(cl, " All parts:\n"); + cmdline_printf(cl, " Allocated: %" PRIu32 "\n", total_allocated); + cmdline_printf(cl, " Reused: %" PRIu64 "\n", total_reused); + + cmdline_printf(cl, " IPv6:\n"); + cmdline_printf(cl, " Parameters:\n"); + cmdline_printf(cl, " Total: %" PRIu32 "\n", global_prm->memory_configs.ipv6.flows_number); + cmdline_printf(cl, " Parts: %d\n", (int) global_prm->memory_configs.ipv6.parts_of_flow); + cmdline_printf(cl, " Parts mask: %" PRIu32 "\n", global_prm->memory_configs.ipv6.mask_parts_flow); + cmdline_printf(cl, " Cache size: %" PRIu32 "\n", global_prm->memory_configs.ipv6.recs_number); + cmdline_printf(cl, " Current usage:\n"); + + cmdline_printf(cl, " Per part:\n"); + + uint32_t total_allocated_ipv6 = 0; + uint64_t total_reused_ipv6 = 0; + + for(int i=0; i < global_prm->memory_configs.ipv6.parts_of_flow; i++) + { + cmdline_printf(cl, " Part %d:\n", i); + FlowStorageIPV6 *fs_ipv6 = (FlowStorageIPV6 *) worker_params[worker_id].flows_ipv6.flows[i]; + cmdline_printf(cl, " Allocated: %" PRIu32 "\n", fs_ipv6->counters.alloc); + cmdline_printf(cl, " Reused: %" PRIu64 "\n", fs_ipv6->counters.reuse); + total_allocated_ipv6 += fs_ipv6->counters.alloc; + total_reused_ipv6 += fs_ipv6->counters.reuse; + cmdline_printf(cl, " Alfs:\n"); + cmdline_printf(cl, " Short:\n"); + cmdline_printf(cl, " Allctd: %" PRIu32 "\n", fs_ipv6->short_alfs.getAllocated()); + cmdline_printf(cl, " Oldest: %" PRIu64 "\n", fs_ipv6->short_alfs.counters.oldest); + cmdline_printf(cl, " Alien: %" PRIu64 "\n", fs_ipv6->short_alfs.counters.alien_rec); + cmdline_printf(cl, " No mem: %" PRIu64 "\n", fs_ipv6->short_alfs.counters.err_alloc); + cmdline_printf(cl, " Long:\n"); + cmdline_printf(cl, " Allctd: %" PRIu32 "\n", fs_ipv6->long_alfs.getAllocated()); + cmdline_printf(cl, " Oldest: %" PRIu64 "\n", fs_ipv6->long_alfs.counters.oldest); + cmdline_printf(cl, " Alien: %" PRIu64 "\n", fs_ipv6->long_alfs.counters.alien_rec); + cmdline_printf(cl, " No mem: %" PRIu64 "\n", fs_ipv6->long_alfs.counters.err_alloc); + alfs_short_allocated_ipv6 += fs_ipv6->short_alfs.getAllocated(); + alfs_short_oldest_ipv6 += fs_ipv6->short_alfs.counters.oldest; + alfs_short_alien_ipv6 += fs_ipv6->short_alfs.counters.alien_rec; + alfs_short_error_ipv6 += fs_ipv6->short_alfs.counters.err_alloc; + alfs_long_allocated_ipv6 += fs_ipv6->long_alfs.getAllocated(); + alfs_long_oldest_ipv6 += fs_ipv6->long_alfs.counters.oldest; + alfs_long_alien_ipv6 += fs_ipv6->long_alfs.counters.alien_rec; + alfs_long_error_ipv6 += fs_ipv6->long_alfs.counters.err_alloc; + } + total_allocated_flows_ipv6 += total_allocated_ipv6; + total_reused_flows_ipv6 += total_reused_ipv6; + cmdline_printf(cl, " All parts:\n"); + cmdline_printf(cl, " Allocated: %" PRIu32 "\n", total_allocated_ipv6); + cmdline_printf(cl, " Reused: %" PRIu64 "\n", total_reused_ipv6); +} + +static void display_worker_memory_stats_all(struct cmdline *cl) +{ + cmdline_printf(cl, " Total alfs ipv4:\n"); + cmdline_printf(cl, " Short:\n"); + cmdline_printf(cl, " Allctd: %" PRIu32 "\n", alfs_short_allocated_ipv4); + cmdline_printf(cl, " Oldest: %" PRIu64 "\n", alfs_short_oldest_ipv4); + cmdline_printf(cl, " Alien: %" PRIu64 "\n", alfs_short_alien_ipv4); + cmdline_printf(cl, " No mem: %" PRIu64 "\n", alfs_short_error_ipv4); + cmdline_printf(cl, " Long:\n"); + cmdline_printf(cl, " Allctd: %" PRIu32 "\n", alfs_long_allocated_ipv4); + cmdline_printf(cl, " Oldest: %" PRIu64 "\n", alfs_long_oldest_ipv4); + cmdline_printf(cl, " Alien: %" PRIu64 "\n", alfs_long_alien_ipv4); + cmdline_printf(cl, " No mem: %" PRIu64 "\n", alfs_long_error_ipv4); + + cmdline_printf(cl, " Total alfs ipv6:\n"); + cmdline_printf(cl, " Short:\n"); + cmdline_printf(cl, " Allctd: %" PRIu32 "\n", alfs_short_allocated_ipv6); + cmdline_printf(cl, " Oldest: %" PRIu64 "\n", alfs_short_oldest_ipv6); + cmdline_printf(cl, " Alien: %" PRIu64 "\n", alfs_short_alien_ipv6); + cmdline_printf(cl, " No mem: %" PRIu64 "\n", alfs_short_error_ipv6); + cmdline_printf(cl, " Long:\n"); + cmdline_printf(cl, " Allctd: %" PRIu32 "\n", alfs_long_allocated_ipv6); + cmdline_printf(cl, " Oldest: %" PRIu64 "\n", alfs_long_oldest_ipv6); + cmdline_printf(cl, " Alien: %" PRIu64 "\n", alfs_long_alien_ipv6); + cmdline_printf(cl, " No mem: %" PRIu64 "\n", alfs_long_error_ipv6); + + cmdline_printf(cl, " Total allocated ipv4 flows: %" PRIu32 "\n", total_allocated_flows_ipv4); + cmdline_printf(cl, " Total allocated ipv6 flows: %" PRIu32 "\n", total_allocated_flows_ipv6); + cmdline_printf(cl, " Total reused ipv4 flows: %" PRIu64 "\n", total_reused_flows_ipv4); + cmdline_printf(cl, " Total reused ipv6 flows: %" PRIu64 "\n", total_reused_flows_ipv6); + cmdline_printf(cl, " Free ipv4 flows: %" PRIu32 "\n", (global_prm->memory_configs.ipv4.flows_number - total_allocated_flows_ipv4)); + cmdline_printf(cl, " Free ipv6 flows: %" PRIu32 "\n", (global_prm->memory_configs.ipv6.flows_number - total_allocated_flows_ipv6)); } static void cmd_showworker_parsed(void* parsed_result, struct cmdline* cl, void* data) @@ -418,11 +629,11 @@ static void cmd_showworker_parsed(void* parsed_result, struct cmdline* cl, void* cmdline_printf(cl, "\n"); } } else { - int core_id = ::atoi(res->workernum); + int worker_id = ::atoi(res->workernum); extFilter *f = extFilter::instance(); for(auto const &thread : f->getThreadsVec()) { - if(core_id == (int)thread->getCoreId()) + if(worker_id == (static_cast(thread))->getWorkerID()) { const ThreadStats stats=(static_cast(thread))->getStats(); cmdline_printf(cl, "Worker '%s' on core %d\n", (static_cast(thread))->getThreadName().c_str(), (int)thread->getCoreId()); @@ -431,12 +642,61 @@ static void cmd_showworker_parsed(void* parsed_result, struct cmdline* cl, void* } } } + } else if (!strcmp(res->what, "memory")) + { + total_allocated_flows_ipv4 = 0; + total_allocated_flows_ipv6 = 0; + + alfs_short_allocated_ipv4 = 0; + alfs_short_oldest_ipv4 = 0; + alfs_short_alien_ipv4 = 0; + alfs_short_error_ipv4 = 0; + alfs_short_allocated_ipv6 = 0; + alfs_short_oldest_ipv6 = 0; + alfs_short_alien_ipv6 = 0; + alfs_short_error_ipv6 = 0; + + alfs_long_allocated_ipv4 = 0; + alfs_long_oldest_ipv4 = 0; + alfs_long_alien_ipv4 = 0; + alfs_long_error_ipv4 = 0; + + alfs_long_allocated_ipv6 = 0; + alfs_long_oldest_ipv6 = 0; + alfs_long_alien_ipv6 = 0; + alfs_long_error_ipv6 = 0; + + if(!strcmp(res->workernum,"all")) + { + extFilter *f = extFilter::instance(); + cmdline_printf(cl, "Working %lu workers:\n", f->getThreadsVec().size()); + for(auto const &thread : f->getThreadsVec()) + { + cmdline_printf(cl, "Worker '%s' on core %d\n", (static_cast(thread))->getThreadName().c_str(), (int)thread->getCoreId()); + display_worker_memory_stats(cl, (static_cast(thread))->getWorkerID()); + cmdline_printf(cl, "\n"); + } + display_worker_memory_stats_all(cl); + cmdline_printf(cl, "\n"); + } else { + int worker_id = ::atoi(res->workernum); + extFilter *f = extFilter::instance(); + for(auto const &thread : f->getThreadsVec()) + { + if(worker_id == (static_cast(thread))->getWorkerID()) + { + cmdline_printf(cl, "Worker '%s' on core %d\n", (static_cast(thread))->getThreadName().c_str(), (int)thread->getCoreId()); + display_worker_memory_stats(cl, (static_cast(thread))->getWorkerID()); + cmdline_printf(cl, "\n"); + } + } + } } } cmdline_parse_token_string_t cmd_showworker_show = TOKEN_STRING_INITIALIZER(struct cmd_showworker_result, show, "show#clear"); cmdline_parse_token_string_t cmd_showworker_worker = TOKEN_STRING_INITIALIZER(struct cmd_showworker_result, worker, "worker"); -cmdline_parse_token_string_t cmd_showworker_what = TOKEN_STRING_INITIALIZER(struct cmd_showworker_result, what, "stats"); +cmdline_parse_token_string_t cmd_showworker_what = TOKEN_STRING_INITIALIZER(struct cmd_showworker_result, what, "stats#memory"); cmdline_parse_token_string_t cmd_showworker_workernum = TOKEN_STRING_INITIALIZER(struct cmd_showworker_result, workernum, NULL); cmdline_parse_token_string_t cmd_showworker_option = TOKEN_STRING_INITIALIZER(struct cmd_showworker_result, option, "-j#json"); @@ -446,7 +706,7 @@ static cmdline_parse_inst_t * init_cmd_showworker_json() cmd_showport = (cmdline_parse_inst_t *)calloc(1, sizeof(cmdline_parse_inst_t) + sizeof(void *) * 6); cmd_showport->f = cmd_showworker_parsed; cmd_showport->data = (void *)1; - cmd_showport->help_str = "show|clear worker stats X (X = core id or all) -j|json"; + cmd_showport->help_str = "show|clear worker stats|memory X (X = core id or all) -j|json"; cmd_showport->tokens[0] = &cmd_showworker_show.hdr; cmd_showport->tokens[1] = &cmd_showworker_worker.hdr; cmd_showport->tokens[2] = &cmd_showworker_what.hdr; @@ -462,7 +722,7 @@ static cmdline_parse_inst_t * init_cmd_showworker() cmd_showport = (cmdline_parse_inst_t *)calloc(1, sizeof(cmdline_parse_inst_t) + sizeof(void *) * 5); cmd_showport->f = cmd_showworker_parsed; cmd_showport->data = nullptr; - cmd_showport->help_str = "show|clear worker stats X (X = core id or all)"; + cmd_showport->help_str = "show|clear worker stats|memory X (X = core id or all)"; cmd_showport->tokens[0] = &cmd_showworker_show.hdr; cmd_showport->tokens[1] = &cmd_showworker_worker.hdr; cmd_showport->tokens[2] = &cmd_showworker_what.hdr; diff --git a/src/flow.cpp b/src/flow.cpp index ed99357..5838a08 100644 --- a/src/flow.cpp +++ b/src/flow.cpp @@ -1,9 +1,30 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ #include "flow.h" +#include "params.h" +#include "cfg.h" #include #include #include +#include //#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 #include @@ -72,44 +93,176 @@ static int compare_ipv6(const void *key1, const void *key2, size_t key_len) } -flowHash::flowHash(int socket_id, int thread_id, uint32_t flowHashSizeIPv4, uint32_t flowHashSizeIPv6) : _logger(Poco::Logger::get("FlowHash_" + std::to_string(thread_id))), - _flowHashSizeIPv4(flowHashSizeIPv4), - _flowHashSizeIPv6(flowHashSizeIPv6) +void initFlowStorages() { + int socketid = 0; + Poco::Util::Application& app = Poco::Util::Application::instance(); + + // allocate mempool for all flows + app.logger().information("Allocating %Lu bytes (%u entries) for ipv4 flow pool", (uint64_t) ((global_prm->memory_configs.ipv4.flows_number)*sizeof(struct ext_dpi_flow_info_ipv4)), global_prm->memory_configs.ipv4.flows_number); + rte_mempool *flows_pool_ipv4 = rte_mempool_create("ipv4_flows_p", global_prm->memory_configs.ipv4.flows_number, sizeof(struct ext_dpi_flow_info_ipv4), 0, 0, NULL, NULL, NULL, NULL, socketid, 0); + if(flows_pool_ipv4 == nullptr) + { + app.logger().fatal("Not enough memory for ipv4 flows pool. Tried to allocate %Lu bytes on socket %d", (uint64_t) ((global_prm->memory_configs.ipv4.flows_number)*sizeof(struct ext_dpi_flow_info_ipv4)), socketid); + throw Poco::Exception("Not enough memory for flows pool"); + } + + app.logger().information("Allocating %Lu bytes (%u entries) for ipv6 flow pool", (uint64_t) ((global_prm->memory_configs.ipv6.flows_number)*sizeof(struct ext_dpi_flow_info_ipv6)), global_prm->memory_configs.ipv6.flows_number); + + rte_mempool *flows_pool_ipv6 = rte_mempool_create("ipv6_flows_p", (global_prm->memory_configs.ipv6.flows_number), sizeof(struct ext_dpi_flow_info_ipv6), 0, 0, NULL, NULL, NULL, NULL, socketid, 0); + if(flows_pool_ipv6 == nullptr) + { + app.logger().fatal("Not enough memory for ipv6 flows pool. Tried to allocate %Lu bytes on socket %d", (uint64_t) ((global_prm->memory_configs.ipv6.flows_number )*sizeof(struct ext_dpi_flow_info_ipv6)), socketid); + throw Poco::Exception("Not enough memory for flows pool"); + } + + flow_storage_params_t prm; + prm.p_lifetime = global_prm->flow_lifetime; + for(int i=0; i < global_prm->workers_number; i++) + { + prm.worker_id = i; + prm.mempool = flows_pool_ipv4; + prm.recs_number = global_prm->memory_configs.ipv4.recs_number / global_prm->memory_configs.ipv4.parts_of_flow; + flow_storage_t *flows = &worker_params[i].flows_ipv4; + if(global_prm->memory_configs.ipv4.parts_of_flow > 0) + { + flows->flows = new (std::nothrow) FlowStorage*[global_prm->memory_configs.ipv4.parts_of_flow]; + if(!flows->flows) + { + app.logger().fatal("Not enough memory for FlowStorage pointers"); + throw Poco::Exception("Not enough memory for FlowStorage pointers"); + } + memset(flows->flows, 0, sizeof(FlowStorage *)*global_prm->memory_configs.ipv4.parts_of_flow); + for(int z=0; z < global_prm->memory_configs.ipv4.parts_of_flow; z++) + { + prm.part_no = z; + flows->flows[z] = new FlowStorageIPV4(&prm); + if(flows->flows[z]->init(&prm)) + { + app.logger().fatal("Unable to init FlowStorageIPV4"); + throw Poco::Exception("Unable to init FlowStorageIPV4"); + } + } + } + + prm.mempool = flows_pool_ipv6; + prm.recs_number = global_prm->memory_configs.ipv6.recs_number / global_prm->memory_configs.ipv6.parts_of_flow; + flows = &worker_params[i].flows_ipv6; + // setup ipv6 + if(global_prm->memory_configs.ipv6.parts_of_flow > 0) + { + flows->flows = new (std::nothrow) FlowStorage*[global_prm->memory_configs.ipv6.parts_of_flow]; + if(!flows->flows) + { + app.logger().fatal("Not enough memory for FlowStorage pointers"); + throw Poco::Exception("Not enough memory for FlowStorage pointers"); + } + memset(flows->flows, 0, sizeof(FlowStorage *)*global_prm->memory_configs.ipv6.parts_of_flow); + for(int z=0; z < global_prm->memory_configs.ipv6.parts_of_flow; z++) + { + prm.part_no = z; + flows->flows[z] = new FlowStorageIPV6(&prm); + if(flows->flows[z]->init(&prm)) + { + app.logger().fatal("Unable to init FlowStorageIPV6"); + throw Poco::Exception("Unable to init FlowStorageIPV6"); + } + } + } + + } +} + +// ipv4 +FlowStorageIPV4::FlowStorageIPV4(flow_storage_params_t *prm) : FlowStorage(prm->mempool), + _logger(Poco::Logger::get("FlowStorageIPV4")) +{ + // init hash + _logger.information("Allocate %d bytes (%d entries, element size %z) for flow hash ipv4", (int)(prm->recs_number * sizeof(union ipv4_5tuple_host)), (int)prm->recs_number, sizeof(union ipv4_5tuple_host)); struct rte_hash_parameters ipv4_hash_params = {0}; - std::string ipv4_hash_name("ipv4_flow_hash_" + std::to_string(thread_id)); - ipv4_hash_params.entries = _flowHashSizeIPv4; + ipv4_hash_params.entries = prm->recs_number; ipv4_hash_params.key_len = sizeof(union ipv4_5tuple_host); ipv4_hash_params.hash_func = ipv4_hash_crc; ipv4_hash_params.hash_func_init_val = 0; - ipv4_hash_params.name = ipv4_hash_name.c_str(); - ipv4_FlowHash = rte_hash_create(&ipv4_hash_params); - if(!ipv4_FlowHash) + std::string hash_name("ipv4_fh" + std::to_string(prm->worker_id) + "_" + std::to_string(prm->part_no)); + ipv4_hash_params.name = hash_name.c_str(); + hash = rte_hash_create(&ipv4_hash_params); + if(!hash) { _logger.fatal("Unable to create ipv4 flow hash"); throw Poco::Exception("Unable to create ipv4 flow hash"); } - rte_hash_set_cmp_func(ipv4_FlowHash, compare_ipv4); - std::string ipv6_hash_name("ipv6_flow_hash_" + std::to_string(thread_id)); + rte_hash_set_cmp_func(hash, compare_ipv4); + + // init pointers for hash data + std::string mem_name("ipv4_f" + std::to_string(prm->worker_id) + "_" + std::to_string(prm->part_no)); + _logger.information("Allocating %d bytes (%d entries) for ipv4_flows", (int) (sizeof(struct ext_dpi_flow_info_ipv4 *) * prm->recs_number), (int)prm->recs_number); + data = (struct ext_dpi_flow_info_ipv4 **)rte_zmalloc(mem_name.c_str(), prm->recs_number * sizeof(struct ext_dpi_flow_info_ipv4 *), RTE_CACHE_LINE_SIZE); + if(data == nullptr) + { + _logger.fatal("Not enough memory for ipv4 flows"); + throw Poco::Exception("Not enough memory for ipv4 flows"); + } + +} + +FlowStorageIPV4::~FlowStorageIPV4() +{ + // delete hash + rte_hash_free(hash); + // delete data + rte_free(data); +} + +int FlowStorageIPV4::init(flow_storage_params_t *prm) +{ + if(short_alfs.init(prm->recs_number, prm->worker_id, en_alfs_short, prm->p_lifetime[0], 32)) + return -1; + return long_alfs.init(prm->recs_number, prm->worker_id, en_alfs_long, prm->p_lifetime[1], 8); +} + +// ipv6 +FlowStorageIPV6::FlowStorageIPV6(flow_storage_params_t *prm) : FlowStorage(prm->mempool), + _logger(Poco::Logger::get("FlowStorageIPV6")) +{ + // init hash + _logger.information("Allocate %d bytes (%d entries, element size %z) for flow hash ipv6", (int)(prm->recs_number * sizeof(union ipv6_5tuple_host)), (int)prm->recs_number, sizeof(union ipv6_5tuple_host)); struct rte_hash_parameters ipv6_hash_params = {0}; - ipv6_hash_params.entries = _flowHashSizeIPv6; + ipv6_hash_params.entries = prm->recs_number; ipv6_hash_params.key_len = sizeof(union ipv6_5tuple_host); ipv6_hash_params.hash_func = ipv6_hash_crc; ipv6_hash_params.hash_func_init_val = 0; - ipv6_hash_params.name = ipv6_hash_name.c_str(); - ipv6_FlowHash = rte_hash_create(&ipv6_hash_params); - if(!ipv4_FlowHash) + std::string hash_name("ipv6_fh" + std::to_string(prm->worker_id) + "_" + std::to_string(prm->part_no)); + ipv6_hash_params.name = hash_name.c_str(); + hash = rte_hash_create(&ipv6_hash_params); + if(!hash) { _logger.fatal("Unable to create ipv6 flow hash"); throw Poco::Exception("Unable to create ipv6 flow hash"); } - rte_hash_set_cmp_func(ipv6_FlowHash, compare_ipv6); + rte_hash_set_cmp_func(hash, compare_ipv6); + // init pointers for hash data + std::string mem_name("ipv6_f" + std::to_string(prm->worker_id) + "_" + std::to_string(prm->part_no)); + _logger.information("Allocating %d bytes (%d entries) for ipv6_flows", (int) (sizeof(struct ext_dpi_flow_info_ipv6 *) * prm->recs_number), (int)prm->recs_number); + data = (struct ext_dpi_flow_info_ipv6 **)rte_zmalloc(mem_name.c_str(), prm->recs_number * sizeof(struct ext_dpi_flow_info_ipv6 *), RTE_CACHE_LINE_SIZE); + if(data == nullptr) + { + _logger.fatal("Not enough memory for ipv6 flows"); + throw Poco::Exception("Not enough memory for ipv6 flows"); + } } - -flowHash::~flowHash() +FlowStorageIPV6::~FlowStorageIPV6() { - rte_hash_free(ipv4_FlowHash); - rte_hash_free(ipv6_FlowHash); + // delete hash + rte_hash_free(hash); + // delete data + rte_free(data); } +int FlowStorageIPV6::init(flow_storage_params_t *prm) +{ + if(short_alfs.init(prm->recs_number, prm->worker_id, en_alfs_short, prm->p_lifetime[0], 32)) + return -1; + return long_alfs.init(prm->recs_number, prm->worker_id, en_alfs_long, prm->p_lifetime[1], 8); +} diff --git a/src/http.cpp b/src/http.cpp new file mode 100644 index 0000000..877aaa1 --- /dev/null +++ b/src/http.cpp @@ -0,0 +1,132 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#include "http.h" +#include "worker.h" + +namespace http +{ + +int on_url_ext (http_parser *p, const char* at, size_t length, dpi_pkt_infos_t* pkt_informations, void** flow_specific_user_data, void* user_data) +{ + WorkerThread *obj = (WorkerThread *) user_data; +// if(length > 0 && p->type==HTTP_REQUEST && (p->method == DPI_HTTP_POST || p->method == DPI_HTTP_GET || p->method == DPI_HTTP_HEAD)) + if(length > 0 && p->type==HTTP_REQUEST && p->method == DPI_HTTP_GET) + { + struct http_req_buf *d = (struct http_req_buf *) *flow_specific_user_data; + if(d == nullptr) + { + d = obj->allocateHTTPBuf(); + if(unlikely(d == nullptr)) + { + obj->getStats().dpi_no_mempool_http++; + return 0; + } + *flow_specific_user_data = d; + } + if(d->uri.length + length > d->uri.buf_size) + length = d->uri.buf_size - d->uri.length; + if(likely(length > 0)) + { + if(d->uri.length != 0) + { + rte_memcpy(d->uri.buf + d->uri.length, at, length); + d->uri.length += length; + } else { + rte_memcpy(d->uri.buf, at, length); + d->uri.length = length; + } + } + } + return 0; +} + +int on_header_field_ext(http_parser *p, const char *at, size_t length, dpi_pkt_infos_t* pkt_informations, void** flow_specific_user_data, void* user_data) +{ + if(*flow_specific_user_data != NULL && length > 0) + { + struct http_req_buf *d = (struct http_req_buf *) *flow_specific_user_data; + for(size_t z = 0; z < length; z++) + { + switch (d->h_state) + { + case hstate_nothing: + switch (at[z]) + { + case 'h': + case 'H': + d->h_state = http::hstate_host; + d->h_prev_char = at[z]; + break; + } + break; + case hstate_host: + if(((d->h_prev_char == 'h' || d->h_prev_char == 'H') && (at[z] == 'o' || at[z] == 'O')) || ((d->h_prev_char == 'O' || d->h_prev_char == 'o') && (at[z] == 's' || at[z] == 'S'))) + { + d->h_prev_char = at[z]; + break; + } + if((d->h_prev_char == 's' || d->h_prev_char == 'S') && (at[z] == 't' || at[z] == 'T')) + { + break; + } + d->h_state = hstate_nothing; + break; + default: + d->h_prev_char = 0; + d->h_state = hstate_nothing; + break; + } + } + } + return 0; +} + +int on_header_value_ext(http_parser *p, const char *at, size_t length, dpi_pkt_infos_t* pkt_informations, void** flow_specific_user_data, void* user_data) +{ + if(*flow_specific_user_data != NULL && length > 0) + { + struct http::http_req_buf *d = (struct http::http_req_buf *) *flow_specific_user_data; + switch (d->h_state) + { + case http::hstate_host: + if(d->host_r.length + length > d->host_r.buf_size) + length = d->host_r.buf_size - d->host_r.length; + if(length > 0) + { + if(d->host_r.length != 0) + { + rte_memcpy(d->host_r.buf + d->host_r.length, at, length); + d->host_r.length += length; + } else { + rte_memcpy(d->host_r.buf, at, length); + d->host_r.length = length; + } + } + break; + default: + break; + } + } + return 0; +} + + + +} \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index f8b4403..8436d02 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -21,17 +21,17 @@ #include #include "worker.h" #include "main.h" -#include "dpi.h" +#include "dtypes.h" #include -#include "AhoCorasickPlus.h" -#include "sendertask.h" +#include "cfg.h" #include "statistictask.h" #include "reloadtask.h" #include "acl.h" #include "cmdlinetask.h" #include "notification.h" #include "config.h" +#include "tries.h" #define MBUF_CACHE_SIZE 256 @@ -54,19 +54,28 @@ struct lcore_conf extFilter::_lcore_conf[RTE_MAX_LCORE]; uint8_t sender_mac[6]; +const global_params_t *global_prm = nullptr; // основные параметры системы +worker_params_t worker_params[MAX_WORKER_THREADS] __rte_cache_aligned; // параметры для worker'ов + +uint8_t m_RSSKey[40] = { + 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, + 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, + 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, + 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, + 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A +}; + void flow_delete_cb(void* flow_specific_user_data) { - if(flow_specific_user_data) + if(flow_specific_user_data != nullptr) { - struct dpi_flow_info *u = (struct dpi_flow_info *) flow_specific_user_data; - u->free_mem(); - if(u->dpi_mempool == nullptr) - free(u); - else - rte_mempool_put(u->dpi_mempool, u); + struct http::http_req_buf *d = (struct http::http_req_buf *) flow_specific_user_data; + if(d->mempool != nullptr) + rte_mempool_put(d->mempool, d); } } + extFilter::extFilter(): _helpRequested(false), _listDPDKPorts(false), _numa_on(1), @@ -82,22 +91,118 @@ extFilter::extFilter(): _helpRequested(false), memset(&_lcore_conf[i], 0, sizeof(lcore_conf)); } _instance = this; + if(global_prm == nullptr) + { + global_prm = new global_params_t; + } + // Poco::ErrorHandler::set(&_errorHandler); } extFilter::~extFilter() { + delete global_prm; } +int _calc_scale(int scale, int min_val, int max_val) +{ + return min_val + (((double)scale - 1.0) / 9.0 * (double)(max_val - min_val) + 0.5); +} -uint8_t m_RSSKey[40] = { - 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, - 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, - 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, - 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, - 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A -}; +int _calc_number_recs(int n_workers, int num_flows) +{ + int result = num_flows; + if ( n_workers != 1 ) + { + if ( n_workers <= 3 ) + result = num_flows / n_workers * 1.50; // +50 % + else + result = num_flows / n_workers * 1.25; // +25 % + } + return result; +} + + +void extFilter::initParams() +{ + global_params_t *prm = (global_params_t *)global_prm; + memset(prm, 0, sizeof(global_params_t)); + + int scale = config().getInt("dpi.scale", 10); + if(scale < 1 && scale > 10) + { + throw Poco::Exception("scale must be between 1..10"); + } + + // flow params + + prm->memory_configs.ipv4.flows_number = config().getInt("dpi.max_active_flows_ipv4", 0); + if(prm->memory_configs.ipv4.flows_number == 0) + { + prm->memory_configs.ipv4.flows_number = _calc_scale(scale, 500000, 10000000); + } + + prm->memory_configs.ipv6.flows_number = config().getInt("dpi.max_active_flows_ipv6", 0); + if(prm->memory_configs.ipv6.flows_number == 0) + { + prm->memory_configs.ipv6.flows_number = _calc_scale(scale, 20000, 50000); + } + + prm->frag_configs.ipv6.state = config().getBool("dpi.fragmentation_ipv6_state", true); + prm->frag_configs.ipv4.state = config().getBool("dpi.fragmentation_ipv4_state", true); + + if(prm->frag_configs.ipv4.state) + prm->frag_configs.ipv4.table_size = config().getInt("dpi.fragmentation_ipv4_table_size", 512); + if(prm->frag_configs.ipv6.state) + prm->frag_configs.ipv6.table_size = config().getInt("dpi.fragmentation_ipv6_table_size", 512); + + prm->tcp_reordering = config().getBool("dpi.tcp_reordering", true); + + + int parts_of_flow = config().getInt("dpi.parts_of_flow_ipv4", 0); + if(parts_of_flow == 0) + { + static int parts_of_flow_IPv4[11] = { 4, 4, 4, 4, 8, 8, 8, 8, 16, 16, 16 }; + parts_of_flow = parts_of_flow_IPv4[scale]; + } + if(parts_of_flow != 4 && parts_of_flow != 8 && parts_of_flow != 16) + { + throw Poco::Exception("parts_of_flow must be 4, 8 or 16"); + } + prm->memory_configs.ipv4.parts_of_flow = parts_of_flow; + + parts_of_flow = config().getInt("dpi.parts_of_flow_ipv6", 0); + if(parts_of_flow == 0) + { + static int parts_of_flow_IPv6[11] = { 2, 2, 2, 2, 4, 4, 4, 4, 8, 8, 8 }; + parts_of_flow = parts_of_flow_IPv6[scale]; + } + if(parts_of_flow != 2 && parts_of_flow != 4 && parts_of_flow != 8) + { + throw Poco::Exception("parts_of_flow must be 2, 4 or 8"); + } + prm->memory_configs.ipv6.parts_of_flow = parts_of_flow; + // calc masks + prm->memory_configs.ipv6.mask_parts_flow = prm->memory_configs.ipv6.parts_of_flow - 1; + prm->memory_configs.ipv4.mask_parts_flow = prm->memory_configs.ipv4.parts_of_flow - 1; + + prm->workers_number = _nb_lcore_params; + + prm->memory_configs.ipv4.recs_number = _calc_number_recs(_nb_lcore_params, prm->memory_configs.ipv4.flows_number); + prm->memory_configs.ipv6.recs_number = _calc_number_recs(_nb_lcore_params, prm->memory_configs.ipv6.flows_number); + + prm->flow_lifetime[0] = 30; + prm->flow_lifetime[1] = 300; + + prm->memory_configs.http_entries = _calc_scale(scale, 70000, 250000); + + prm->answer_duplication = config().getInt("answer_duplication", 0); + if(prm->answer_duplication > 3) + { + logger().warning("answer_duplication set to 3, it must be between 0 and 3"); + } +} static inline unsigned get_port_max_rx_queues(uint8_t port_id) { @@ -225,8 +330,8 @@ int extFilter::initSenderPort(uint8_t port, struct ether_addr *addr, uint8_t nb_ portConf.rxmode.jumbo_frame = DPDK_CONFIG_JUMBO_FRAME; portConf.rxmode.hw_strip_crc = DPDK_CONFIG_HW_STRIP_CRC; portConf.rxmode.mq_mode = DPDK_CONFIG_MQ_MODE; - portConf.rx_adv_conf.rss_conf.rss_key = m_RSSKey; - portConf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_IPV4 | ETH_RSS_IPV6; + portConf.rx_adv_conf.rss_conf.rss_key = NULL; + portConf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_IP; portConf.txmode.mq_mode = ETH_MQ_TX_NONE; retval = rte_eth_dev_configure(port, 1, nb_tx_queue, &portConf); @@ -618,25 +723,61 @@ void extFilter::initialize(Application& self) { loadConfiguration(); ServerApplication::initialize(self); +/* + std::string fl("/usr/local/etc/extfilter/domains"); + std::string ur("/usr/local/etc/extfilter/urls"); + std::string sn("/usr/local/etc/extfilter/ssl_host"); - _dpi_max_active_flows_ipv4 = config().getInt("dpi.max_active_flows_ipv4", 1000000); - _dpi_max_active_flows_ipv6 = config().getInt("dpi.max_active_flows_ipv6", 20000); - _dpi_fragmentation_ipv6_state = config().getBool("dpi.fragmentation_ipv6_state", true); - _dpi_fragmentation_ipv4_state = config().getBool("dpi.fragmentation_ipv4_state", true); - if(_dpi_fragmentation_ipv4_state) - _dpi_fragmentation_ipv4_table_size = config().getInt("dpi.fragmentation_ipv4_table_size", 512); - if(_dpi_fragmentation_ipv6_state) - _dpi_fragmentation_ipv6_table_size = config().getInt("dpi.fragmentation_ipv6_table_size", 512); - _dpi_tcp_reordering = config().getBool("dpi.tcp_reordering", true); - _dpi_maximum_url_size = config().getInt("dpi.max_url_size", 600); + const char *bl = "notify.tushino.com/blacklist"; + if(_tries.getBLManager()->init(fl, ur, sn, bl, strlen(bl))) + { + std::cout << "error!" << std::endl; + exit(0); + } + std::cout << "everything is ok" << std::endl; + + const char *host = "archive.is"; + const char *uri = "/20150813064134/http://www.maxi24-az.com/ru/obyavlenie/amfetamin-skorost-89612877418-krasnodar-stimulyator-metamfetamin_1690872.html"; + + char *redir_url = nullptr; + int z = 0; + if((z=_tries.checkURLBlocked(0, host, strlen(host), uri, strlen(uri), &redir_url))) + { + std::cout << "URL is blocked" << std::endl; + if(redir_url) + { + std::cout << "redir to: " << redir_url << ", length: " << z << std::endl; + } + } else { + std::cout << "URL is not blocked" << std::endl; + } + exit(0); +*/ + +/* std::string emp; + fl = "/usr/local/etc/extfilter/ssl_host"; + _tries.getSNIBlacklist()->load(fl, emp); + if(_tries.checkSNIBlocked(0, host, strlen(host))) + { + std::cout << "SNI is blocked" << std::endl; + } else { + std::cout << "SNI is not blocked" << std::endl; + } +*/ + +/*#include "utils.h" + const char *b = "f\x09ucked"; + char buf[4096]; + url_encode(buf, b, strlen(b)); + std::cout << "buf: '" << buf << "'" << std::endl; + + const char abc[]="abc"; + std::cout << "size: " << sizeof(abc) << std::endl; +*/ +// exit(0); _num_of_senders = config().getInt("num_of_senders", 1); - _lower_host = config().getBool("lower_host", false); - _match_url_exactly = config().getBool("match_url_exactly", false); _block_ssl_no_sni = config().getBool("block_ssl_no_sni", false); - _http_redirect = config().getBool("http_redirect", true); - _url_normalization = config().getBool("url_normalization", true); - _remove_dot = config().getBool("remove_dot", false); _statistic_interval = config().getInt("statistic_interval", 0); _urlsFile = config().getString("urllist",""); _domainsFile = config().getString("domainlist",""); @@ -656,26 +797,12 @@ void extFilter::initialize(Application& self) _sender_params.code=http_code; logger().debug("HTTP code set to %s", http_code); } - _sender_params.redirect_url=config().getString("redirect_url",""); + std::string redirect_url = config().getString("redirect_url",""); + _sender_params.redirect_url = redirect_url; + _sender_params.send_rst_to_server=config().getBool("rst_to_server",false); _sender_params.mtu=config().getInt("out_mtu",1500); - std::string add_p_type=config().getString("url_additional_info","none"); - std::transform(add_p_type.begin(), add_p_type.end(), add_p_type.begin(), ::tolower); - - std::map add_type_s; - add_type_s["none"]=A_TYPE_NONE; - add_type_s["line"]=A_TYPE_ID; - add_type_s["url"]=A_TYPE_URL; - - std::map::iterator it=add_type_s.find(add_p_type); - if(it == add_type_s.end()) - { - throw Poco::Exception("Unknown url_additional_info type '" + add_p_type + "'",404); - } - _add_p_type=it->second; - logger().debug("URL additional info set to %s", add_p_type); - _notify_enabled = config().getBool("notify_enabled", false); _notify_acl_file = config().getString("notify_acl_file",""); @@ -793,7 +920,7 @@ void extFilter::initialize(Application& self) { if(cnt_sender > 0) { - logger().fatal("Too many sender ports"); + logger().fatal("Too many senders ports"); throw Poco::Exception("Congfiguration error"); } ++cnt_sender; @@ -812,11 +939,10 @@ void extFilter::initialize(Application& self) { logger().fatal("Destination mac address not found for port %d", (int)i); throw Poco::Exception("Congfiguration error"); - } int last = 0; int rc = sscanf(mac.c_str(), "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx%n", sender_mac + 0, sender_mac + 1, sender_mac + 2, sender_mac + 3, sender_mac + 4, sender_mac + 5, &last); - if(rc != 6 || mac.size() != last) + if(rc != 6 || mac.size() != (std::size_t) last) { logger().fatal("Invalid mac address '%s' for port %d", mac, (int)i); throw Poco::Exception("Congfiguration error"); @@ -830,7 +956,11 @@ void extFilter::initialize(Application& self) throw Poco::Exception("Congfiguration error"); } Poco::StringTokenizer restTokenizer(p, ";"); - std::vector lcores; + if(restTokenizer.count() > MAX_WORKER_THREADS) + { + logger().fatal("Exceeded max number of worker threads: %z", restTokenizer.count()); + throw Poco::Exception("Configuration error"); + } int nb_lcores_per_port = 0; for(auto itr=restTokenizer.begin(); itr!=restTokenizer.end(); ++itr) { @@ -842,7 +972,6 @@ void extFilter::initialize(Application& self) logger().fatal("Exceeded max number of lcore params: %d", (int) _nb_lcore_params); throw Poco::Exception("Configuration error"); } - lcores.push_back(lcore_id); _lcore_params_array[_nb_lcore_params].port_id = i; _lcore_params_array[_nb_lcore_params].port_type = port_type; _lcore_params_array[_nb_lcore_params].queue_id = queue_id; @@ -858,8 +987,13 @@ void extFilter::initialize(Application& self) throw Poco::Exception("Configuration error"); } -// _flowhash_size_per_worker=rte_align32pow2((_flowhash_size/_nb_lcore_params) * n_ports); + if(!cnt_sender) + { + logger().fatal("The senders port is not defined"); + throw Poco::Exception("Configuration error"); + } + initParams(); _nb_ports = rte_eth_dev_count(); if(_nb_ports == 0) @@ -883,13 +1017,19 @@ void extFilter::initialize(Application& self) // init acl _acl = new ACL(); - + if(loadACL()) throw Poco::Exception("Can't init ACL"); + + if(_tries.getBLManager()->init(_domainsFile, _urlsFile, _sslFile, redirect_url.empty() ? nullptr : redirect_url.c_str(), redirect_url.empty() ? 0 : redirect_url.length())) + { + logger().fatal("Unable to load blacklists"); + throw Poco::Exception("Unable to load blacklists"); + } + // init value... _tsc_hz = rte_get_tsc_hz(); - } void extFilter::uninitialize() @@ -1054,22 +1194,22 @@ int extFilter::main(const ArgVec& args) WorkerConfig workerConfigArr[RTE_MAX_LCORE]; Poco::TaskManager tm; - if(_mp == nullptr) - { - for(int i=1; i <= _num_of_senders; i++) - { - tm.start(new SenderTask(new CSender(_sender_params), i)); - } - } - - NotifyManager *nm = new NotifyManager(20000, _notify_groups); - tm.start(nm); +// NotifyManager *nm = new NotifyManager(20000, _notify_groups); +// tm.start(nm); - int max_ipv4_flows_per_core = ceil((float)_dpi_max_active_flows_ipv4/(float)(_nb_lcore_params)); - int max_ipv6_flows_per_core = ceil((float)_dpi_max_active_flows_ipv6/(float)(_nb_lcore_params)); + std::string pool_name("DPIHTTPPool"); + logger().information("Create pool '%s' for the http dissector with number of entries: %u, element size %z size: %Lu bytes", pool_name, global_prm->memory_configs.http_entries, sizeof(http::http_req_buf),(uint64_t)(global_prm->memory_configs.http_entries * sizeof(http::http_req_buf))); + struct rte_mempool *dpi_http_mempool = rte_mempool_create(pool_name.c_str(), global_prm->memory_configs.http_entries, sizeof(http::http_req_buf), 0, 0, NULL, NULL, NULL, NULL, 0, 0); + if(dpi_http_mempool == nullptr) + { + logger().fatal("Unable to create mempool for the http dissector."); + return Poco::Util::Application::EXIT_CONFIG; + } + initFlowStorages(); + uint8_t worker_id = 0; uint16_t tx_queue_id = 0; /* launch per-lcore init on every lcore */ RTE_LCORE_FOREACH(lcore_id) @@ -1077,35 +1217,12 @@ int extFilter::main(const ArgVec& args) qconf = &_lcore_conf[lcore_id]; if (qconf->n_rx_queue != 0) { - if(!_domainsFile.empty() && !_urlsFile.empty()) - { - workerConfigArr[lcore_id].atm_new = new AhoCorasickPlus(); - loadDomainsURLs(_domainsFile, _urlsFile, workerConfigArr[lcore_id].atm_new); - workerConfigArr[lcore_id].atm_new->finalize(); - } - workerConfigArr[lcore_id].block_ssl_no_sni = _block_ssl_no_sni; - if(!_sslFile.empty()) - { - workerConfigArr[lcore_id].atmSSLDomains_new = new AhoCorasickPlus(); - loadDomains(_sslFile, workerConfigArr[lcore_id].atmSSLDomains_new); - workerConfigArr[lcore_id].atmSSLDomains_new->finalize(); - } - workerConfigArr[lcore_id].match_url_exactly = _match_url_exactly; - workerConfigArr[lcore_id].lower_host = _lower_host; - workerConfigArr[lcore_id].http_redirect = _http_redirect; - workerConfigArr[lcore_id].url_normalization = _url_normalization; - workerConfigArr[lcore_id].remove_dot = _remove_dot; - workerConfigArr[lcore_id].add_p_type = _add_p_type; - workerConfigArr[lcore_id].notify_enabled = _notify_enabled; - workerConfigArr[lcore_id].nm = nm; - workerConfigArr[lcore_id].sender_port = _dpdk_send_port; - workerConfigArr[lcore_id].tx_queue_id = tx_queue_id; - workerConfigArr[lcore_id].maximum_url_size = _dpi_maximum_url_size; - - logger().information("Initializing dpi flow hash with ipv4 max flows %d, ipv6 max flows %d.", max_ipv4_flows_per_core, max_ipv6_flows_per_core); - flowHash *mFlowHash = new flowHash(rte_lcore_to_socket_id(lcore_id), lcore_id, max_ipv4_flows_per_core, max_ipv6_flows_per_core); - -// dpi_library_state_t* dpi_state = dpi_init_stateful(ipv4_buckets, ipv6_buckets, _dpi_max_active_flows_ipv4, _dpi_max_active_flows_ipv6); + workerConfigArr[worker_id].notify_enabled = _notify_enabled; +// workerConfigArr[worker_id].nm = nm; + workerConfigArr[worker_id].sender_port = _dpdk_send_port; + workerConfigArr[worker_id].tx_queue_id = tx_queue_id; + workerConfigArr[worker_id].block_ssl_no_sni = _block_ssl_no_sni; + dpi_library_state_t* dpi_state = dpi_init_stateless(); dpi_set_max_trials(dpi_state, 1); dpi_inspect_nothing(dpi_state); @@ -1118,28 +1235,24 @@ int extFilter::main(const ArgVec& args) protocol.l7prot = DPI_PROTOCOL_TCP_SSL; dpi_set_protocol(dpi_state, protocol); - dpi_set_flow_cleaner_callback(dpi_state, &flow_delete_cb); - if(!_dpi_tcp_reordering) + if(!global_prm->tcp_reordering) dpi_tcp_reordering_disable(dpi_state); else dpi_tcp_reordering_enable(dpi_state); - if(_dpi_fragmentation_ipv4_state) - dpi_ipv4_fragmentation_enable(dpi_state, _dpi_fragmentation_ipv4_table_size); + if(global_prm->frag_configs.ipv4.state) + dpi_ipv4_fragmentation_enable(dpi_state, global_prm->frag_configs.ipv4.table_size); else dpi_ipv4_fragmentation_disable(dpi_state); - if(_dpi_fragmentation_ipv6_state) - dpi_ipv6_fragmentation_enable(dpi_state, _dpi_fragmentation_ipv6_table_size); + if(global_prm->frag_configs.ipv6.state) + dpi_ipv6_fragmentation_enable(dpi_state, global_prm->frag_configs.ipv6.table_size); else dpi_ipv6_fragmentation_disable(dpi_state); - - - - std::string workerName("WorkerThread-" + std::to_string(lcore_id)); + std::string workerName("WorkerThrd_" + std::to_string(worker_id)); logger().debug("Preparing thread '%s'", workerName); ESender::nparams prms; if(_mp != nullptr) @@ -1149,21 +1262,8 @@ int extFilter::main(const ArgVec& args) prms.to_mac = &sender_mac[0]; } - struct rte_mempool *url_mempool = nullptr; - unsigned url_entries = (max_ipv4_flows_per_core+max_ipv6_flows_per_core)*PERCENT_URL_ENTRIES; - std::string pool_name("URLPool-" + std::to_string(lcore_id)); - logger().information("Create pool '%s' for urls with number of entries: %d, size: %d bytes", pool_name, (int) url_entries, (int)(url_entries * (_dpi_maximum_url_size+1))); - url_mempool = rte_mempool_create(pool_name.c_str(), url_entries, _dpi_maximum_url_size+1, 0, 0, NULL, NULL, NULL, NULL, rte_lcore_to_socket_id(lcore_id), 0); - if(url_mempool == nullptr) - logger().warning("Unable to create mempool for url. Will be use malloc. This may affect performance."); - struct rte_mempool *dpi_mempool = nullptr; - pool_name.assign("DPIPool-" + std::to_string(lcore_id)); - logger().information("Create pool '%s' for http dissector with number of entries: %d, size: %d bytes", pool_name, (int) url_entries, (int)(url_entries * sizeof(dpi_flow_info))); - dpi_mempool = rte_mempool_create(pool_name.c_str(), url_entries, sizeof(dpi_flow_info), 0, 0, NULL, NULL, NULL, NULL, rte_lcore_to_socket_id(lcore_id), 0); - if(dpi_mempool == nullptr) - logger().warning("Unable to create mempool for http dissector. Will be use malloc. This may affect performance."); - - WorkerThread* newWorker = new WorkerThread(workerName, workerConfigArr[lcore_id], dpi_state, rte_lcore_to_socket_id(lcore_id), mFlowHash, prms, _mp, url_mempool, dpi_mempool); + + WorkerThread* newWorker = new WorkerThread(worker_id, workerName, workerConfigArr[worker_id], dpi_state, rte_lcore_to_socket_id(lcore_id), prms, _mp, dpi_http_mempool); int err = rte_eal_remote_launch(dpdkWorkerThreadStart, newWorker, lcore_id); if (err != 0) @@ -1174,6 +1274,7 @@ int extFilter::main(const ArgVec& args) _workerThreadVec.push_back(newWorker); pthread_setname_np(lcore_config[lcore_id].thread_id, workerName.c_str()); tx_queue_id++; + worker_id++; } } @@ -1188,6 +1289,7 @@ int extFilter::main(const ArgVec& args) { rte_eth_stats_reset(port_id); } + waitForTerminationRequest(); for (auto iter = _workerThreadVec.begin(); iter != _workerThreadVec.end(); iter++) @@ -1195,9 +1297,7 @@ int extFilter::main(const ArgVec& args) (*iter)->stop(); rte_eal_wait_lcore((*iter)->getCoreId()); } - tm.cancelAll(); - SenderTask::queue.wakeUpAll(); tm.joinAll(); for (uint8_t portid = 0; portid < _nb_ports; portid++) @@ -1213,160 +1313,6 @@ int extFilter::main(const ArgVec& args) return Poco::Util::Application::EXIT_OK; } -void extFilter::loadDomainsURLs(std::string &domains, std::string &urls, AhoCorasickPlus *dm_atm) -{ - logger().debug("Loading domains from file %s",domains); - Poco::FileInputStream df(domains); - int entry_id=0; - if(df.good()) - { - int lineno=0; - while(!df.eof()) - { - lineno++; - std::string str; - getline(df,str); - if(!str.empty()) - { - if(str[0] == '#' || str[0] == ';') - continue; - AhoCorasickPlus::EnumReturnStatus status; - AhoCorasickPlus::PatternId patId = lineno; - std::size_t pos = str.find("*."); - bool exact_match=true; - std::string insert=str; - patId <<= 2; - if(pos != std::string::npos) - { - exact_match=false; - insert=str.substr(pos+2,str.length()-2); - } - patId |= exact_match; - patId |= E_TYPE_DOMAIN << 1; - status = dm_atm->addPattern(insert, patId); - if (status != AhoCorasickPlus::RETURNSTATUS_SUCCESS) - { - if(status == AhoCorasickPlus::RETURNSTATUS_DUPLICATE_PATTERN) - { - logger().warning("Pattern '%s' already present in the database from file %s",insert,domains); - continue; - } else { - logger().error("Failed to add '%s' from line %d from file %s",insert,lineno,domains); - } - } - } - entry_id++; - } - } else - throw Poco::OpenFileException(domains); - df.close(); - logger().debug("Finish loading domains"); - logger().debug("Loading URLS from file %s",urls); - Poco::FileInputStream uf(urls); - if(uf.good()) - { - int lineno=0; - while(!uf.eof()) - { - lineno++; - std::string str; - getline(uf,str); - if(!str.empty()) - { - if(str[0] == '#' || str[0] == ';') - continue; - AhoCorasickPlus::EnumReturnStatus status; - AhoCorasickPlus::PatternId patId = lineno; - if(_url_normalization) - { - std::string url = "http://" + str; - try - { - Poco::URI url_p(url); - url_p.normalize(); - url.assign(url_p.toString()); - } catch (Poco::SyntaxException &ex) - { - logger().error("An SyntaxException occured: '%s' on URI: '%s'", ex.displayText(), url); - } - str.assign(url.c_str()+7, url.length()-7); - } -/* std::string url = str; - std::size_t http_pos = url.find("http://"); - if(http_pos == std::string::npos || http_pos > 0) - { - url.insert(0,"http://"); - }*/ - patId <<= 2; - patId |= 0; - patId |= E_TYPE_URL << 1; - - status = dm_atm->addPattern(str, patId); - if (status != AhoCorasickPlus::RETURNSTATUS_SUCCESS) - { - if(status == AhoCorasickPlus::RETURNSTATUS_DUPLICATE_PATTERN) - { - logger().warning("Pattern '%s' already present in the URL database from file %s",str,urls); - continue; - } else { - logger().error("Failed to add '%s' from line %d from file %s",str,lineno,urls); - } - } - } - entry_id++; - } - } else - throw Poco::OpenFileException(urls); - uf.close(); - logger().debug("Finish loading URLS"); -} - -void extFilter::loadDomains(std::string &fn, AhoCorasickPlus *dm_atm) -{ - logger().debug("Loading domains from file %s",fn); - Poco::FileInputStream df(fn); - if(df.good()) - { - int lineno=1; - while(!df.eof()) - { - std::string str; - getline(df,str); - if(!str.empty()) - { - if(str[0] == '#' || str[0] == ';') - continue; - AhoCorasickPlus::EnumReturnStatus status; - AhoCorasickPlus::PatternId patId = lineno; - std::size_t pos = str.find("*."); - bool exact_match=true; - std::string insert=str; - if(pos != std::string::npos) - { - exact_match=false; - insert=str.substr(pos+2,str.length()-2); - } - patId <<= 2; - patId |= exact_match; - patId |= E_TYPE_DOMAIN << 1; - status = dm_atm->addPattern(insert, patId); - if (status!=AhoCorasickPlus::RETURNSTATUS_SUCCESS) - { - if(status == AhoCorasickPlus::RETURNSTATUS_DUPLICATE_PATTERN) - { - logger().warning("Pattern '%s' already present in the database from file %s",insert,fn); - } else { - logger().error("Failed to add '%s' from line %d from file %s",insert,lineno,fn); - } - } - } - lineno++; - } - } else - throw Poco::OpenFileException(fn); - df.close(); - logger().debug("Finish loading domains"); -} bool extFilter::loadACL(std::set *to_del) { diff --git a/src/notification.cpp b/src/notification.cpp index 0ec48e2..fb4e1e9 100644 --- a/src/notification.cpp +++ b/src/notification.cpp @@ -1,3 +1,22 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #include "notification.h" #include #include @@ -99,7 +118,7 @@ void NotifyManager::runTask() { _logger.error("Unable to find sender for group with id %d", notify_group); } else { - r->second->Redirect(pNotifyNf->user_port(), pNotifyNf->dst_port(), pNotifyNf->user_ip(), pNotifyNf->dst_ip(), pNotifyNf->ip_version(), pNotifyNf->acknum(), pNotifyNf->seqnum(), pNotifyNf->f_psh(), pNotifyNf->additional_param().c_str()); + r->second->HTTPRedirect(pNotifyNf->user_port(), pNotifyNf->dst_port(), pNotifyNf->user_ip(), pNotifyNf->dst_ip(), pNotifyNf->ip_version(), pNotifyNf->acknum(), pNotifyNf->seqnum(), pNotifyNf->f_psh(), pNotifyNf->additional_param().c_str(), pNotifyNf->additional_param().length()); // struct redirect_params rp = r->second; // std::string full_url("@HTTP/1.1 "+rp.code+"\r\nLocation: " + rp.redirect_url + pNotifyNf->additional_param() + "\r\nConnection: close\r\n"); // sender->Redirect(pNotifyNf->user_port(), pNotifyNf->dst_port(), pNotifyNf->user_ip(), pNotifyNf->dst_ip(), pNotifyNf->ip_version(), pNotifyNf->acknum(), pNotifyNf->seqnum(), pNotifyNf->f_psh(), full_url); diff --git a/src/reloadtask.cpp b/src/reloadtask.cpp index c930966..b9d258b 100644 --- a/src/reloadtask.cpp +++ b/src/reloadtask.cpp @@ -17,10 +17,8 @@ * */ -#include "dtypes.h" #include "reloadtask.h" #include "main.h" -#include "AhoCorasickPlus.h" #include "worker.h" #include "acl.h" @@ -58,63 +56,11 @@ void ReloadTask::runTask() } else { _logger.information("ACLs successfully loaded"); } - for(std::vector::iterator it=workerThreadVec.begin(); it != workerThreadVec.end(); it++) + if(_parent->getTriesManager()->getBLManager()->update()) { - if(dynamic_cast(*it) == nullptr) - continue; - WorkerConfig& config=(static_cast(*it))->getConfig(); - AhoCorasickPlus *to_del_atm; - AhoCorasickPlus *atm_new; - if(!_parent->getSSLFile().empty()) - { - atm_new = new AhoCorasickPlus(); - try - { - _parent->loadDomains(_parent->getSSLFile(), atm_new); - atm_new->finalize(); - to_del_atm = config.atmSSLDomains; - config.atmSSLDomains_new = atm_new; - rte_mb(); - int cnt = 15; - do { - sleep(10); - --cnt; - } while (cnt > 0 && config.atmSSLDomains_new != config.atmSSLDomains); - if(cnt == 0) - _logger.warning("Something wrong with worker thread on core %u", (*it)->getCoreId()); - delete to_del_atm; - _logger.information("Reloaded data for ssl domains list for core %u", (*it)->getCoreId()); - } catch (Poco::Exception &excep) - { - _logger.error("Got exception while reload ssl data: %s", excep.displayText()); - delete atm_new; - } - } - if(!_parent->getDomainsFile().empty() && !_parent->getURLsFile().empty()) - { - atm_new = new AhoCorasickPlus(); - try - { - _parent->loadDomainsURLs(_parent->getDomainsFile(), _parent->getURLsFile(), atm_new); - atm_new->finalize(); - to_del_atm = config.atm; - config.atm_new = atm_new; - rte_mb(); - int cnt = 15; - do { - sleep(10); - --cnt; - } while (cnt > 0 && config.atm_new != config.atm); - if(cnt == 0) - _logger.warning("Something wrong with worker thread on core %u", (*it)->getCoreId()); - delete to_del_atm; - _logger.information("Reloaded data for domains and urls list for core %u", (*it)->getCoreId()); - } catch (Poco::Exception &excep) - { - _logger.error("Got exception while reload domains and urls data: %s", excep.displayText()); - delete atm_new; - } - } + _logger.error("Unable to update blacklists"); + } else { + _logger.information("Blacklists successfully loaded"); } for(auto it = to_del.begin(); it != to_del.end(); it++) { diff --git a/src/sender.cpp b/src/sender.cpp index 6222e11..34be11f 100644 --- a/src/sender.cpp +++ b/src/sender.cpp @@ -17,15 +17,9 @@ * */ -#define __STDC_FORMAT_MACROS -#include #include "sender.h" #include -#include #include -#include -#include -#include #include #include #include "worker.h" @@ -50,8 +44,6 @@ struct ipv6_pseudo_hdr BSender::BSender(const char *cn, struct params &prm) : _logger(Poco::Logger::get(cn)), _parameters(prm) { - this->rHeader = "HTTP/1.1 "+_parameters.code+"\r\nLocation: " + _parameters.redirect_url + "\r\nConnection: close\r\n\r\n"; - _logger.debug("Default header is %s", rHeader); pkt_id = 1; } @@ -60,97 +52,12 @@ BSender::~BSender() } -int BSender::makePacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, std::string &dt, int f_reset, int f_psh, uint8_t *buffer) -{ - int pkt_len; - pkt_id++; - - // IP header - struct iphdr *iph = (struct iphdr *) buffer; - struct ip6_hdr *iph6 = (struct ip6_hdr *) buffer; - - // TCP header - struct tcphdr *tcph = (struct tcphdr *) (buffer + (ip_ver == 4 ? sizeof(struct iphdr) : sizeof(struct ip6_hdr))); - - int payloadlen=dt.size(); - // Data part - uint8_t *data = (uint8_t *)tcph + sizeof(struct tcphdr); - - if(!dt.empty()) - rte_memcpy(data, dt.c_str(), payloadlen); - - if(_logger.getLevel() == Poco::Message::PRIO_DEBUG) - { - Poco::Net::IPAddress ipa(ip_to, ip_ver == 4 ? sizeof(in_addr) : sizeof(in6_addr)); - _logger.debug("Trying to send packet to %s port %d", ipa.toString(), port_to); - } - if(ip_ver == 4) - { - // Fill the IPv4 header - iph->ihl = 5; - iph->version = 4; - iph->tos=0; - iph->tot_len = rte_cpu_to_be_16(sizeof(struct iphdr) + sizeof(struct tcphdr) + payloadlen); - iph->id = rte_cpu_to_be_16(pkt_id); - iph->frag_off = 0; - iph->ttl = _parameters.ttl; - iph->protocol = IPPROTO_TCP; - iph->check = 0; - iph->saddr = ((in_addr *)ip_from)->s_addr; - iph->daddr = ((in_addr *)ip_to)->s_addr;; - pkt_len = sizeof(struct iphdr) + sizeof(struct tcphdr) + payloadlen; - } else { - // IPv6 version (4 bits), Traffic class (8 bits), Flow label (20 bits) - iph6->ip6_flow = htonl ((6 << 28) | (0 << 20) | 0); - // Payload length (16 bits): TCP header + TCP data - iph6->ip6_plen = rte_cpu_to_be_16 (sizeof(struct tcphdr) + payloadlen); - // Next header (8 bits): 6 for TCP - iph6->ip6_nxt = IPPROTO_TCP; - // Hop limit (8 bits): default to maximum value - iph6->ip6_hops = 250; - rte_mov16((uint8_t *)&iph6->ip6_src, (uint8_t *)ip_from); - rte_mov16((uint8_t *)&iph6->ip6_dst, (uint8_t *)ip_to); - pkt_len = (sizeof(struct ip6_hdr) + sizeof(struct tcphdr) + payloadlen); - } - // TCP Header - tcph->source = port_from; - tcph->dest = port_to; - tcph->seq = acknum; - tcph->doff = 5; - tcph->syn = 0; - tcph->res1 = 0; - tcph->res2 = 0; - tcph->rst = f_reset; - tcph->psh = f_psh; - if(f_reset) - { - tcph->ack = 1; - tcph->ack_seq = seqnum; - tcph->fin = 0; - tcph->window = rte_cpu_to_be_16(0xEF); - } else { - tcph->ack_seq = seqnum; - tcph->ack = 1; - tcph->fin = 1; - tcph->window = rte_cpu_to_be_16(5885); - } - tcph->urg = 0; - tcph->check = 0; - tcph->urg_ptr = 0; - - if(ip_ver == 4) - tcph->check = rte_ipv4_udptcp_cksum((const ipv4_hdr*)iph,tcph); - else - tcph->check = rte_ipv6_udptcp_cksum((const ipv6_hdr*)iph6,tcph); - return pkt_len; -} - -void BSender::sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, std::string &dt, int f_reset, int f_psh) +void BSender::sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, int f_reset, int f_psh) { uint8_t datagram[4096]; - int pkt_len = makePacket(ip_from, ip_to, ip_ver, port_from, port_to, acknum, seqnum, dt, f_reset, f_psh, &datagram[0]); + int pkt_len = makePacket(ip_from, ip_to, ip_ver, port_from, port_to, acknum, seqnum, dt_buf, dt_len, f_reset, f_psh, &datagram[0]); struct sockaddr_in sin; struct sockaddr_in6 sin6; @@ -187,40 +94,45 @@ void BSender::sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, return; } -void BSender::Redirect(int user_port, int dst_port, void *user_ip, void *dst_ip, int ip_ver, uint32_t acknum, uint32_t seqnum, int f_psh, const char *additional_param) + +void BSender::HTTPRedirect(int user_port, int dst_port, void *user_ip, void *dst_ip, int ip_ver, uint32_t acknum, uint32_t seqnum, int f_psh, const char *redir_url, size_t r_len) { - // формируем дополнительные параметры - std::string tstr = rHeader; - if(additional_param != nullptr && additional_param[0] == '@' && (tstr.length() < (_parameters.mtu - (ip_ver == 4 ? sizeof(struct iphdr) : sizeof(struct ip6_hdr)) + sizeof(struct tcphdr) - sizeof(struct ether_hdr)))) + char payload[OUR_PAYLOAD_SIZE]; + size_t payload_size = sizeof(f_lines) - 1; + const char *payload_ptr = f_lines; + if(redir_url != nullptr && r_len + OUR_REDIR_SIZE < OUR_PAYLOAD_SIZE) { - tstr.assign(additional_param+1); - } else { - if(additional_param != nullptr && _parameters.redirect_url[_parameters.redirect_url.length()-1] == '?' && (tstr.length() < (_parameters.mtu - (ip_ver == 4 ? sizeof(struct iphdr) : sizeof(struct ip6_hdr)) + sizeof(struct tcphdr) - sizeof(struct ether_hdr)))) - { - tstr = "HTTP/1.1 "+_parameters.code+"\r\nLocation: " + _parameters.redirect_url + additional_param + "\r\nConnection: close\r\n\r\n"; - } + rte_memcpy(payload, r_line1, sizeof(r_line1)-1); + rte_memcpy(payload + sizeof(r_line1) - 1, r_line2, sizeof(r_line2) -1); + rte_memcpy(payload + sizeof(r_line1) - 1 + sizeof(r_line2) - 1 , redir_url, r_len); + rte_memcpy(payload + sizeof(r_line1) - 1 + sizeof(r_line2) - 1 + r_len, r_line3, sizeof(r_line3) - 1); + payload_size = sizeof(r_line1) - 1 + sizeof(r_line2) - 1 + sizeof(r_line3) - 1 + r_len; + payload_ptr = payload; } - this->sendPacket(dst_ip, user_ip, ip_ver, dst_port, user_port, acknum, seqnum, tstr, 0, f_psh); - - // And reset session with server + this->sendPacket(dst_ip, user_ip, ip_ver, dst_port, user_port, acknum, seqnum, payload_ptr, payload_size, 0, f_psh); +// sendPacket(dst_ip, user_ip, ip_ver, dst_port, user_port, rte_cpu_to_be_32(rte_be_to_cpu_32(acknum) + payload_size), 0, nullptr, 0, 1, 0); // send rst... + // And reset session with server, if needed + if(_parameters.send_rst_to_server) + this->sendPacket(user_ip, dst_ip, ip_ver, user_port, dst_port, seqnum, acknum, nullptr, 0, 1, 0); +} + +void BSender::HTTPForbidden(int user_port, int dst_port, void *user_ip, void *dst_ip, int ip_ver, uint32_t acknum, uint32_t seqnum, int f_psh) +{ + this->sendPacket(dst_ip, user_ip, ip_ver, dst_port, user_port, acknum, seqnum, f_lines, sizeof(f_lines) - 1, 0, f_psh); + // And reset session with server, if needed if(_parameters.send_rst_to_server) { - std::string empty_str; - this->sendPacket(user_ip, dst_ip, ip_ver, user_port, dst_port, seqnum, acknum, empty_str, 1, 0); + this->sendPacket(user_ip, dst_ip, ip_ver, user_port, dst_port, seqnum, acknum, nullptr, 0, 1, 0); } - return; } - - void BSender::SendRST(int user_port, int dst_port, void *user_ip, void *dst_ip, int ip_ver, uint32_t acknum, uint32_t seqnum, int f_psh) { - std::string empty_str; // send rst to the client - this->sendPacket(dst_ip, user_ip, ip_ver, dst_port, user_port, acknum, seqnum, empty_str, 1, 0); + this->sendPacket(dst_ip, user_ip, ip_ver, dst_port, user_port, acknum, seqnum, nullptr, 0, 1, 0); // send rst to the server if(_parameters.send_rst_to_server) - this->sendPacket(user_ip, dst_ip, ip_ver, user_port, dst_port, seqnum, acknum, empty_str, 1, 0); + this->sendPacket(user_ip, dst_ip, ip_ver, user_port, dst_port, seqnum, acknum, nullptr, 0, 1, 0); } CSender::CSender(struct params &prm) : BSender("CSender", prm) @@ -268,8 +180,8 @@ DSender::DSender(struct BSender::params &prm, uint8_t port, uint8_t *mac, uint8_ _port(port), _mp(mp) { - memcpy(&_eth_hdr.s_addr, mac, 6); - memcpy(&_eth_hdr.d_addr, to_mac, 6); + rte_memcpy(&_eth_hdr.s_addr, mac, 6); + rte_memcpy(&_eth_hdr.d_addr, to_mac, 6); } DSender::~DSender() @@ -315,8 +227,8 @@ ESender::ESender(struct nparams &prm, uint8_t port, struct rte_mempool *mp, Work _mp(mp), _wt(wt) { - memcpy(&_eth_hdr.s_addr, prm.mac, 6); - memcpy(&_eth_hdr.d_addr, prm.to_mac, 6); + rte_memcpy(&_eth_hdr.s_addr, prm.mac, 6); + rte_memcpy(&_eth_hdr.d_addr, prm.to_mac, 6); } ESender::~ESender() @@ -324,7 +236,7 @@ ESender::~ESender() } -void ESender::sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, std::string &dt, int f_reset, int f_psh) +void ESender::sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, int port_to, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, int f_reset, int f_psh) { pkt_id++; struct rte_mbuf *pkt = rte_pktmbuf_alloc(_mp); @@ -336,10 +248,11 @@ void ESender::sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); uint8_t *pkt_buf = ((uint8_t *)eth_hdr + sizeof(struct ether_hdr)); - int pkt_len = makePacket(ip_from, ip_to, ip_ver, port_from, port_to, acknum, seqnum, dt, f_reset, f_psh, pkt_buf) + sizeof(struct ether_hdr); + int pkt_len = makePacket(ip_from, ip_to, ip_ver, port_from, port_to, acknum, seqnum, dt_buf, dt_len, f_reset, f_psh, pkt_buf) + sizeof(struct ether_hdr); pkt->data_len = pkt_len; pkt->pkt_len = pkt_len; - rte_memcpy(eth_hdr, &_eth_hdr, sizeof(struct ether_hdr)); + ether_addr_copy(&_eth_hdr.s_addr, ð_hdr->s_addr); + ether_addr_copy(&_eth_hdr.d_addr, ð_hdr->d_addr); if(ip_ver == 4) { eth_hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); @@ -352,13 +265,178 @@ void ESender::sendPacket(void *ip_from, void *ip_to, int ip_ver, int port_from, if(likely(_wt->_n_send_pkts < EXTFILTER_WORKER_BURST_SIZE)) { _wt->_sender_buf[_wt->_n_send_pkts] = pkt; - _wt->_n_send_pkts += 1; + _wt->_n_send_pkts++; + } else { + _logger.error("Can't send packet. Buffer is full."); + rte_pktmbuf_free(pkt); + return; + } + return; +} + +void ESender::sendPacketIPv4(const uint8_t *l3_pkt, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, bool f_reset, bool f_psh) +{ + struct rte_mbuf *pkt = rte_pktmbuf_alloc(_mp); + if(unlikely(pkt == nullptr)) + { + _logger.error("Unable to allocate buffer for the packet"); + return; + } + struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + uint8_t *pkt_buf = ((uint8_t *)eth_hdr + sizeof(struct ether_hdr)); + int pkt_len = makeSwapPacketIPv4(l3_pkt, acknum, seqnum, dt_buf, dt_len, f_reset, f_psh, pkt_buf) + sizeof(struct ether_hdr); + pkt->data_len = pkt_len; + pkt->pkt_len = pkt_len; + ether_addr_copy(&_eth_hdr.s_addr, ð_hdr->s_addr); + ether_addr_copy(&_eth_hdr.d_addr, ð_hdr->d_addr); + eth_hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); + struct ipv4_hdr *ip_hdr = (struct ipv4_hdr *) pkt_buf; + ip_hdr->hdr_checksum = rte_ipv4_cksum(ip_hdr); + + if(likely(_wt->_n_send_pkts < EXTFILTER_WORKER_BURST_SIZE)) + { + _wt->_sender_buf[_wt->_n_send_pkts] = pkt; + _wt->_n_send_pkts++; } else { _logger.error("Can't send packet. Buffer is full."); rte_pktmbuf_free(pkt); return; } + return; +} + +void ESender::sendPacketIPv6(const uint8_t *l3_pkt, uint32_t acknum, uint32_t seqnum, const char *dt_buf, size_t dt_len, bool f_reset, bool f_psh) +{ + struct rte_mbuf *pkt = rte_pktmbuf_alloc(_mp); + if(unlikely(pkt == nullptr)) + { + _logger.error("Unable to allocate buffer for the packet"); + return; + } + struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + uint8_t *pkt_buf = ((uint8_t *)eth_hdr + sizeof(struct ether_hdr)); + int pkt_len = makeSwapPacketIPv6(l3_pkt, acknum, seqnum, dt_buf, dt_len, f_reset, f_psh, pkt_buf) + sizeof(struct ether_hdr); + pkt->data_len = pkt_len; + pkt->pkt_len = pkt_len; + ether_addr_copy(&_eth_hdr.s_addr, ð_hdr->s_addr); + ether_addr_copy(&_eth_hdr.d_addr, ð_hdr->d_addr); + eth_hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6); + if(likely(_wt->_n_send_pkts < EXTFILTER_WORKER_BURST_SIZE)) + { + _wt->_sender_buf[_wt->_n_send_pkts] = pkt; + _wt->_n_send_pkts++; + } else { + _logger.error("Can't send packet. Buffer is full."); + rte_pktmbuf_free(pkt); + return; + } return; } +void ESender::HTTPRedirectIPv4(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, bool f_psh, const char *redir_url, size_t r_len) +{ + char payload[OUR_PAYLOAD_SIZE]; + size_t payload_size = sizeof(f_lines) - 1; + const char *payload_ptr = f_lines; + if(redir_url != nullptr && r_len + OUR_REDIR_SIZE < OUR_PAYLOAD_SIZE) + { + rte_memcpy(payload, r_line1, sizeof(r_line1)-1); + rte_memcpy(payload + sizeof(r_line1) - 1, r_line2, sizeof(r_line2) -1); + rte_memcpy(payload + sizeof(r_line1) - 1 + sizeof(r_line2) - 1 , redir_url, r_len); + rte_memcpy(payload + sizeof(r_line1) - 1 + sizeof(r_line2) - 1 + r_len, r_line3, sizeof(r_line3) - 1); + payload_size = sizeof(r_line1) - 1 + sizeof(r_line2) - 1 + sizeof(r_line3) - 1 + r_len; + payload_ptr = payload; + } + sendPacketIPv4(pkt, acknum, seqnum, payload_ptr, payload_size, false, f_psh); + if(global_prm->answer_duplication > 0) + { + for(uint8_t z = 0; z < global_prm->answer_duplication; z++) + sendPacketIPv4(pkt, acknum, seqnum, payload_ptr, payload_size, false, f_psh); + } +// sendPacketIPv4(pkt, rte_cpu_to_be_32(rte_be_to_cpu_32(acknum) + payload_size), 0, nullptr, 0, true, false); // send rst... + // And reset session with server, if needed + if(_parameters.send_rst_to_server) + this->sendPacketIPv4(pkt, seqnum, acknum, nullptr, 0, 1, 0); +} + +void ESender::HTTPForbiddenIPv4(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, bool f_psh) +{ + sendPacketIPv4(pkt, acknum, seqnum, f_lines, sizeof(f_lines)-1, 0, f_psh); + if(global_prm->answer_duplication > 0) + { + for(uint8_t z = 0; z < global_prm->answer_duplication; z++) + sendPacketIPv4(pkt, acknum, seqnum, f_lines, sizeof(f_lines)-1, 0, f_psh); + } + // And reset session with server, if needed + if(_parameters.send_rst_to_server) + this->sendPacketIPv4(pkt, seqnum, acknum, nullptr, 0, 1, 0); +} + +void ESender::HTTPRedirectIPv6(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, bool f_psh, const char *redir_url, size_t r_len) +{ + char payload[OUR_PAYLOAD_SIZE]; + size_t payload_size = sizeof(f_lines) - 1; + const char *payload_ptr = f_lines; + if(redir_url != nullptr && r_len + OUR_REDIR_SIZE < OUR_PAYLOAD_SIZE) + { + rte_memcpy(payload, r_line1, sizeof(r_line1)-1); + rte_memcpy(payload + sizeof(r_line1) - 1, r_line2, sizeof(r_line2) -1); + rte_memcpy(payload + sizeof(r_line1) - 1 + sizeof(r_line2) - 1 , redir_url, r_len); + rte_memcpy(payload + sizeof(r_line1) - 1 + sizeof(r_line2) - 1 + r_len, r_line3, sizeof(r_line3) - 1); + payload_size = sizeof(r_line1) - 1 + sizeof(r_line2) - 1 + sizeof(r_line3) - 1 + r_len; + payload_ptr = payload; + } + sendPacketIPv6(pkt, acknum, seqnum, payload_ptr, payload_size, false, f_psh); + if(global_prm->answer_duplication > 0) + { + for(uint8_t z = 0; z < global_prm->answer_duplication; z++) + sendPacketIPv6(pkt, acknum, seqnum, payload_ptr, payload_size, false, f_psh); + } +// sendPacketIPv6(pkt, rte_cpu_to_be_32(rte_be_to_cpu_32(acknum) + payload_size), 0, nullptr, 0, true, false); // send rst... + // And reset session with server, if needed + if(_parameters.send_rst_to_server) + sendPacketIPv6(pkt, seqnum, acknum, nullptr, 0, 1, 0); +} + +void ESender::HTTPForbiddenIPv6(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum, bool f_psh) +{ + sendPacketIPv6(pkt, acknum, seqnum, f_lines, sizeof(f_lines)-1, 0, f_psh); + if(global_prm->answer_duplication > 0) + { + for(uint8_t z = 0; z < global_prm->answer_duplication; z++) + sendPacketIPv6(pkt, acknum, seqnum, f_lines, sizeof(f_lines)-1, 0, f_psh); + } + // And reset session with server, if needed + if(_parameters.send_rst_to_server) + this->sendPacketIPv6(pkt, seqnum, acknum, nullptr, 0, 1, 0); +} + + +void ESender::SendRSTIPv4(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum) +{ + // send rst to the client + sendPacketIPv4(pkt, acknum, seqnum, nullptr, 0, true, false); + if(global_prm->answer_duplication > 0) + { + for(uint8_t z = 0; z < global_prm->answer_duplication; z++) + sendPacketIPv4(pkt, acknum, seqnum, nullptr, 0, true, false); + } + // send rst to the server + if(_parameters.send_rst_to_server) + sendPacketIPv4(pkt, seqnum, acknum, nullptr, 0, true, false); +} + +void ESender::SendRSTIPv6(const uint8_t *pkt, uint32_t acknum, uint32_t seqnum) +{ + // send rst to the client + sendPacketIPv6(pkt, acknum, seqnum, nullptr, 0, true, false); + if(global_prm->answer_duplication > 0) + { + for(uint8_t z = 0; z < global_prm->answer_duplication; z++) + sendPacketIPv6(pkt, acknum, seqnum, nullptr, 0, true, false); + } + // send rst to the server + if(_parameters.send_rst_to_server) + sendPacketIPv6(pkt, seqnum, acknum, nullptr, 0, true, false); +} diff --git a/src/statistictask.cpp b/src/statistictask.cpp index f8872cf..5488dfa 100644 --- a/src/statistictask.cpp +++ b/src/statistictask.cpp @@ -16,8 +16,6 @@ * along with this program. If not, see . * */ - - #define __STDC_FORMAT_MACROS #include #include @@ -116,15 +114,10 @@ void StatisticTask::OutStatistic() uint64_t ipv6_packets=0; uint64_t bytes=0; uint64_t matched_ip_port=0; - uint64_t matched_ssl=0; + uint64_t matched_ssl_sni=0; uint64_t matched_ssl_ip=0; - uint64_t matched_domains=0; - uint64_t matched_urls=0; - uint64_t redirected_domains=0; - uint64_t redirected_urls=0; + uint64_t redirected_http_bl=0; uint64_t sended_rst=0; - uint64_t active_flows=0; - uint64_t deleted_flows=0; uint64_t r_received_packets=0; uint64_t r_missed_packets=0; uint64_t r_rx_nombuf = 0; @@ -132,9 +125,11 @@ void StatisticTask::OutStatistic() uint64_t ipv4_fragments=0; uint64_t ipv6_fragments=0; uint64_t ipv4_short_packets=0; - uint64_t ndpi_ipv6_flows_count=0; - uint64_t ndpi_ipv4_flows_count=0; - + uint64_t matched_http_bl = 0; + uint64_t sended_forbidden_ipv4 = 0; + uint64_t sended_forbidden_ipv6 = 0; + uint64_t sended_rst_ipv4 = 0; + uint64_t sended_rst_ipv6 = 0; Poco::FileOutputStream os; if(!_statisticsFile.empty()) { @@ -175,26 +170,23 @@ void StatisticTask::OutStatistic() ipv6_packets += stats.ipv6_packets; bytes += stats.total_bytes; matched_ip_port += stats.matched_ip_port; - matched_ssl += stats.matched_ssl; + matched_ssl_sni += stats.matched_ssl_sni; matched_ssl_ip += stats.matched_ssl_ip; - matched_domains += stats.matched_domains; - matched_urls += stats.matched_urls; - redirected_domains += stats.redirected_domains; - redirected_urls += stats.redirected_urls; - sended_rst += stats.sended_rst; - active_flows += stats.ndpi_flows_count; - deleted_flows += stats.ndpi_flows_deleted; + matched_http_bl += stats.matched_http_bl_ipv4 + stats.matched_http_bl_ipv6; + redirected_http_bl += stats.redirected_http_bl_ipv4 + stats.redirected_http_bl_ipv6; + sended_rst += stats.sended_rst_ipv4 + stats.sended_rst_ipv6; + sended_rst_ipv4 += stats.sended_rst_ipv4; + sended_rst_ipv6 += stats.sended_rst_ipv6; ipv4_fragments += stats.ipv4_fragments; ipv6_fragments += stats.ipv6_fragments; ipv4_short_packets += stats.ipv4_short_packets; - ndpi_ipv6_flows_count += stats.ndpi_ipv6_flows_count; - ndpi_ipv4_flows_count += stats.ndpi_ipv4_flows_count; + sended_forbidden_ipv4 += stats.sended_forbidden_ipv4; + sended_forbidden_ipv6 += stats.sended_forbidden_ipv6; app.logger().information("Thread seen packets: %" PRIu64 ", IP packets: %" PRIu64 " (IPv4 packets: %" PRIu64 ", IPv6 packets: %" PRIu64 "), seen bytes: %" PRIu64 ", Average packet size: %" PRIu32 " bytes, Traffic throughput: %s pps", stats.total_packets, stats.ip_packets, stats.ipv4_packets, stats.ipv6_packets, stats.total_bytes, avg_pkt_size, formatPackets(t)); app.logger().information("Thread IPv4 fragments: %" PRIu64 ", IPv6 fragments: %" PRIu64 ", IPv4 short packets: %" PRIu64, stats.ipv4_fragments, stats.ipv6_fragments, stats.ipv4_short_packets); - app.logger().information("Thread matched by ip/port: %" PRIu64 ", matched by ssl: %" PRIu64 ", matched by ssl/ip: %" PRIu64 ", matched by domain: %" PRIu64 ", matched by url: %" PRIu64, stats.matched_ip_port, stats.matched_ssl, stats.matched_ssl_ip, stats.matched_domains, stats.matched_urls); - app.logger().information("Thread redirected domains: %" PRIu64 ", redirected urls: %" PRIu64 ", rst sended: %" PRIu64, stats.redirected_domains,stats.redirected_urls,stats.sended_rst); - app.logger().information("Thread active flows: %" PRIu64 " (IPv4 flows: %" PRIu64 ", IPv6 flows: %" PRIu64 "), deleted flows: %" PRIu64 " already detected blocked: %" PRIu64, stats.ndpi_flows_count, stats.ndpi_ipv4_flows_count, stats.ndpi_ipv6_flows_count, stats.ndpi_flows_deleted, stats.already_detected_blocked); + app.logger().information("Thread matched by ip/port: %" PRIu64 ", ssl SNI: %" PRIu64 ", ssl/ip: %" PRIu64 ", http IPv4: %" PRIu64 ", http IPv6: %" PRIu64, stats.matched_ip_port, stats.matched_ssl_sni, stats.matched_ssl_ip, stats.matched_http_bl_ipv4, stats.matched_http_bl_ipv6); + app.logger().information("Thread redirected blocked http IPv4: %" PRIu64 ", redirected http IPv6: %" PRIu64 ", sended forbidden IPv4: %" PRIu64 ", sended forbidden IPv6: %" PRIu64 ", rst sended IPv4: %" PRIu64 ", rst sended IPv6: %" PRIu64, stats.redirected_http_bl_ipv4, stats.redirected_http_bl_ipv6, stats.sended_forbidden_ipv4, stats.sended_forbidden_ipv6, stats.sended_rst_ipv4, stats.sended_rst_ipv6); if(stats.latency_counters.blocked_pkts != 0 && stats.latency_counters.total_pkts != 0) app.logger().information("Thread packets latency all packets: %" PRIu64 " cycles (%.0f ns), blocked packets: %" PRIu64 " (%.0f ns)", (stats.latency_counters.total_cycles / stats.latency_counters.total_pkts), cycles_to_ns(stats.latency_counters.total_cycles / stats.latency_counters.total_pkts), (stats.latency_counters.blocked_cycles / stats.latency_counters.blocked_pkts), cycles_to_ns(stats.latency_counters.blocked_cycles / stats.latency_counters.blocked_pkts)); if(!_statisticsFile.empty()) @@ -206,11 +198,10 @@ void StatisticTask::OutStatistic() os << worker_name << ".ipv6_packets=" << stats.ipv6_packets << std::endl; os << worker_name << ".total_bytes=" << stats.total_bytes << std::endl; os << worker_name << ".matched_ip_port=" << stats.matched_ip_port << std::endl; - os << worker_name << ".matched_ssl=" << stats.matched_ssl << std::endl; + os << worker_name << ".matched_ssl_sni=" << stats.matched_ssl_sni << std::endl; os << worker_name << ".matched_ssl_ip=" << stats.matched_ssl_ip << std::endl; - os << worker_name << ".matched_domains=" << stats.matched_domains << std::endl; - os << worker_name << ".matched_ulrs=" << stats.matched_urls << std::endl; - os << worker_name << ".active_flows=" << stats.ndpi_flows_count << std::endl; + os << worker_name << ".matched_http_bl_ipv4=" << stats.matched_http_bl_ipv4 << std::endl; + os << worker_name << ".matched_http_bl_ipv6=" << stats.matched_http_bl_ipv6 << std::endl; os << worker_name << ".ipv4_fragments=" << stats.ipv4_fragments << std::endl; os << worker_name << ".ipv6_fragments=" << stats.ipv6_fragments << std::endl; os << worker_name << ".ipv4_short_packets=" << stats.ipv4_short_packets << std::endl; @@ -220,9 +211,8 @@ void StatisticTask::OutStatistic() gettimeofday(&begin_time, NULL); app.logger().information("All worker threads seen packets: %" PRIu64 ", IP packets: %" PRIu64 " (IPv4 packets: %" PRIu64 ", IPv6 packets: %" PRIu64 "), seen bytes: %" PRIu64 ", traffic throughtput: %s pps", total_packets, ip_packets, ipv4_packets, ipv6_packets, bytes, formatPackets(traffic_throughput)); app.logger().information("All worker IPv4 fragments: %" PRIu64 ", IPv6 fragments: %" PRIu64 ", IPv4 short packets: %" PRIu64, ipv4_fragments, ipv6_fragments, ipv4_short_packets); - app.logger().information("All worker threads matched by ip/port: %" PRIu64 ", matched by ssl: %" PRIu64 ", matched by ssl/ip: %" PRIu64 ", matched by domain: %" PRIu64 ", matched by url: %" PRIu64, matched_ip_port, matched_ssl, matched_ssl_ip, matched_domains, matched_urls); - app.logger().information("All worker threads redirected domains: %" PRIu64 ", redirected urls: %" PRIu64 ", rst sended: %" PRIu64, redirected_domains, redirected_urls, sended_rst); - app.logger().information("All worker threads active flows: %" PRIu64 " (IPv4 flows: %" PRIu64 ", IPv6 flows: %" PRIu64 "), deletet flows: %" PRIu64 , active_flows, ndpi_ipv4_flows_count, ndpi_ipv6_flows_count, deleted_flows); + app.logger().information("All worker threads matched by ip/port: %" PRIu64 ", matched by ssl SNI: %" PRIu64 ", matched by ssl/ip: %" PRIu64 ", matched by HTTP: %" PRIu64, matched_ip_port, matched_ssl_sni, matched_ssl_ip, matched_http_bl); + app.logger().information("All worker threads redirected blocked http: %" PRIu64 ", sended forbidden IPv4: %" PRIu64 ", sended forbidden IPv6: %" PRIu64 ", rst sended IPv4: %" PRIu64 ", rst sended IPv6: %" PRIu64, redirected_http_bl, sended_forbidden_ipv4, sended_forbidden_ipv6, sended_rst_ipv4, sended_rst_ipv6); if(!_statisticsFile.empty()) { std::string worker_name("allworkers"); @@ -232,11 +222,9 @@ void StatisticTask::OutStatistic() os << worker_name << ".ipv6_packets=" << ipv6_packets << std::endl; os << worker_name << ".total_bytes=" << bytes << std::endl; os << worker_name << ".matched_ip_port=" << matched_ip_port << std::endl; - os << worker_name << ".matched_ssl=" << matched_ssl << std::endl; + os << worker_name << ".matched_ssl_sni=" << matched_ssl_sni << std::endl; os << worker_name << ".matched_ssl_ip=" << matched_ssl_ip << std::endl; - os << worker_name << ".matched_domains=" << matched_domains << std::endl; - os << worker_name << ".matched_ulrs=" << matched_urls << std::endl; - os << worker_name << ".active_flows=" << active_flows << std::endl; + os << worker_name << ".matched_http_bl=" << matched_http_bl << std::endl; os << worker_name << ".ipv4_fragments=" << ipv4_fragments << std::endl; os << worker_name << ".ipv6_fragments=" << ipv6_fragments << std::endl; os << worker_name << ".ipv4_short_packets=" << ipv4_short_packets << std::endl; diff --git a/src/tries.cpp b/src/tries.cpp new file mode 100644 index 0000000..42a3ee5 --- /dev/null +++ b/src/tries.cpp @@ -0,0 +1,379 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#include "tries.h" +#include +#include +#include +#include +#include +#include "utils.h" +#include +#include +#include + +// value 1-5 +#define TRIES_NUMBER 1 + +static int param_num_tries = TRIES_NUMBER; +static marisa::TailMode param_tail_mode = MARISA_TEXT_TAIL; +static marisa::NodeOrder param_node_order = MARISA_WEIGHT_ORDER; +static marisa::CacheLevel param_cache_level = MARISA_DEFAULT_CACHE; + +bool Tries::search_prefix(marisa::Agent *agent, char *rhost, std::size_t rhost_size, char *url, std::size_t url_size) +{ + if(_is_mdomains_ready) + { + if(mdomains_trie.size() != 0 && (agent->set_query(rhost, rhost_size), mdomains_trie.common_prefix_search(*agent) )) + { + return true; + } + } + if(_is_urls_ready) + { + if(urls_trie.size() != 0) + { + agent->set_query(url, url_size); + return urls_trie.common_prefix_search(*agent); + } + } + return false; +} + +bool Tries::lookup(marisa::Agent *agent, char *rhost, std::size_t rhost_size, char *url, std::size_t url_size) +{ + if(_is_mdomains_ready) + { + if(mdomains_trie.size() != 0 && (agent->set_query(rhost, rhost_size), mdomains_trie.common_prefix_search(*agent) )) + { + return true; + } + } + if(_is_urls_ready) + { + if(urls_trie.size() != 0) + { + agent->set_query(url, url_size); + return urls_trie.lookup(*agent); + } + } + return false; +} + +TriesControl::TriesControl(): + active_trie(0), + _logger(Poco::Logger::get("TriesControl")), + _first_load(true), + _domains_ch_time(0), + _urls_ch_time(0), + load_time(0) +{ +} + + +int read_keys(std::istream &input, marisa::Keyset *m_domains, marisa::Keyset *urls) +{ + int lines = 0; + std::string line; + while (std::getline(input, line)) + { + lines++; + if(line[0] == '#' || line[0] == ';') + continue; + std::size_t pos = line.find("*."); + if(pos != line.npos) + { + std::string s = line.substr(pos+1, line.length()-1); + urls->push_back(s.c_str()+1, s.length()-1); // store domain without previous dot + std::reverse(s.begin(), s.end()); + m_domains->push_back(s.c_str(), s.length()); // store reverse + } else { + urls->push_back(line.c_str(), line.length()); + } + } + return lines; +} + +bool TriesControl::load(std::string &domains_f, std::string &urls_f) +{ + marisa::Keyset m_domains; + marisa::Keyset urls; + int domains_lines = 0; + int urls_lines = 0; + struct stat f_stat; + if(!domains_f.empty()) + { + if(stat(domains_f.c_str(), &f_stat)) + { + _logger.error("Unable to stat file '%s'", domains_f); + return true; + } + if(_domains_ch_time != 0 && _domains_ch_time == f_stat.st_mtim.tv_sec) + { + _logger.information("Domains file '%s' is not changed from the last load", domains_f); + } else { + try + { + std::ifstream domains_file(domains_f, std::ios::binary); + if(!domains_file) + { + _logger.error("Failed to open domains file '%s'", domains_f); + return true; + } + domains_lines = read_keys(domains_file, &m_domains, &urls); + } catch (const marisa::Exception &ex) + { + _logger.error("Working with domains failed: %s", std::string(ex.what())); + return true; + } catch (...) + { + _logger.error("Exception occured while working with domains file"); + return true; + } + _domains_ch_time = f_stat.st_mtim.tv_sec; + _logger.information("Loaded %d lines from the domains file '%s'", domains_lines, domains_f); + } + } + if(!urls_f.empty()) + { + if(stat(urls_f.c_str(), &f_stat)) + { + _logger.error("Unable to stat file '%s'", urls_f); + return true; + } + if(_urls_ch_time != 0 && _urls_ch_time == f_stat.st_mtim.tv_sec) + { + _logger.information("URLs file '%s' is not changed from the last load", urls_f); + } else { + try + { + std::ifstream urls_file(urls_f, std::ios::binary); + if(!urls_file) + { + _logger.error("Failed to open urls file '%s'", urls_f); + return true; + } + urls_lines = read_keys(urls_file, &m_domains, &urls); + } catch (const marisa::Exception &ex) + { + _logger.error("Working with urls failed: %s", std::string(ex.what())); + return true; + } catch (...) + { + _logger.error("Exception occured while working with urls file"); + return true; + } + _urls_ch_time = f_stat.st_mtim.tv_sec; + _logger.information("Loaded %d lines from the urls file '%s'", urls_lines, urls_f); + } + } + uint8_t need_load_trie = 0; + uint8_t need_del_trie = (active_trie != 0); + if(!_first_load) // not first load + need_load_trie = (active_trie == 0); + if(!m_domains.empty()) + { + try + { + tries[need_load_trie].mdomains_trie.build(m_domains, param_num_tries | param_tail_mode | param_node_order | param_cache_level); + tries[need_load_trie].setMDomainsReady(); + _logger.information("Masked domains: #keys %z, #nodes %z", tries[need_load_trie].mdomains_trie.num_keys(), tries[need_load_trie].mdomains_trie.num_nodes()); + } catch (const marisa::Exception &ex) + { + _logger.error("Unable to build marisa trie for masked domains: %s", std::string(ex.what())); + return true; + } + + } + if(!urls.empty()) + { + try + { + tries[need_load_trie].urls_trie.build(urls, param_num_tries | param_tail_mode | param_node_order | param_cache_level); + tries[need_load_trie].setURLsReady(); + _logger.information("URLS: #keys %z, #nodes %z", tries[need_load_trie].urls_trie.num_keys(), tries[need_load_trie].urls_trie.num_nodes()); + } catch (const marisa::Exception &ex) + { + _logger.error("Unable to build marisa trie for urls: %s", std::string(ex.what())); + return true; + } + } + if(m_domains.empty() && urls.empty()) // nothing to do... + return false; + active_trie = need_load_trie; + _logger.information("Set active trie in the slot %d", (int) active_trie); + if(!_first_load) // not first load + { + sleep(1); + _logger.information("Deleting trie in the slot %d", (int) need_del_trie); + tries[need_del_trie].mdomains_trie.clear(); + tries[need_del_trie].urls_trie.clear(); + } + _first_load = false; + return false; +} + +int TriesManager::checkURLBlocked(int thread_id, const char *hostname, uint32_t host_len, const char *uri, uint32_t uri_len, char **redir_url) +{ + char *rhost; + char revhostname[4097]; + char url_buf[4097]; + int buf_len; + char *url_entry; + char enc_url[4096]; + revhostname[sizeof(revhostname)-1] = 0; + if(host_len > sizeof(revhostname)-1) + { + buf_len = sizeof(revhostname)-1; + host_len = sizeof(revhostname)-1; + host_len = sizeof(revhostname)-1; + url_entry = &url_buf[sizeof(revhostname)-1]; + rhost = &revhostname[0]; + uri_len = 0; + } else { + if(host_len + uri_len > sizeof(url_buf)-1) + uri_len = sizeof(url_buf) - 1 - host_len; + buf_len = host_len + uri_len; + url_entry = &url_buf[host_len]; + rhost = &revhostname[sizeof(revhostname) - 1 - host_len]; + } + // перевод hostname в маленькие буквы и копирование hostname в обратном порядке в revhostname + char *rh = &revhostname[sizeof(revhostname)-2]; + for(uint32_t i = 0; i < host_len; i++, rh--) + { + char v = hostname[i]; + if( v >= 'A' && v <= 'Z') + v |= 0x20; + url_buf[i] = v; + *rh = v; + } + if(uri_len != 0) + { + if(uri_len == 1) + *url_entry = *uri; + else + rte_memcpy(url_entry, uri, uri_len); + } + url_buf[buf_len] = 0; + try + { + if(_bl_manager.getHttpBlacklist()->getActiveTrie()->search_prefix(&_agents[thread_id], rhost, host_len, url_buf, buf_len)) + { + struct BlacklistsManager::bl_service_profile *sp = _bl_manager.getActiveSP(); + if(redir_url && sp->redir_url[0] != 0) // make redir url + { + char *r_url = &_url[thread_id][0]; + int res = sp->redir_url_size; + rte_memcpy(r_url, sp->redir_url, res); + if(sp->need_add_url) + { + size_t enc_len = url_encode(enc_url, url_buf, buf_len, sizeof(enc_url) - 1); + char *cptr = r_url + sp->redir_url_size; + res += sizeof(uri_p) - 1 + enc_len; + rte_memcpy(cptr, uri_p, sizeof(uri_p) - 1); + cptr += sizeof(uri_p) - 1; + rte_memcpy(cptr, enc_url, enc_len); + } + *redir_url = r_url; + return res; + } + return 1; + } + } catch (const marisa::Exception &ex) + { + _logger.error("Exception occured while search http: %s", std::string(ex.what())); + } + return 0; +} + +int TriesManager::checkSNIBlocked(int thread_id, const char *sni, uint32_t sni_len) +{ + char revsni[4097]; + revsni[sizeof(revsni)-1] = 0; + char *rptr = &revsni[sizeof(revsni)-2]; + if(sni_len > sizeof(revsni)-1) + sni_len = sizeof(revsni)-1; + for(uint32_t z=0; z < sni_len; z++, rptr--) + { + *rptr = sni[z]; + } + try + { +// if(_bl_manager.getSNIBlacklist()->getActiveTrie()->search_prefix(&_agents[thread_id], (rptr+1), sni_len, (char *)sni, sni_len)) + if(_bl_manager.getSNIBlacklist()->getActiveTrie()->lookup(&_agents[thread_id], (rptr+1), sni_len, (char *)sni, sni_len)) + { + return 1; + } + } catch (const marisa::Exception &ex) + { + _logger.error("Exception occured while search sni: %s", std::string(ex.what())); + } + return 0; +} + +void BlacklistsManager::fillRedirURL(uint8_t profile, const char *redir_url, size_t url_length) +{ + _sp[profile].redir_url_size = url_length; + if(!memcmp(redir_url, "http://", 7) || !memcmp(redir_url, "https://", 8)) + { + strncpy(_sp[profile].redir_url, redir_url, sizeof(_sp[profile].redir_url)-1); + } else { + strncpy(stpcpy(_sp[profile].redir_url, "http://"), redir_url, sizeof(_sp[profile].redir_url)-1-7); + _sp[profile].redir_url_size += 7; + } + _sp[profile].need_add_url = false; + if(redir_url[url_length-1] == '?' || redir_url[url_length-1] == '&') + _sp[profile].need_add_url = true; + if(_sp[profile].redir_url_size > sizeof(_sp[profile].redir_url)-1) + _sp[profile].redir_url_size = sizeof(_sp[profile].redir_url)-1; +} + +void BlacklistsManager::fillProfile(uint8_t profile, std::string &_domains_file, std::string &_urls_file, std::string &_sni_file, const char *redir_url, size_t url_length) +{ + _sp[profile].domains_file = _domains_file; + _sp[profile].urls_file = _urls_file; + _sp[profile].sni_file = _sni_file; + if(redir_url && url_length != 0 && redir_url[0] != 0) + { + fillRedirURL(profile, redir_url, url_length); + } else { + _sp[profile].redir_url[0] = 0; + _sp[profile].redir_url_size = 0; + _sp[profile].need_add_url = false; + } + +} + +bool BlacklistsManager::init(std::string &_domains_file, std::string &_urls_file, std::string &_sni_file, const char *redir_url, size_t url_length) +{ + fillProfile(0, _domains_file, _urls_file, _sni_file, redir_url, url_length); + _http_bl.load(_domains_file, _urls_file); + std::string empty_s; + _sni_bl.load(_sni_file, empty_s); + _active_profile = 0; + return false; +} + +bool BlacklistsManager::update() +{ + if(_http_bl.load(_sp[_active_profile].domains_file, _sp[_active_profile].urls_file)) + return true; + std::string empty_s; + return _sni_bl.load(_sp[_active_profile].sni_file, empty_s); +} diff --git a/src/utils.cpp b/src/utils.cpp new file mode 100644 index 0000000..89e2a46 --- /dev/null +++ b/src/utils.cpp @@ -0,0 +1,49 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + +#include "utils.h" + +#include +static char hex_chars[]="0123456789ABCDEF"; + +std::size_t url_encode(char *buf, const char *from, std::size_t len, std::size_t buf_size) +{ + std::size_t res = 0; + if(from) + { + while (*from != 0 && len > 0 && buf_size > 0) + { + res++; + buf_size--; + if(isalnum(*from) || *from == '-' || *from == '_' || *from == '.' || *from == '~') + *buf++ = *from; + else { + *buf++ = '%'; + *buf++ = hex_chars[(*from >> 4) & 0x0f]; + *buf++ = hex_chars[*from & 0x0f]; + res += 2; + buf_size -= 2; + } + from++; + len--; + } + } + *buf = 0; + return res; +} \ No newline at end of file diff --git a/src/worker.cpp b/src/worker.cpp index fb19bc1..ec81a41 100644 --- a/src/worker.cpp +++ b/src/worker.cpp @@ -1,3 +1,22 @@ +/* +* +* Copyright (C) Max +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +*/ + #define __STDC_FORMAT_MACROS #include #include @@ -24,7 +43,9 @@ #include "acl.h" #include #include "notification.h" -#include "dpi.h" +#include "utils.h" +#include "http.h" +#include "dtypes.h" #define tcphdr(x) ((struct tcphdr *)(x)) @@ -35,358 +56,114 @@ inline u_int8_t ext_dpi_v6_addresses_equal(uint64_t *x, uint64_t *y) return 0; } -void host_cb(dpi_http_message_informations_t* http_informations, const u_char* app_data, u_int32_t data_length, dpi_pkt_infos_t* pkt, void** flow_specific_user_data, void* user_data) +int on_header_complete_ext(http_parser* p, dpi_pkt_infos_t* pkt_informations, void** flow_specific_user_data, void* user_data) { - if(*flow_specific_user_data != NULL && data_length > 0 && (http_informations->method_or_code == DPI_HTTP_POST || http_informations->method_or_code == DPI_HTTP_GET || http_informations->method_or_code == DPI_HTTP_HEAD)) + if(*flow_specific_user_data != NULL) { - struct dpi_flow_info *u = (struct dpi_flow_info *)*flow_specific_user_data; WorkerThread *obj = (WorkerThread *) user_data; - std::string &uri = obj->getUri(); - uri.assign("http://", 7); - uri.append((char *)app_data, data_length); - uri.append(u->url, u->url_size); - obj->setNeedBlock(obj->checkHTTP(uri, pkt)); - } -} - -void url_cb_mempool(const unsigned char* url, u_int32_t url_length, dpi_pkt_infos_t* pkt_informations, void** flow_specific_user_data, void* user_data) -{ - if(url_length == 0) - return ; - WorkerThread *obj = (WorkerThread *) user_data; - struct dpi_flow_info *u = (struct dpi_flow_info *) *flow_specific_user_data; - if(u == nullptr) - { - if(rte_mempool_get(obj->getDPIMempool(), (void **)&u) != 0) - { - obj->getStats().dpi_no_mempool_http++; - return ; - } else { - memset(u, 0, sizeof(dpi_flow_info)); - u->dpi_mempool = obj->getDPIMempool(); - *flow_specific_user_data = u; - } - } - struct rte_mempool *mempool = obj->getUrlMempool(); - if(url_length+1 > obj->getConfig().maximum_url_size) - { - url_length = obj->getConfig().maximum_url_size; - } - u->mempool = mempool; - if(u->url == nullptr) - { - if(mempool != nullptr) - { - if(rte_mempool_get(mempool, (void **)&u->url) != 0) - u->mempool = nullptr; - else - u->use_pool = true; - } - if(!u->use_pool) - { - obj->getStats().dpi_use_url_malloc++; - u->url = (char *)malloc(url_length+1); - } - } else { - if(!u->use_pool) - { - if((url_length+1) > ((u_int32_t)u->url_size+1)) - u->url = (char *)realloc(u->url, url_length+1); - } + struct http::http_req_buf *d = (struct http::http_req_buf *) *flow_specific_user_data; + obj->setNeedBlock(obj->checkURLBlocked(d->host_r.buf, d->host_r.length, d->uri.buf, d->uri.length, pkt_informations)); } - memcpy(u->url, url, url_length); - u->url_size = url_length; + return 1; // no need to check body... } -void url_cb(const unsigned char* url, u_int32_t url_length, dpi_pkt_infos_t* pkt_informations, void** flow_specific_user_data, void* user_data) -{ - if(url_length == 0) - return ; - struct dpi_flow_info *u = (struct dpi_flow_info *) *flow_specific_user_data; - if(u == nullptr) - { - u = (struct dpi_flow_info *)calloc(1, sizeof(dpi_flow_info)); - *flow_specific_user_data = u; - } - WorkerThread *obj = (WorkerThread *) user_data; - struct rte_mempool *mempool = obj->getUrlMempool(); - if(url_length+1 > obj->getConfig().maximum_url_size) - { - url_length = obj->getConfig().maximum_url_size; - } - u->mempool = mempool; - if(u->url == nullptr) - { - if(mempool != nullptr) - { - if(rte_mempool_get(mempool, (void **)&u->url) != 0) - u->mempool = nullptr; - else - u->use_pool = true; - } - if(!u->use_pool) - { - obj->getStats().dpi_use_url_malloc++; - u->url = (char *)malloc(url_length+1); - } - } else { - if(!u->use_pool) - { - if((url_length+1) > ((u_int32_t)u->url_size+1)) - u->url = (char *)realloc(u->url, url_length+1); - } - } - memcpy(u->url, url, url_length); - u->url_size = url_length; -} - void ssl_cert_cb(char *certificate, int size, void *user_data, dpi_pkt_infos_t *pkt) { WorkerThread *obj = (WorkerThread *) user_data; - std::string &cert = obj->getCert(); - cert.assign(certificate, size > 255 ? 255 : size); - obj->setNeedBlock(obj->checkSSL(cert, pkt)); + obj->setNeedBlock(obj->checkSNIBlocked((const char *)certificate, size > 255 ? 255 : size, pkt)); } -WorkerThread::WorkerThread(const std::string& name, WorkerConfig &workerConfig, dpi_library_state_t* state, int socketid, flowHash *fh, struct ESender::nparams &sp, struct rte_mempool *mp, struct rte_mempool *url_mempool, struct rte_mempool *dpi_mempool) : +WorkerThread::WorkerThread(uint8_t worker_id,const std::string& name, WorkerConfig &workerConfig, dpi_library_state_t* state, int socketid, struct ESender::nparams &sp, struct rte_mempool *mp, struct rte_mempool *dpi_http_mempool) : m_WorkerConfig(workerConfig), m_Stop(true), _logger(Poco::Logger::get(name)), dpi_state(state), _name(name), - m_FlowHash(fh), - _n_send_pkts(0) + _n_send_pkts(0), + _worker_id(worker_id) { - uri.reserve(URI_RESERVATION_SIZE); - certificate.reserve(CERT_RESERVATION_SIZE); - - // setup peafowl - static dpi_http_header_field_callback* single_cb[1]={&host_cb}; + static dpi_external_http_callbacks_t ext_callbacks = { + .on_url = http::on_url_ext, + .on_header_field = http::on_header_field_ext, + .on_header_value = http::on_header_value_ext, + .on_headers_complete = on_header_complete_ext + }; + dpi_http_activate_ext_callbacks(dpi_state, &ext_callbacks, this); - static const char* headers[1]={"host"}; - static dpi_http_callbacks_t callback={.header_url_callback = (dpi_mempool == nullptr ? url_cb : url_cb_mempool), .header_names = headers, .num_header_types = 1, .header_types_callbacks = single_cb, .header_completion_callback = 0, .http_body_callback = 0}; - dpi_http_activate_callbacks(dpi_state, &callback, this); static dpi_ssl_callbacks_t ssl_callback = {.certificate_callback = ssl_cert_cb }; dpi_ssl_activate_callbacks(state, &ssl_callback, this); - // setup hash - std::string mem_name("IPv4Flows_"+name); - ipv4_flows = (struct ext_dpi_flow_info **)rte_zmalloc_socket(mem_name.c_str(), fh->getHashSizeIPv4()*sizeof(struct ext_dpi_flow_info *), RTE_CACHE_LINE_SIZE, socketid); - if(ipv4_flows == nullptr) - { - _logger.fatal("Not enough memory for ipv4 flows"); - throw Poco::Exception("Not enough memory for ipv4 flows"); - } - mem_name.assign("IPv6Flows_"+name); - ipv6_flows = (struct ext_dpi_flow_info **)rte_zmalloc_socket(mem_name.c_str(), fh->getHashSizeIPv6()*sizeof(struct ext_dpi_flow_info *), RTE_CACHE_LINE_SIZE, socketid); - if(ipv6_flows == nullptr) - { - _logger.fatal("Not enough memory for ipv6 flows"); - throw Poco::Exception("Not enough memory for ipv6 flows"); - } - _logger.debug("Allocating %d bytes for flow pool", (int) ((fh->getHashSizeIPv4() + fh->getHashSizeIPv6())*sizeof(struct ext_dpi_flow_info))); - std::string mempool_name("flows_pool_" + name); - flows_pool = rte_mempool_create(mempool_name.c_str(), (fh->getHashSizeIPv4() + fh->getHashSizeIPv6()), sizeof(struct ext_dpi_flow_info), 0, 0, NULL, NULL, NULL, NULL, socketid, 0); - if(flows_pool == nullptr) - { - _logger.fatal("Not enough memory for flows pool. Tried to allocate %d bytes on socket %d", (int) ((fh->getHashSizeIPv4() + fh->getHashSizeIPv6())*sizeof(struct ext_dpi_flow_info)), socketid); - throw Poco::Exception("Not enough memory for flows pool"); - } + ipv4_flow_mask = global_prm->memory_configs.ipv4.mask_parts_flow; + ipv6_flow_mask = global_prm->memory_configs.ipv6.mask_parts_flow; + if(mp != nullptr) { // setup sender _snd = new ESender(sp, m_WorkerConfig.sender_port, mp, this); } else { - _snd = nullptr; + throw Poco::Exception("ESender is null!"); } - _url_mempool = url_mempool; - _dpi_mempool = dpi_mempool; - uri_p = new Poco::URI("http://www.longurlmakerrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr.com/go?id=83olengthy1195stretchingShortURL2s11Beam.tocYepItShortlinksqstretchxrunning1c3stretched2sy436vxfaraway0096ShredURLFwdURL4n111aLiteURL09f0307whighgreatrangy5erunningEasyURL0b6sh6aprotracted1140elongated120towering3DecentURL1g0remotea14great168lEasyURL4continued70f6runningURLviUlimit20v94prolonged0r07Ne1stretcheddistant6URLCutterhighEasyURLlnk.inextensivedestretched1003bShim1041FhURLa39aURLvirunning590kShrinkURLaa1w7elongated010lofty1312549h6bShim9045ar01drawn%2Bout0g8egj10PiURL2bf113protracted154100flingeringShim7prolongedspread%2Bout10301URL8loftysustainedenduringdeep0SHurlaf043far%2Breachingo1deep17enlargede01drawn%2Bout1d57lnk.inSHurlfar%2Breaching001Smallrk2runningDigBigSimURLEasyURLflengthenedURLPie004301URL0spun%2Boutwt1expandede0910GetShortytowering0distant6ffhigheShim2loftyspun%2Bout1NanoRef1401spread%2Boutcetallexpanded5stretched0RubyURLURLHawkloftyaB654UrlTea0URLcut1prolonged2dx7SHurl74j41c2301URLWapURL60lDoiopMyURLTightURL01Redirx21stringyDoiopURLvi4YepItb0URLcut0620stretchingd180lengthened2171FwdURLc1b5URLHawk35lingeringCanURLdrawn%2Boutlengthened0c0rangySimURLprotracted78440muganglingShrtnds2oa00greatb30hyfar%2Breaching1k7Smallr110o715far%2Bofflingering41elongate9k1running3TraceURL3towering6rangy0lanky1EasyURLURLHawkstretchingstretch076jdeep151far%2BofffShortURL05TinyLink78f32715ufdistantprolongedstretchingwd30lengthened1elongated0c8NanoRefsustained7Metamark3w9301URLIs.gd11URL.co.ukDecentURL5extensive1ShoterLinkShorl00v39lengthyntall8f0041f6d5prolonged111EasyURLcontinuedShortlinks4c4408stringym5d0drawn%2Boutf9dShrinkURLURLCutterURLCutter3agangling3SnipURL0G8L00adiYepIt0Minilien91l1URLPie0SnipURLlofty00Shim5hdeepsa1continuedprotracted15765fSnipURLA2Nfar%2Boff1qfar%2Boffstretchinglengthyfar%2Boffc78drawn%2Bout21outstretchedspun%2Boutz52sremoteremoteprolongedeq0yUlimitb1B651CanURL6sustainedj02h117010URLHawk8high0outstretched8aafvstretch0037runningaextensive9ndeep0U7611yab5URl.ieShortenURLsustainedShredURLx60WapURL8aremote9expanded2tall09601gangling21A2N9d48rangysustained36far%2Breachingstretching2lengthened41NotLong11210Ulimit0814Is.gdPiURL89"); + _dpi_http_mempool = dpi_http_mempool; } WorkerThread::~WorkerThread() { dpi_terminate(dpi_state); - delete uri_p; if(_snd != nullptr) delete _snd; } -bool WorkerThread::checkSSL(std::string &certificate, dpi_pkt_infos_t *pkt) +bool WorkerThread::checkSNIBlocked(const char *sni, size_t sni_len, dpi_pkt_infos_t* pkt) { - struct ipv4_hdr *ipv4_header = (struct ipv4_hdr *) pkt->pkt; - struct ipv6_hdr *ipv6_header = (struct ipv6_hdr *) pkt->pkt; - struct tcphdr* tcph; - tcph = (struct tcphdr *)((uint8_t *) pkt->pkt + (pkt->ip_version == 4 ? sizeof(struct ipv4_hdr) : sizeof(struct ipv6_hdr))); - - if(likely(m_WorkerConfig.atmSSLDomains != nullptr)) + if(extFilter::instance()->getTriesManager()->checkSNIBlocked(getWorkerID(), sni, sni_len)) { - if(m_WorkerConfig.lower_host) - std::transform(certificate.begin(), certificate.end(), certificate.begin(), ::tolower); - AhoCorasickPlus::Match match; - std::size_t host_len=certificate.length(); - bool found=false; - m_WorkerConfig.atmSSLDomains->search(certificate, false); - while(m_WorkerConfig.atmSSLDomains->findNext(match) && !found) - { - if(match.pattern.ptext.length != host_len) - { - bool exact_match=match.id & 0x01; - if(exact_match) - continue; - if(certificate[host_len-match.pattern.ptext.length-1] != '.') - continue; - } - found=true; - } - if(found) + struct tcphdr *tcph = (struct tcphdr *)((uint8_t *) pkt->pkt + (pkt->ip_version == 4 ? sizeof(struct ipv4_hdr) : sizeof(struct ipv6_hdr))); + m_ThreadStats.matched_ssl_sni++; + if(pkt->ip_version == 4) { - m_ThreadStats.matched_ssl++; - if(likely(_snd != nullptr)) - { - _snd->SendRST(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0); - } else { - SenderTask::queue.enqueueNotification(new RedirectNotificationG(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0, nullptr, true)); - } - m_ThreadStats.sended_rst++; - return true; + _snd->SendRSTIPv4(pkt->pkt, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.sended_rst_ipv4++; + } else { + _snd->SendRSTIPv6(pkt->pkt, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.sended_rst_ipv6++; } + return true; } return false; } -bool WorkerThread::checkHTTP(std::string &uri, dpi_pkt_infos_t *pkt) +bool WorkerThread::checkURLBlocked(const char *host, size_t host_len, const char *uri, size_t uri_len, dpi_pkt_infos_t* pkt) { - struct ipv4_hdr *ipv4_header = (struct ipv4_hdr *) pkt->pkt; - struct ipv6_hdr *ipv6_header = (struct ipv6_hdr *) pkt->pkt; - struct tcphdr* tcph; - tcph = (struct tcphdr *)((uint8_t *) pkt->pkt + (pkt->ip_version == 4 ? sizeof(struct ipv4_hdr) : sizeof(struct ipv6_hdr))); - if(likely(m_WorkerConfig.atm != nullptr)) + int redir_size = 0; + char *redir_url = nullptr; + if((redir_size = extFilter::instance()->getTriesManager()->checkURLBlocked(getWorkerID(), host, host_len, uri, uri_len, &redir_url)) != 0) { - if(m_WorkerConfig.url_normalization) + struct tcphdr *tcph = (struct tcphdr *)((uint8_t *) pkt->pkt + (pkt->ip_version == 4 ? sizeof(struct ipv4_hdr) : sizeof(struct ipv6_hdr))); + if(likely(redir_url != nullptr)) { - try - { - *uri_p = uri; - uri_p->normalize(); - uri.assign(uri_p->toString()); - } catch (Poco::SyntaxException &ex) + if(pkt->ip_version == 4) { - _logger.debug("An SyntaxException occured: '%s' on URI: '%s'", ex.displayText(), uri); - } - } - if(m_WorkerConfig.remove_dot || (!m_WorkerConfig.url_normalization && m_WorkerConfig.lower_host)) - { - // remove dot after domain... - size_t f_slash_pos=uri.find('/',10); - if(!m_WorkerConfig.url_normalization && m_WorkerConfig.lower_host && f_slash_pos != std::string::npos) - { - std::transform(uri.begin()+7, uri.begin()+f_slash_pos, uri.begin()+7, ::tolower); - } - if(m_WorkerConfig.remove_dot && f_slash_pos != std::string::npos) - { - if(uri[f_slash_pos-1] == '.') - uri.erase(f_slash_pos-1,1); - } - } - AhoCorasickPlus::Match match; - bool found=false; - size_t uri_length=uri.length() - 7; - char const *uri_ptr=uri.c_str() + 7; - m_WorkerConfig.atm->search((char *)uri_ptr, uri_length, false); - while(m_WorkerConfig.atm->findNext(match) && !found) - { - if(match.pattern.ptext.length != uri_length) - { - int r=match.position-match.pattern.ptext.length; - if(((match.id & 0x02) >> 1) == E_TYPE_DOMAIN) - { - if(r > 0) - { - if(match.id & 0x01) - continue; - if(*(uri_ptr+r-1) != '.') - continue; - } - } else if(((match.id & 0x02) >> 1) == E_TYPE_URL) - { - if(m_WorkerConfig.match_url_exactly) - continue; - if(r > 0) - { - if(*(uri_ptr+r-1) != '.') - continue; - } - } + _snd->HTTPRedirectIPv4(pkt->pkt, /*acknum*/ tcph->ack_seq, /*seqnum*/ rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+pkt->data_length), true, redir_url, redir_size); + m_ThreadStats.matched_http_bl_ipv4++; + m_ThreadStats.redirected_http_bl_ipv4++; + } else { + _snd->HTTPRedirectIPv6(pkt->pkt, /*acknum*/ tcph->ack_seq, /*seqnum*/ rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+pkt->data_length), true, redir_url, redir_size); + m_ThreadStats.matched_http_bl_ipv6++; + m_ThreadStats.redirected_http_bl_ipv6++; } - found=true; - } - if(found) - { - if(((match.id & 0x02) >> 1) == E_TYPE_DOMAIN) // block by domain... - { - m_ThreadStats.matched_domains++; - if(m_WorkerConfig.http_redirect) - { - std::string add_param; - switch (m_WorkerConfig.add_p_type) - { - case A_TYPE_ID: add_param="id="+std::to_string(match.id >> 2); - break; - case A_TYPE_URL: add_param="url="+uri; - break; - default: break; - } - if(likely(_snd != nullptr)) - { - _snd->Redirect(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+pkt->data_length), 1, add_param.empty() ? nullptr : (char *)add_param.c_str()); - } else { - SenderTask::queue.enqueueNotification(new RedirectNotificationG(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+pkt->data_length), 1, add_param.empty() ? nullptr : (char *)add_param.c_str())); - } - m_ThreadStats.redirected_domains++; - } else { - if(likely(_snd != nullptr)) - { - _snd->SendRST(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0); - } else { - SenderTask::queue.enqueueNotification(new RedirectNotificationG(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0, nullptr, true)); - } - m_ThreadStats.sended_rst++; - } - return true; - } else if(((match.id & 0x02) >> 1) == E_TYPE_URL) // block by url... + } else { + if(pkt->ip_version == 4) { - m_ThreadStats.matched_urls++; - if(m_WorkerConfig.http_redirect) - { - std::string add_param; - switch (m_WorkerConfig.add_p_type) - { - case A_TYPE_ID: add_param="id="+std::to_string(match.id >> 2); - break; - case A_TYPE_URL: add_param="url="+uri; - break; - default: break; - } - if(likely(_snd != nullptr)) - { - _snd->Redirect(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+pkt->data_length), 1, add_param.empty() ? nullptr : (char *)add_param.c_str()); - } else { - SenderTask::queue.enqueueNotification(new RedirectNotificationG(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+pkt->data_length), 1, add_param.empty() ? nullptr : (char *)add_param.c_str())); - } - m_ThreadStats.redirected_urls++; - } else { - if(likely(_snd != nullptr)) - { - _snd->SendRST(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0); - } else { - SenderTask::queue.enqueueNotification(new RedirectNotificationG(pkt->srcport, pkt->dstport, pkt->ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, pkt->ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, pkt->ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0, nullptr, true)); - } - m_ThreadStats.sended_rst++; - } - return true; + _snd->HTTPForbiddenIPv4(pkt->pkt, tcph->ack_seq, rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+pkt->data_length), true); + m_ThreadStats.sended_forbidden_ipv4++; + m_ThreadStats.matched_http_bl_ipv4++; + } else { + _snd->HTTPForbiddenIPv6(pkt->pkt, tcph->ack_seq, rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+pkt->data_length), true); + m_ThreadStats.sended_forbidden_ipv6++; + m_ThreadStats.matched_http_bl_ipv6++; } } + return true; } return false; } @@ -407,9 +184,9 @@ dpi_identification_result_t WorkerThread::identifyAppProtocol(const unsigned cha return r; } - if(infos.l4prot != IPPROTO_TCP && infos.l4prot != IPPROTO_UDP) + if(unlikely(infos.l4prot != IPPROTO_TCP && infos.l4prot != IPPROTO_UDP)) { - r.status=DPI_ERROR_TRANSPORT_PROTOCOL_NOTSUPPORTED; + r.status = DPI_ERROR_TRANSPORT_PROTOCOL_NOTSUPPORTED; return r; } @@ -437,204 +214,319 @@ dpi_identification_result_t WorkerThread::getAppProtocol(uint8_t *host_key, uint { dpi_identification_result_t r; r.status = DPI_STATUS_OK; + r.protocol.l7prot = DPI_PROTOCOL_UNKNOWN; - dpi_flow_infos_t* flow_infos=NULL; + dpi_flow_infos_t* flow_infos = nullptr; int32_t hash_idx = 0; - ext_dpi_flow_info *fi = getFlow(host_key, timestamp, &hash_idx, sig, pkt_infos); + FlowStorageIPV4 *fs_ipv4 = nullptr; + FlowStorageIPV6 *fs_ipv6 = nullptr; - if(unlikely(fi==NULL)) - { - r.status=DPI_ERROR_MAX_FLOWS; - return r; - } - - flow_infos = &(fi->infos); + tcphdr *tcph = nullptr; - r = dpi_stateless_get_app_protocol(dpi_state, flow_infos, pkt_infos); + en_alfs_type_t alfs_type = en_alfs_short; - if(r.status == DPI_STATUS_TCP_CONNECTION_TERMINATED) + if(pkt_infos->l4prot == IPPROTO_TCP) { - if(pkt_infos->ip_version == 4) + tcph = (struct tcphdr*) (pkt_infos->pkt + pkt_infos->l4offset); + if(tcph->fin == 0 && tcph->syn == 0 && tcph->rst == 0) + alfs_type = en_alfs_long; + } + ext_dpi_flow_info_ipv4 *node = nullptr; + ext_dpi_flow_info_ipv6 *node_ipv6 = nullptr; + if(pkt_infos->ip_version == 4) + { + fs_ipv4 = (FlowStorageIPV4 *) worker_params[_worker_id].flows_ipv4.flows[ipv4_flow_mask & sig]; + node = fs_ipv4->searchFlow(host_key, sig, pkt_infos, &hash_idx); + if(node == nullptr) { - int32_t delr=rte_hash_del_key(m_FlowHash->getIPv4Hash(), host_key); - if(delr < 0) + if(tcph != nullptr) { - _logger.error("Error (%d) occured while delete data from the ipv4 flow hash table", (int)delr); - } else { - ipv4_flows[hash_idx]->free_mem(dpi_state->flow_cleaner_callback); - rte_mempool_put(flows_pool, ipv4_flows[hash_idx]); - ipv4_flows[hash_idx] = nullptr; - m_ThreadStats.ndpi_flows_count--; - m_ThreadStats.ndpi_ipv4_flows_count--; - m_ThreadStats.ndpi_flows_deleted++; + // проверяем флаги tcp. этих flow нет в кэше, поэтому пропускаем ничего не делая + // rst - нечего завершать, т.к. dpi не знает flow + // syn+ack - нечего подтверждать, т.к. нет flow + // fin - нечего завершать, т.к. нет flow + if(tcph->rst || (tcph->ack && tcph->syn) || tcph->fin) +// if(tcph->rst || tcph->fin) + { + m_ThreadStats.no_create_flow++; + return r; + } } - } else { - int32_t delr=rte_hash_del_key(m_FlowHash->getIPv6Hash(), host_key); - if(delr < 0) + pkt_infos->direction=0; + if((node = fs_ipv4->short_alfs.getOldestMoveBack(timestamp, &fs_ipv4->long_alfs)) != nullptr || (node = fs_ipv4->long_alfs.getOldestMoveBack(timestamp, &fs_ipv4->short_alfs)) != nullptr) { - _logger.error("Error (%d) occured while delete data from the ipv6 flow hash table", (int)delr); + // есть просроченная запись... + // очищаем всю память, занятую ранее... + node->free_mem(dpi_state->flow_cleaner_callback); + // надо удалить по ключу из старой записи... + fs_ipv4->removeFlow(node->cmn.hash_idx); + node->init(timestamp, node->cmn.owner_worker_id, _worker_id, node->cmn.idx_alfs, node->cmn.hash_idx); + + node->src_addr_t.ipv4_srcaddr = pkt_infos->src_addr_t.ipv4_srcaddr; + node->dst_addr_t.ipv4_dstaddr = pkt_infos->dst_addr_t.ipv4_dstaddr; + node->srcport = pkt_infos->srcport; + node->dstport = pkt_infos->dstport; + node->l4prot = pkt_infos->l4prot; + + dpi_init_flow_infos(dpi_state, &(node->infos), pkt_infos->l4prot); + node->cmn.alfs_type = alfs_type; + + fs_ipv4->reuseFlow(host_key, sig, node); + m_ThreadStats.new_flow++; + m_ThreadStats.reuse_flow++; + flow_infos = &node->infos; } else { - ipv6_flows[hash_idx]->free_mem(dpi_state->flow_cleaner_callback); - rte_mempool_put(flows_pool,ipv6_flows[hash_idx]); - ipv6_flows[hash_idx] = nullptr; - m_ThreadStats.ndpi_flows_count--; - m_ThreadStats.ndpi_ipv6_flows_count--; - m_ThreadStats.ndpi_flows_deleted++; - } - } - } - return r; -} - - + // нет просроченных записей, надо создавать новую... + node = fs_ipv4->newFlow(); + if(likely(node != nullptr)) + { + node->init(timestamp, _worker_id, _worker_id, 0, 0); + node->src_addr_t.ipv4_srcaddr = pkt_infos->src_addr_t.ipv4_srcaddr; + node->dst_addr_t.ipv4_dstaddr = pkt_infos->dst_addr_t.ipv4_dstaddr; + node->srcport = pkt_infos->srcport; + node->dstport = pkt_infos->dstport; + node->l4prot = pkt_infos->l4prot; + dpi_init_flow_infos(dpi_state, &(node->infos), pkt_infos->l4prot); + node->cmn.alfs_type = alfs_type; + if(unlikely(fs_ipv4->addFlow(host_key, sig, node))) + { + m_ThreadStats.hash_add_fail_flow++; + if(m_ThreadStats.hash_add_fail_flow % 100000 == 0) + _logger.error("Can't add flow to the hash"); + } -ext_dpi_flow_info *WorkerThread::getFlow(uint8_t *host_key, uint64_t timestamp, int32_t *idx, uint32_t sig, dpi_pkt_infos_t *pkt_infos) -{ - if(pkt_infos->ip_version == 6) - { - int32_t ret = rte_hash_lookup_with_hash(m_FlowHash->getIPv6Hash(), host_key, sig); - if(ret >= 0) - { - if(pkt_infos->l4prot == IPPROTO_TCP && ipv6_flows[ret]->infos.tracking.seen_rst && ((struct tcphdr*) (pkt_infos->pkt + pkt_infos->l4offset))->syn) + if(likely(fs_ipv4->short_alfs.can_add_rec() && fs_ipv4->long_alfs.can_add_rec())) + { + fs_ipv4->short_alfs.add_rec(node); + uint32_t short_alfs_idx = node->cmn.idx_alfs; + fs_ipv4->long_alfs.add_rec(node); + if(short_alfs_idx != node->cmn.idx_alfs) + _logger.error("idx_alfs not equal: short %d <> long %d", (int)short_alfs_idx, (int) node->cmn.idx_alfs); + m_ThreadStats.new_flow++; + } else { + m_ThreadStats.alfs_fail_flow++; + if(m_ThreadStats.alfs_fail_flow % 100000 == 0) + _logger.error("Can't add ipv4 flow to the alfs!"); + } + flow_infos = &node->infos; + } else { + m_ThreadStats.error_alloc_flow++; + if(m_ThreadStats.error_alloc_flow % 100000 == 0) + { + _logger.error("Unable to allocate flow record. Repeat error %" PRIu64, m_ThreadStats.error_alloc_flow); + } + } + } + } else { + if(unlikely(pkt_infos->l4prot == IPPROTO_TCP && node->infos.tracking.seen_rst && ((struct tcphdr*) (pkt_infos->pkt + pkt_infos->l4offset))->syn)) { - // Delete old flow. - ipv6_flows[ret]->free_mem(dpi_state->flow_cleaner_callback); - rte_mempool_put(flows_pool, ipv6_flows[ret]); - ipv6_flows[ret] = nullptr; - m_ThreadStats.ndpi_flows_count--; - m_ThreadStats.ndpi_ipv6_flows_count--; - m_ThreadStats.ndpi_flows_deleted++; - // Force the following code to create a new flow. - ret = -ENOENT; + // recycling flow, reset all data for dpi. + m_ThreadStats.recycling_flow++; + node->free_mem(dpi_state->flow_cleaner_callback); + node->init(timestamp, node->cmn.owner_worker_id, _worker_id, node->cmn.idx_alfs, node->cmn.hash_idx); + node->src_addr_t.ipv4_srcaddr = pkt_infos->src_addr_t.ipv4_srcaddr; + node->dst_addr_t.ipv4_dstaddr = pkt_infos->dst_addr_t.ipv4_dstaddr; + node->srcport = pkt_infos->srcport; + node->dstport = pkt_infos->dstport; + node->l4prot = pkt_infos->l4prot; + dpi_init_flow_infos(dpi_state, &(node->infos), pkt_infos->l4prot); } else { - *idx = ret; - if(ext_dpi_v6_addresses_equal((uint64_t *)&(ipv6_flows[ret]->src_addr_t.ipv6_srcaddr),(uint64_t *) &pkt_infos->src_addr_t.ipv6_srcaddr) && ipv6_flows[ret]->srcport == pkt_infos->srcport) + if(node->src_addr_t.ipv4_srcaddr == pkt_infos->src_addr_t.ipv4_srcaddr && node->srcport == pkt_infos->srcport) pkt_infos->direction=0; else pkt_infos->direction=1; - ipv6_flows[ret]->last_timestamp = timestamp; - return ipv6_flows[ret]; } + node->cmn.alfs_type = alfs_type; + fs_ipv4->short_alfs.moveBack(node, timestamp, &fs_ipv4->long_alfs); + flow_infos = &node->infos; } - if(ret == -EINVAL) - { - _logger.error("Bad parameter in ipv6 hash lookup"); - return NULL; - } - if(ret == -ENOENT) + } else if(pkt_infos->ip_version == 6) + { + fs_ipv6 = (FlowStorageIPV6 *) worker_params[_worker_id].flows_ipv6.flows[ipv6_flow_mask & sig]; + node_ipv6 = fs_ipv6->searchFlow(host_key, sig, pkt_infos, &hash_idx); + if(node_ipv6 == nullptr) { - struct ext_dpi_flow_info *newflow; - if(rte_mempool_get(flows_pool, (void **)&newflow) != 0) + if(tcph != nullptr) { - _logger.fatal("Not enough memory for the flow in the flows_pool"); - return NULL; + // проверяем флаги tcp. этих flow нет в кэше, поэтому пропускаем ничего не делая + // rst - нечего завершать, т.к. dpi не знает flow + // syn+ack - нечего подтверждать, т.к. нет flow + //if(tcph->rst || (tcph->ack && tcph->syn)) + if(tcph->rst || (tcph->ack && tcph->syn) || tcph->fin) +// if(tcph->rst || tcph->fin) + { + m_ThreadStats.no_create_flow_ipv6++; + return r; + } } - memset(newflow, 0, sizeof(struct ext_dpi_flow_info)); - newflow->last_timestamp = timestamp; - rte_memcpy(&newflow->src_addr_t.ipv6_srcaddr, &pkt_infos->src_addr_t.ipv6_srcaddr, IPV6_ADDR_LEN * 2); - newflow->srcport=pkt_infos->srcport; - newflow->dstport=pkt_infos->dstport; - newflow->l4prot=pkt_infos->l4prot; - - dpi_init_flow_infos(dpi_state, &(newflow->infos), pkt_infos->l4prot); - - pkt_infos->direction = 0; - ret = rte_hash_add_key_with_hash(m_FlowHash->getIPv6Hash(), host_key, sig); - if(ret == -EINVAL) + pkt_infos->direction=0; + if((node_ipv6 = fs_ipv6->short_alfs.getOldestMoveBack(timestamp, &fs_ipv6->long_alfs)) != nullptr || (node_ipv6 = fs_ipv6->long_alfs.getOldestMoveBack(timestamp, &fs_ipv6->short_alfs)) != nullptr) { - rte_mempool_put(flows_pool,newflow); - _logger.fatal("Bad parameters in hash add"); - return NULL; + // есть просроченная запись... + // очищаем всю память, занятую ранее... + node_ipv6->free_mem(dpi_state->flow_cleaner_callback); + // надо удалить по ключу из старой записи... + fs_ipv6->removeFlow(node_ipv6->cmn.hash_idx); + node_ipv6->init(timestamp, node_ipv6->cmn.owner_worker_id, _worker_id, node_ipv6->cmn.idx_alfs, node_ipv6->cmn.hash_idx); + rte_memcpy(&node_ipv6->src_addr_t.ipv6_srcaddr, &pkt_infos->src_addr_t.ipv6_srcaddr, IPV6_ADDR_LEN * 2); + node_ipv6->srcport = pkt_infos->srcport; + node_ipv6->dstport = pkt_infos->dstport; + node_ipv6->l4prot = pkt_infos->l4prot; + dpi_init_flow_infos(dpi_state, &(node_ipv6->infos), pkt_infos->l4prot); + node_ipv6->cmn.alfs_type = alfs_type; + + fs_ipv6->reuseFlow(host_key, sig, node_ipv6); + m_ThreadStats.new_flow_ipv6++; + m_ThreadStats.reuse_flow_ipv6++; + flow_infos = &node_ipv6->infos; + } else { + // нет просроченных записей, надо создавать новую... + node_ipv6 = fs_ipv6->newFlow(); + if(likely(node_ipv6 != nullptr)) + { + node_ipv6->init(timestamp, _worker_id, _worker_id, 0, 0); + rte_memcpy(&node_ipv6->src_addr_t.ipv6_srcaddr, &pkt_infos->src_addr_t.ipv6_srcaddr, IPV6_ADDR_LEN * 2); + node_ipv6->srcport = pkt_infos->srcport; + node_ipv6->dstport = pkt_infos->dstport; + node_ipv6->l4prot = pkt_infos->l4prot; + dpi_init_flow_infos(dpi_state, &(node_ipv6->infos), pkt_infos->l4prot); + node_ipv6->cmn.alfs_type = alfs_type; + + if(unlikely(fs_ipv6->addFlow(host_key, sig, node_ipv6))) + { + m_ThreadStats.hash_add_fail_flow_ipv6++; + if(m_ThreadStats.hash_add_fail_flow_ipv6 % 100000 == 0) + _logger.error("Can't add ipv6 flow to the hash"); + } + + if(unlikely(fs_ipv6->short_alfs.can_add_rec() && fs_ipv6->long_alfs.can_add_rec())) + { + fs_ipv6->short_alfs.add_rec(node_ipv6); + uint32_t short_alfs_idx = node_ipv6->cmn.idx_alfs; + fs_ipv6->long_alfs.add_rec(node_ipv6); + if(short_alfs_idx != node_ipv6->cmn.idx_alfs) + _logger.error("idx_alfs not equal: short %d <> long %d", (int)short_alfs_idx, (int) node_ipv6->cmn.idx_alfs); + m_ThreadStats.new_flow_ipv6++; + } else { + m_ThreadStats.alfs_fail_flow_ipv6++; + if(m_ThreadStats.alfs_fail_flow_ipv6 % 100000 == 0) + _logger.error("Can't add ipv6 flow to the alfs!"); + } + flow_infos = &node_ipv6->infos; + } else { + m_ThreadStats.error_alloc_flow_ipv6++; + if(m_ThreadStats.error_alloc_flow_ipv6 % 100000 == 0) + { + _logger.error("Unable to allocate flow ipv6 record. Repeat error %" PRIu64, m_ThreadStats.error_alloc_flow_ipv6); + } + } } - if(ret == -ENOSPC) + } else { + if(pkt_infos->l4prot == IPPROTO_TCP && node_ipv6->infos.tracking.seen_rst && ((struct tcphdr*) (pkt_infos->pkt + pkt_infos->l4offset))->syn) { - rte_mempool_put(flows_pool,newflow); - _logger.fatal("There is no space in the ipv6 flow hash"); - return NULL; + // recycling flow, reset all data for dpi. + m_ThreadStats.recycling_flow_ipv6++; + node_ipv6->free_mem(dpi_state->flow_cleaner_callback); + node_ipv6->init(timestamp, node_ipv6->cmn.owner_worker_id, _worker_id, node_ipv6->cmn.idx_alfs, node_ipv6->cmn.hash_idx); + rte_memcpy(&node_ipv6->src_addr_t.ipv6_srcaddr, &pkt_infos->src_addr_t.ipv6_srcaddr, IPV6_ADDR_LEN * 2); + node_ipv6->srcport = pkt_infos->srcport; + node_ipv6->dstport = pkt_infos->dstport; + node_ipv6->l4prot = pkt_infos->l4prot; + dpi_init_flow_infos(dpi_state, &(node_ipv6->infos), pkt_infos->l4prot); + } else { + if(ext_dpi_v6_addresses_equal((uint64_t *)&(node_ipv6->src_addr_t.ipv6_srcaddr),(uint64_t *) &pkt_infos->src_addr_t.ipv6_srcaddr) && node_ipv6->srcport == pkt_infos->srcport) + pkt_infos->direction=0; + else + pkt_infos->direction=1; } - ipv6_flows[ret] = newflow; - *idx = ret; - m_ThreadStats.ndpi_ipv6_flows_count++; - m_ThreadStats.ndpi_flows_count++; - return newflow; + node_ipv6->cmn.alfs_type = alfs_type; + fs_ipv6->short_alfs.moveBack(node_ipv6, timestamp, &fs_ipv6->long_alfs); + flow_infos = &node_ipv6->infos; } - return NULL; } + if(unlikely(flow_infos == nullptr)) + { + _logger.error("Unable to extract flow_infos"); + r.status = DPI_ERROR_MAX_FLOWS; + return r; + } + + r = dpi_stateless_get_app_protocol(dpi_state, flow_infos, pkt_infos); + if(pkt_infos->ip_version == 4) { - int32_t ret = rte_hash_lookup_with_hash(m_FlowHash->getIPv4Hash(), host_key, sig); - if(ret >= 0) + if(_need_block) { - if(pkt_infos->l4prot == IPPROTO_TCP && ipv4_flows[ret]->infos.tracking.seen_rst && ((struct tcphdr*) (pkt_infos->pkt + pkt_infos->l4offset))->syn) + node->cmn.blocked = true; + } else { + if(_need_block == false && pkt_infos->data_length > 0 && node->cmn.blocked) { - // Delete old flow. - ipv4_flows[ret]->free_mem(dpi_state->flow_cleaner_callback); - rte_mempool_put(flows_pool, ipv4_flows[ret]); - ipv4_flows[ret] = nullptr; - m_ThreadStats.ndpi_flows_count--; - m_ThreadStats.ndpi_ipv4_flows_count--; - m_ThreadStats.ndpi_flows_deleted++; - // Force the following code to create a new flow. - ret = -ENOENT; - } else { - *idx = ret; - if(ipv4_flows[ret]->src_addr_t.ipv4_srcaddr == pkt_infos->src_addr_t.ipv4_srcaddr && ipv4_flows[ret]->srcport == pkt_infos->srcport) - pkt_infos->direction=0; - else - pkt_infos->direction=1; - ipv4_flows[ret]->last_timestamp = timestamp; - return ipv4_flows[ret]; + switch (r.protocol.l7prot) + { + case DPI_PROTOCOL_TCP_SSL: + _snd->SendRSTIPv4(pkt_infos->pkt, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.seen_already_blocked_ssl_ipv4++; + m_ThreadStats.sended_rst_ipv4++; + break; + case DPI_PROTOCOL_TCP_HTTP: + _snd->SendRSTIPv4(pkt_infos->pkt, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.seen_already_blocked_http_ipv4++; + m_ThreadStats.sended_rst_ipv4++; + break; + default: + break; + } + } } - if(ret == -EINVAL) - { - _logger.error("Bad parameter in ipv4 hash lookup"); - return NULL; - } - if(ret == -ENOENT) + } else if (pkt_infos->ip_version == 6) + { + if(_need_block) { - struct ext_dpi_flow_info *newflow; - if(rte_mempool_get(flows_pool, (void **)&newflow) != 0) + node_ipv6->cmn.blocked = true; + } else { + if(_need_block == false && pkt_infos->data_length > 0 && node_ipv6->cmn.blocked) { - _logger.fatal("Not enough memory for the flow in the flows_pool"); - return NULL; + switch (r.protocol.l7prot) + { + case DPI_PROTOCOL_TCP_SSL: + _snd->SendRSTIPv6(pkt_infos->pkt, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.seen_already_blocked_ssl_ipv6++; + m_ThreadStats.sended_rst_ipv6++; + break; + case DPI_PROTOCOL_TCP_HTTP: + _snd->SendRSTIPv6(pkt_infos->pkt, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.seen_already_blocked_http_ipv6++; + m_ThreadStats.sended_rst_ipv6++; + break; + default: + break; + } } - memset(newflow, 0, sizeof(struct ext_dpi_flow_info)); - newflow->last_timestamp = timestamp; - - newflow->src_addr_t.ipv4_srcaddr = pkt_infos->src_addr_t.ipv4_srcaddr; - newflow->dst_addr_t.ipv4_dstaddr = pkt_infos->dst_addr_t.ipv4_dstaddr; - newflow->srcport=pkt_infos->srcport; - newflow->dstport=pkt_infos->dstport; - newflow->l4prot=pkt_infos->l4prot; - - dpi_init_flow_infos(dpi_state, &(newflow->infos), pkt_infos->l4prot); + } + + } - pkt_infos->direction = 0; - ret = rte_hash_add_key_with_hash(m_FlowHash->getIPv4Hash(), host_key, sig); - if(ret == -EINVAL) - { - rte_mempool_put(flows_pool,newflow); - _logger.fatal("Bad parameters in hash add"); - return NULL; - } - if(ret == -ENOSPC) - { - rte_mempool_put(flows_pool,newflow); - _logger.fatal("There is no space in the ipv4 flow hash"); - return NULL; - } - ipv4_flows[ret] = newflow; - *idx = ret; - m_ThreadStats.ndpi_ipv4_flows_count++; - m_ThreadStats.ndpi_flows_count++; - return newflow; + if(r.status == DPI_STATUS_TCP_CONNECTION_TERMINATED) + { + if(pkt_infos->ip_version == 4) + { + m_ThreadStats.close_flow++; + node->free_mem(dpi_state->flow_cleaner_callback); + node->init(timestamp, node->cmn.owner_worker_id, _worker_id, node->cmn.idx_alfs, node->cmn.hash_idx); + node->cmn.alfs_type = alfs_type; + fs_ipv4->short_alfs.moveBack(node, timestamp, &fs_ipv4->long_alfs); + } else { + m_ThreadStats.close_flow_ipv6++; + node_ipv6->free_mem(dpi_state->flow_cleaner_callback); + node_ipv6->init(timestamp, node_ipv6->cmn.owner_worker_id, _worker_id, node_ipv6->cmn.idx_alfs, node_ipv6->cmn.hash_idx); + node_ipv6->cmn.alfs_type = alfs_type; + fs_ipv6->short_alfs.moveBack(node_ipv6, timestamp, &fs_ipv6->long_alfs); } - return NULL; } - return NULL; + return r; } bool WorkerThread::analyzePacket(struct rte_mbuf* m, uint64_t timestamp) @@ -681,7 +573,6 @@ bool WorkerThread::analyzePacket(struct rte_mbuf* m, uint64_t timestamp) if(rte_ipv4_frag_pkt_is_fragmented(ipv4_header)) { m_ThreadStats.ipv4_fragments++; -// return false; } } else if (l3_ptypes == RTE_PTYPE_L3_IPV6) { @@ -694,10 +585,8 @@ bool WorkerThread::analyzePacket(struct rte_mbuf* m, uint64_t timestamp) if(rte_ipv6_frag_get_ipv6_fragment_header(ipv6_header) != NULL) { m_ThreadStats.ipv6_fragments++; -// return false; } } else { - //_logger.debug("Unsupported ethernet type %x", (int) ether_type); return false; } @@ -707,7 +596,6 @@ bool WorkerThread::analyzePacket(struct rte_mbuf* m, uint64_t timestamp) if(ip_protocol != IPPROTO_TCP) { - //_logger.debug("Not TCP protocol"); return false; } @@ -727,32 +615,40 @@ bool WorkerThread::analyzePacket(struct rte_mbuf* m, uint64_t timestamp) m_ThreadStats.analyzed_packets++; - uint16_t tcp_src_port = tcph->source; - uint16_t tcp_dst_port = tcph->dest; - uint32_t acl_action = pkt_info->acl_res & ACL_POLICY_MASK; if(payload_len > 0 && acl_action == ACL::ACL_DROP) { m_ThreadStats.matched_ip_port++; - if(likely(_snd != nullptr)) + if(ip_version == 4) { - _snd->SendRST(tcp_src_port, tcp_dst_port, ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0); - } else { - SenderTask::queue.enqueueNotification(new RedirectNotificationG(tcp_src_port, tcp_dst_port, ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0, nullptr, true)); + _snd->SendRSTIPv4(l3, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.sended_rst_ipv4++; + } + else + { + _snd->SendRSTIPv6(l3, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.sended_rst_ipv6++; } - m_ThreadStats.sended_rst++; return true; } - dpi_identification_result_t r; - uri.clear(); - certificate.clear(); - r = identifyAppProtocol(l3, ip_len, timestamp, (uint8_t *)&((struct packet_info *)m->userdata)->keys, m->hash.usr); - if(_need_block) + switch (r.protocol.l7prot) + { + case DPI_PROTOCOL_TCP_SSL: + m_ThreadStats.ssl_packets++; + break; + case DPI_PROTOCOL_TCP_HTTP: + m_ThreadStats.http_packets++; + break; + default: + break; + } + + if(unlikely(_need_block)) return true; if(payload_len == 0) @@ -760,37 +656,40 @@ bool WorkerThread::analyzePacket(struct rte_mbuf* m, uint64_t timestamp) if(r.protocol.l7prot == DPI_PROTOCOL_TCP_SSL) { - if(m_WorkerConfig.block_ssl_no_sni && certificate.empty()) + if(m_WorkerConfig.block_ssl_no_sni) { if(acl_action == ACL::ACL_SSL && payload_len > 0) { m_ThreadStats.matched_ssl_ip++; - m_ThreadStats.sended_rst++; - if(likely(_snd != nullptr)) + if(ip_version == 4) { - _snd->SendRST(tcp_src_port, tcp_dst_port, ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0); - } else { - SenderTask::queue.enqueueNotification(new RedirectNotificationG(tcp_src_port, tcp_dst_port, ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq, 0, nullptr, true)); + _snd->SendRSTIPv4(l3, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.sended_rst_ipv4++; + } + else + { + _snd->SendRSTIPv6(l3, /*acknum*/ tcph->ack_seq, /*seqnum*/ tcph->seq); + m_ThreadStats.sended_rst_ipv6++; } return true; } } } - if(r.protocol.l7prot == DPI_PROTOCOL_TCP_HTTP && !uri.empty()) - { - if(ip_version == 4 && m_WorkerConfig.nm && m_WorkerConfig.notify_enabled && acl_action == ACL::ACL_NOTIFY) - { - uint32_t notify_group = (pkt_info->acl_res & ACL_NOTIFY_GROUP) >> 4; - if(m_WorkerConfig.nm->needNotify(ipv4_header->src_addr, notify_group)) - { - //std::string add_param("url="+uri); - std::string add_param; - NotifyManager::queue.enqueueNotification(new NotifyRedirect(notify_group, tcp_src_port, tcp_dst_port, ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+payload_len), 1, (char *)add_param.c_str())); - return true; - } - } - } +// if(r.protocol.l7prot == DPI_PROTOCOL_TCP_HTTP && !uri.empty()) +// { +// if(m_WorkerConfig.notify_enabled && m_WorkerConfig.nm && ip_version == 4 && acl_action == ACL::ACL_NOTIFY) +// { +// uint32_t notify_group = (pkt_info->acl_res & ACL_NOTIFY_GROUP) >> 4; +// if(m_WorkerConfig.nm->needNotify(ipv4_header->src_addr, notify_group)) +// { +// std::string add_param; +// NotifyManager::queue.enqueueNotification(new NotifyRedirect(notify_group, tcp_src_port, tcp_dst_port, ip_version == 4 ? (void *)&ipv4_header->src_addr : (void *)&ipv6_header->src_addr, ip_version == 4 ? (void *)&ipv4_header->dst_addr : (void *)&ipv6_header->dst_addr, ip_version, /*acknum*/ tcph->ack_seq, /*seqnum*/ rte_cpu_to_be_32(rte_be_to_cpu_32(tcph->seq)+payload_len), 1, (char *)add_param.c_str())); +// return true; +// } +// } +// } + return false; } @@ -850,9 +749,6 @@ static inline void prepare_acl_parameter(struct rte_mbuf** pkts_in, struct ACL:: } } - - - bool WorkerThread::run(uint32_t coreId) { setCoreId(coreId); @@ -878,24 +774,10 @@ bool WorkerThread::run(uint32_t coreId) const uint64_t timer_interval = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * (1000*1000); - const uint64_t gc_int_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * EXTF_GC_INTERVAL; - - int gc_budget_ipv4 = ((double)m_FlowHash->getHashSizeIPv4()/(EXTF_ALL_GC_INTERVAL*1000*1000))*EXTF_GC_INTERVAL; - - int gc_budget_ipv6 = ((double)m_FlowHash->getHashSizeIPv6()/(EXTF_ALL_GC_INTERVAL*1000*1000))*EXTF_GC_INTERVAL; - - _logger.information("gc_budget_ipv4: %d, gc_budget_ipv6: %d", gc_budget_ipv4, gc_budget_ipv6); - - _logger.information("Running gc clean every %" PRIu64 " cycles. Cycles per second %" PRIu64, gc_int_tsc, rte_get_timer_hz()); - uint64_t last_sec = 0; - uint64_t cur_tsc, diff_timer_tsc, diff_gc_tsc; + uint64_t cur_tsc, diff_timer_tsc; uint64_t prev_timer_tsc = 0; - uint64_t prev_gc_tsc=0; - - uint32_t iter_flows_ipv4 = 0; - uint32_t iter_flows_ipv6 = 0; uint8_t sender_port = m_WorkerConfig.sender_port; uint16_t tx_queue_id = m_WorkerConfig.tx_queue_id; @@ -932,12 +814,6 @@ bool WorkerThread::run(uint32_t coreId) SWAP_ACX(qconf->cur_acx_ipv6, qconf->new_acx_ipv6); #undef SWAP_ACX - if(unlikely(m_WorkerConfig.atm_new != m_WorkerConfig.atm)) - m_WorkerConfig.atm = m_WorkerConfig.atm_new; - - if(unlikely(m_WorkerConfig.atmSSLDomains_new != m_WorkerConfig.atmSSLDomains)) - m_WorkerConfig.atmSSLDomains = m_WorkerConfig.atmSSLDomains_new; - /* * Read packet from RX queues */ @@ -961,7 +837,7 @@ bool WorkerThread::run(uint32_t coreId) prepare_acl_parameter(bufs, &acl_search, nb_rx); - if(likely(qconf->cur_acx_ipv4 && acl_search.num_ipv4)) + if(qconf->cur_acx_ipv4 && acl_search.num_ipv4) { rte_acl_classify(qconf->cur_acx_ipv4, acl_search.data_ipv4, acl_search.res_ipv4, acl_search.num_ipv4, DEFAULT_MAX_CATEGORIES); for(int acli=0; acli < acl_search.num_ipv4; acli++) @@ -972,7 +848,7 @@ bool WorkerThread::run(uint32_t coreId) } } } - if (likely(qconf->cur_acx_ipv6 && acl_search.num_ipv6)) + if (qconf->cur_acx_ipv6 && acl_search.num_ipv6) { rte_acl_classify(qconf->cur_acx_ipv6, acl_search.data_ipv6, acl_search.res_ipv6, acl_search.num_ipv6, DEFAULT_MAX_CATEGORIES); for(int acli=0; acli < acl_search.num_ipv6; acli++) @@ -1017,74 +893,6 @@ bool WorkerThread::run(uint32_t coreId) _n_send_pkts = 0; } } - - diff_gc_tsc = cur_tsc - prev_gc_tsc; - if (unlikely(diff_gc_tsc >= gc_int_tsc)) - { - int z=0; - while(z < gc_budget_ipv4 && iter_flows_ipv4 < m_FlowHash->getHashSizeIPv4()) - { - if(ipv4_flows[iter_flows_ipv4] && (last_sec - (ipv4_flows[iter_flows_ipv4]->last_timestamp) > EXT_DPI_FLOW_TABLE_MAX_IDLE_TIME)) - { - void *key_ptr; - int fr=rte_hash_get_key_with_position(m_FlowHash->getIPv4Hash(), iter_flows_ipv4, &key_ptr); - if(fr < 0) - { - _logger.error("Key not found in the hash for the position %d", (int) iter_flows_ipv4); - } else { - int32_t delr=rte_hash_del_key(m_FlowHash->getIPv4Hash(), key_ptr); - if(delr < 0) - { - _logger.error("Error (%d) occured while delete data from the ipv4 flow hash table", (int)delr); - } else { - ipv4_flows[iter_flows_ipv4]->free_mem(dpi_state->flow_cleaner_callback); - rte_mempool_put(flows_pool, ipv4_flows[iter_flows_ipv4]); - ipv4_flows[iter_flows_ipv4] = nullptr; - m_ThreadStats.ndpi_flows_count--; - m_ThreadStats.ndpi_ipv4_flows_count--; - m_ThreadStats.ndpi_flows_deleted++; - m_ThreadStats.ndpi_flows_expired++; - } - } - } - z++; - iter_flows_ipv4++; - } - if(iter_flows_ipv4 >= m_FlowHash->getHashSizeIPv4()) - iter_flows_ipv4 = 0; - z=0; - while(z < gc_budget_ipv6 && iter_flows_ipv6 < m_FlowHash->getHashSizeIPv6()) - { - if(ipv6_flows[iter_flows_ipv6] && ((last_sec - ipv6_flows[iter_flows_ipv6]->last_timestamp) > EXT_DPI_FLOW_TABLE_MAX_IDLE_TIME)) - { - void *key_ptr; - int fr=rte_hash_get_key_with_position(m_FlowHash->getIPv6Hash(), iter_flows_ipv6, &key_ptr); - if(fr < 0) - { - _logger.error("Key not found in the hash for the position %d", (int) iter_flows_ipv6); - } else { - int32_t delr=rte_hash_del_key(m_FlowHash->getIPv6Hash(), key_ptr); - if(delr < 0) - { - _logger.error("Error (%d) occured while delete data from the ipv6 flow hash table", (int)delr); - } else { - ipv6_flows[iter_flows_ipv6]->free_mem(dpi_state->flow_cleaner_callback); - rte_mempool_put(flows_pool,ipv6_flows[iter_flows_ipv6]); - ipv6_flows[iter_flows_ipv6] = nullptr; - m_ThreadStats.ndpi_flows_count--; - m_ThreadStats.ndpi_ipv6_flows_count--; - m_ThreadStats.ndpi_flows_deleted++; - m_ThreadStats.ndpi_flows_expired++; - } - } - } - z++; - iter_flows_ipv6++; - } - if(iter_flows_ipv6 >= m_FlowHash->getHashSizeIPv6()) - iter_flows_ipv6 = 0; - prev_gc_tsc = cur_tsc; - } diff_timer_tsc = cur_tsc - prev_timer_tsc; if (unlikely(diff_timer_tsc >= timer_interval)) {