Skip to content

Commit

Permalink
Merge pull request #345 from network-intelligence/dev
Browse files Browse the repository at this point in the history
Merging `dev` into `trunk` for VERSION 2.6.2
  • Loading branch information
davidmcgrew authored and GitHub Enterprise committed Dec 17, 2024
2 parents bb917c7 + a820f39 commit 9139040
Show file tree
Hide file tree
Showing 155 changed files with 5,013 additions and 535,898 deletions.
62 changes: 62 additions & 0 deletions .github/workflows/build-wheels.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
name: Mercury wheels Build and Publish to PyPI

on:
workflow_dispatch:
push:
branches:
- main

jobs:
build-mercury-wheels:
name: Build wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
environment:
name: pypi
url: https://pypi.org/p/mercury-python
permissions:
id-token: write
strategy:
matrix:
os: [ubuntu-latest]

steps:
- uses: actions/checkout@v4
with:
detch-depth: 1

- name: Install packages
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev libssl-dev make

- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: '3.x'

- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v3
with:
platforms: all

- name: Install pip packages
run: python -m pip install cython cibuildwheel==2.20.0

- name: Build Mercury
run: ./configure && make && cp -r src/cython/* ./

- name: Build wheels
uses: pypa/[email protected]
env:
CIBW_ARCHS_LINUX: x86_64 aarch64
CIBW_ENVIRONMENT: CC='g++' CXX='g++' MERCURY_DIR='./'
CIBW_SKIP: "*-musllinux_* pp*"
CIBW_BEFORE_ALL: yum install -y openssl-devel make zlib-devel || apt-get install -y zlib1g-dev libssl-dev make || apk add zlib1g-dev libssl-dev make
CIBW_TEST_COMMAND: cd {project} && python mercury_python_test.py

- name: Upload to S3
env:
AWS_ACCESS_KEY_ID: ${{secrets.ACCESS_KEY_ID}}
AWS_SECRET_ACCESS_KEY: ${{secrets.ACCESS_KEY_SECRET}}
AWS_DEFAULT_REGION: us-east-1
S3_BUCKET: ${{secrets.S3_BUCKET}}
run: VERSION=$(cat src/cython/_version.py | tr -d '\n' | cut -d \' -f2) && aws s3 cp wheelhouse/ "s3://$S3_BUCKET/version=$VERSION" --recursive
25 changes: 4 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ else
endif

.PHONY: install install-no-systemd
install: install-mercury install-resources install-etc-config install-systemd
install-nosystemd: install-mercury install-resources install-etc-config
install: install-mercury install-etc-config
install-nosystemd: install-mercury install-etc-config

.PHONY: install-mercury
install-mercury:
Expand All @@ -33,17 +33,7 @@ else
$(INSTALLDATA) mercury /usr/share/bash-completion/completions/ # note: completion script has same name as binary
endif

.PHONY: install-resources
install-resources:
ifneq ($(wildcard src/Makefile), src/Makefile)
@echo $(COLOR_RED) "error: run ./configure before running make (src/Makefile is missing)" $(COLOR_OFF)
else
cd resources && $(MAKE) install
endif

# leave this variable empty; we want to force the user to set it, as a
# reminder that they should create a usable local configuration
MERCURY_CFG =
MERCURY_CFG = mercury.cfg
.PHONY: install-etc-config
install-etc-config:
ifneq ($(wildcard src/Makefile), src/Makefile)
Expand All @@ -53,10 +43,7 @@ ifneq ($(MERCURY_CFG),)
$(INSTALL) -d /etc/mercury
$(INSTALLDATA) $(MERCURY_CFG) /etc/mercury/mercury.cfg
else
@echo $(COLOR_RED) "error: you must specify the configuration file; run as 'make install MERCURY_CFG=filename'" $(COLOR_OFF)
@echo $(COLOR_RED) "where 'filename' is the configuration file you want to use for this installation. You can" $(COLOR_OFF)
@echo $(COLOR_RED) "use mercury.cfg as a template, but you *must* change the interface line to the appropriate" $(COLOR_OFF)
@echo $(COLOR_RED) "network interface for your system. (Use 'cat /proc/net/dev' to see Linux interfaces.)" $(COLOR_OFF)
@echo $(COLOR_RED) "error: no configuration file specified; run as 'make install MERCURY_CFG=filename'" $(COLOR_OFF)
@/bin/false
endif
endif
Expand All @@ -77,7 +64,6 @@ ifneq ($(wildcard src/Makefile), src/Makefile)
@echo $(COLOR_RED) "error: run ./configure before running make (src/Makefile is missing)" $(COLOR_OFF)
else
cd src && $(MAKE) install-nonroot
cd resources && $(MAKE) install-nonroot
endif

.PHONY: install-certtools
Expand All @@ -99,7 +85,6 @@ else
rm -f /etc/mercury/mercury.cfg
rm -rf /etc/mercury
cd src && $(MAKE) uninstall
cd resources && $(MAKE) uninstall
endif

.PHONY: uninstall-systemd
Expand Down Expand Up @@ -189,10 +174,8 @@ ifneq ($(wildcard src/Makefile), src/Makefile)
else
cd src && $(MAKE) distclean
cd test && $(MAKE) distclean
cd resources && $(MAKE) distclean
rm -rf autom4te.cache config.log config.status Makefile_helper.mk
rm -f lib/*.so
-git clean -xf
endif

.PHONY: package-deb
Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2.6.1
2.6.2
6 changes: 2 additions & 4 deletions build_pkg.sh
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,7 @@ if [ "$BUILDTYPE" == "deb" ]; then
--after-remove ./install_mercury/postuninstall_remove \
--deb-after-purge ./install_mercury/postuninstall_purge \
./src/mercury=/usr/local/bin/ mercury.cfg=/etc/mercury/ \
./mercury=/usr/share/bash-completion/completions/ \
./resources/resources.tgz=/usr/local/share/mercury/
./mercury=/usr/share/bash-completion/completions/

elif [ "$BUILDTYPE" == "rpm" ]; then

Expand All @@ -104,6 +103,5 @@ elif [ "$BUILDTYPE" == "rpm" ]; then
--description "$DESCRIPTION" \
--after-remove ./install_mercury/postuninstall_rpm \
./install_mercury/mercury.service=/usr/lib/systemd/system/ \
./src/mercury=/usr/local/bin/ mercury.cfg=/etc/mercury/ \
./resources/pyasn.db=/usr/local/share/mercury/ ./resources/fingerprint_db.json.gz=/usr/local/share/mercury/
./src/mercury=/usr/local/bin/ mercury.cfg=/etc/mercury/
fi
3 changes: 1 addition & 2 deletions configure
Original file line number Diff line number Diff line change
Expand Up @@ -5925,7 +5925,7 @@ else case e in #(
printf "%s\n" "$as_me: WARNING: llvm not found; make fuzz-test may not work" >&2;} ;;
esac
fi
ac_config_files="$ac_config_files src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile resources/Makefile Makefile_helper.mk install_mercury/mercury.service"
ac_config_files="$ac_config_files src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile Makefile_helper.mk install_mercury/mercury.service"

cat >confcache <<\_ACEOF
# This file is a shell script that caches the results of configure
Expand Down Expand Up @@ -6633,7 +6633,6 @@ do
"src/Makefile") CONFIG_FILES="$CONFIG_FILES src/Makefile" ;;
"test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;;
"unit_tests/Makefile") CONFIG_FILES="$CONFIG_FILES unit_tests/Makefile" ;;
"resources/Makefile") CONFIG_FILES="$CONFIG_FILES resources/Makefile" ;;
"Makefile_helper.mk") CONFIG_FILES="$CONFIG_FILES Makefile_helper.mk" ;;
"install_mercury/mercury.service") CONFIG_FILES="$CONFIG_FILES install_mercury/mercury.service" ;;
Expand Down
2 changes: 1 addition & 1 deletion configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -102,5 +102,5 @@ AS_IF([test "x$CLANGPP" = xyes],
AS_IF([test "x$LLVM" = xyes],
[],
[AC_MSG_WARN([llvm not found; make fuzz-test may not work])])
AC_CONFIG_FILES(src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile resources/Makefile Makefile_helper.mk install_mercury/mercury.service)
AC_CONFIG_FILES(src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile Makefile_helper.mk install_mercury/mercury.service)
AC_OUTPUT
26 changes: 26 additions & 0 deletions doc/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,31 @@
# CHANGELOG for Mercury

## VERSION 2.6.2
* Removed default interface from template configuration file
`mercury.cfg` and added runtime check to require that an interface
be specified when a configuration file is used.
* Added `--crypto-assess=<policy>` option, which implements an
assessment of the cryptographic security of TLS, DTLS, and QUIC
sessions and clients. The currently implemented policies are
`quantum_safe` and `quantum_safe_verbose`. The former is the
default, and the latter provides human-readable names.
* Integrated support for the IPSec protocols IKEv2 and ESP.
* Integrated minimal LDAP support, which provides details only for
`bindRequest` and `bindResponse` messages.
* Added support for the `LINKTYPE_LINUX_SLL` (Linux 'Cooked Capture').
* Improved the `stats` test to avoid spurious failues, by allowing
mercury JSON output and mercury stats counts to differ by up to
10% due to lossy stats collection.
* Enabled `--raw-features` and `--reassembly` to be enabled through
the configuration file.
* Refactored `pmercury` to use `c++` code where possible.
* Re-enabled ^C signal handler for PCAP processing.
* Moved `dns.id` and `ip.id` fields to the tail end of the JSON
record, to improve Parquet compressibility.
* Classification improved to allow multiple fingerprints to utilize
the same Weighted Naive Bayes models.
* Various internal improvements and unit test tweaks.

## VERSION 2.6.1
* Improved STUN implementation: added test cases, fixed fingerprint
feature nits, renamed variables for consistency with the RFCs, and
Expand Down
1 change: 1 addition & 0 deletions doc/sphinx/Doxyfile
Original file line number Diff line number Diff line change
Expand Up @@ -842,6 +842,7 @@ FILE_PATTERNS = *.c \
*.dox \
*.py \
*.pyw \
*.pyx \
*.f90 \
*.f95 \
*.f03 \
Expand Down
13 changes: 10 additions & 3 deletions doc/sphinx/source/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,17 @@
# -- General configuration ---------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

import os
import sys
sys.path.append("./docproj/ext/breathe/")
sys.path.append(os.path.abspath('../../../src/cython/'))

extensions = ['breathe']
extensions = [
'breathe',
'sphinx.ext.autodoc',
'sphinx.ext.viewcode',
'sphinx_autodoc_typehints'
]

breathe_default_project = "mercury"

Expand All @@ -29,7 +36,7 @@
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output

html_theme = 'sphinx_rtd_theme' # 'alabaster'
html_static_path = ['_static']
#html_static_path = ['_static']

###################################

Expand All @@ -38,5 +45,5 @@
}

breathe_projects_source = {
"mercury" : ( "../../../", ["src/libmerc/datum.h"] )
"mercury" : ( "../../../", ["src/libmerc/datum.h"])
}
6 changes: 6 additions & 0 deletions doc/sphinx/source/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,12 @@ Mercury Library Documentation
.. doxygengroup:: bitoperations
:project: mercury

Mercury Python Library Documentation
====================================

.. automodule:: mercury
:members:
:show-inheritance:


Indices and tables
Expand Down
12 changes: 6 additions & 6 deletions mercury.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
# ignored, as are unparseable lines.

# network interface for packet capture
capture = ens33
#capture = ens33

# name of JSON output file or directory for fingerprints and metadata
fingerprint = fingerprint.json
fingerprint = metadata.json

# filter out packets based on protocol
#select = dns,dhcp,dtls,tcp,http,tls,wireguard
Expand All @@ -23,13 +23,13 @@ fingerprint = fingerprint.json
# nonselected-udp-data

# 'dns-json' causes DNS responses to be reported with full detail in JSON
# dns-json
dns-json

# 'certs-json' causes certificates to be reported with full detail in JSON
# certs-json
certs-json

# 'metadata' causes extensive metadata to be reported in JSON
# metadata
metadata

# after dropping root privileges, change to this user
user = mercury
Expand All @@ -47,7 +47,7 @@ threads = cpu
buffer = 0.05

# perform analysis, include results in JSON output file
analysis = 1
#analysis = 1

# set resource directory
# resources = /usr/local/share/mercury/resources.tgz
Expand Down
6 changes: 0 additions & 6 deletions python/MANIFEST.in

This file was deleted.

Loading

0 comments on commit 9139040

Please sign in to comment.