From 7c1f0694c20c5c0cf32ba01f92ecd905a680bc06 Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Thu, 16 Jan 2025 10:36:44 +0100 Subject: [PATCH] integration-test Having a test suite that covers the code from an out-of-process perspective helps in testing specific scenarios, e.g. when the concurrency of multiple program instances occurs. Also, using test doubles instead of real SR-IOV capable NICs makes tests fast and portable, in a way that can be run at every commit. Add a test suite that invokes a system-mocked version of the sriov CNI binary, in which the `/sys` filesystem content is defined at code, and the network interfaces are of type dummy. Add `test-integration` make target to invoke the test suite. Signed-off-by: Andrea Panattoni --- .github/workflows/buildtest.yml | 6 ++ Makefile | 8 ++ pkg/utils/testing.go | 154 +++++++++++++++++++++++++++++ test/integration/README.md | 53 ++++++++++ test/integration/sriov-mocked.go | 52 ++++++++++ test/integration/test-ipam-cni | 18 ++++ test/integration/test_sriov_cni.sh | 114 +++++++++++++++++++++ 7 files changed, 405 insertions(+) create mode 100644 test/integration/README.md create mode 100644 test/integration/sriov-mocked.go create mode 100755 test/integration/test-ipam-cni create mode 100644 test/integration/test_sriov_cni.sh diff --git a/.github/workflows/buildtest.yml b/.github/workflows/buildtest.yml index 278c92f31..6fb59ec24 100644 --- a/.github/workflows/buildtest.yml +++ b/.github/workflows/buildtest.yml @@ -33,6 +33,12 @@ jobs: if: ${{ matrix.goarch }} == "amd64" run: sudo make test-race # sudo needed for netns change in test + - name: Integration test for ${{ matrix.goarch }} + env: + GOARCH: ${{ matrix.goarch }} + GOOS: ${{ matrix.goos }} + run: sudo make test-integration + coverage: runs-on: ubuntu-latest needs: build-test diff --git a/Makefile b/Makefile index 307318066..d4ee01e4d 100644 --- a/Makefile +++ b/Makefile @@ -105,6 +105,14 @@ image: ; $(info Building Docker image...) @ ## Build SR-IOV CNI docker image test-image: image $Q $(IMAGEDIR)/image_test.sh $(IMAGE_BUILDER) $(TAG) +BASH_UNIT=$(BINDIR)/bash_unit +$(BASH_UNIT): $(BINDIR) + curl -L https://github.com/pgrange/bash_unit/raw/refs/tags/v2.3.2/bash_unit > bin/bash_unit + chmod a+x bin/bash_unit + +test-integration: $(BASH_UNIT) + $(BASH_UNIT) test/integration/test_*.sh + # Misc .PHONY: deps-update deps-update: ; $(info Updating dependencies...) @ ## Update dependencies diff --git a/pkg/utils/testing.go b/pkg/utils/testing.go index 7db0c19de..04502f96c 100644 --- a/pkg/utils/testing.go +++ b/pkg/utils/testing.go @@ -7,6 +7,9 @@ package utils import ( + "fmt" + "log" + "net" "os" "path/filepath" "syscall" @@ -154,3 +157,154 @@ func (l *FakeLink) Attrs() *netlink.LinkAttrs { func (l *FakeLink) Type() string { return "FakeLink" } + + +func MockNetlinkLib(methodCallRecordingDir string) (func(), error) { + var err error + oldnetlinkLib := netLinkLib + // see `ts` variable in this file + // "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1/sriov_numvfs": []byte("2"), + netLinkLib, err = newPFMockNetlinkLib(methodCallRecordingDir, "enp175s0f1", 2) + + return func() { + netLinkLib = oldnetlinkLib + }, err +} + +// pfMockNetlinkLib creates dummy interfaces for Physical and Virtual functions, recording method calls on a log file in the form +// ... +type pfMockNetlinkLib struct { + pf netlink.Link + methodCallsRecordingFilePath string +} + +func newPFMockNetlinkLib(recordDir, pfName string, numvfs int) (*pfMockNetlinkLib, error) { + ret := &pfMockNetlinkLib{ + pf: &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: pfName, + Vfs: []netlink.VfInfo{}, + }, + }, + } + + for i := 0; i.calls)] + PF{{PF << dummy >>}} + VF1{{VF1 << dummy >>}} + VF2{{VF2 << dummy >>}} + end + + test_sriov_cni.sh + + test_sriov_cni.sh --> sriovmocked + + cnicommands_pkg --> CreateTmpSysFs + cnicommands_pkg --> MockNetlinkLib + + MockNetlinkLib -.write.- calls_file + MockNetlinkLib -..- PF + MockNetlinkLib -..- VF1 + MockNetlinkLib -..- VF2 + + test_sriov_cni.sh -.read.- calls_file + + linkStyle default stroke-width:2px + linkStyle 1,4,5,6 stroke:green,stroke-width:4px +``` diff --git a/test/integration/sriov-mocked.go b/test/integration/sriov-mocked.go new file mode 100644 index 000000000..6a416227e --- /dev/null +++ b/test/integration/sriov-mocked.go @@ -0,0 +1,52 @@ +package main + +import ( + "os" + "runtime" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/version" + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/cnicommands" + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/config" + "github.com/k8snetworkplumbingwg/sriov-cni/pkg/utils" +) + +func init() { + // this ensures that main runs only on main thread (thread group leader). + // since namespace ops (unshare, setns) are done for a single thread, we + // must ensure that the goroutine does not jump from OS thread to thread + runtime.LockOSThread() +} + +func main() { + customCNIDir, ok := os.LookupEnv("DEFAULT_CNI_DIR") + if ok { + config.DefaultCNIDir = customCNIDir + } + + err := utils.CreateTmpSysFs() + if err != nil { + panic(err) + } + + defer func() { + err := utils.RemoveTmpSysFs() + if err != nil { + panic(err) + } + }() + + cancel, err := utils.MockNetlinkLib(config.DefaultCNIDir) + if err != nil { + panic(err) + } + defer cancel() + + + cniFuncs := skel.CNIFuncs{ + Add: cnicommands.CmdAdd, + Del: cnicommands.CmdDel, + Check: cnicommands.CmdCheck, + } + skel.PluginMainFuncs(cniFuncs, version.All, "") +} \ No newline at end of file diff --git a/test/integration/test-ipam-cni b/test/integration/test-ipam-cni new file mode 100755 index 000000000..057f0449a --- /dev/null +++ b/test/integration/test-ipam-cni @@ -0,0 +1,18 @@ +#!/bin/bash + +if [[ -n "${IPAM_MOCK_SLEEP}" ]]; then + sleep "${IPAM_MOCK_SLEEP}" +fi + +cat << EOF +{ + "cniVersion": "0.3.1", + "interfaces": [{ + "name": "${CNI_IFNAME}" + }], + "ips": [{ + "name": "${CNI_IFNAME}", + "address": "192.0.2.1/24" + }] +} +EOF diff --git a/test/integration/test_sriov_cni.sh b/test/integration/test_sriov_cni.sh new file mode 100644 index 000000000..8f8301514 --- /dev/null +++ b/test/integration/test_sriov_cni.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +test_image="docker.io/library/busybox:1.36" +this_folder="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")" +export CNI_PATH=${this_folder} + +setup() { + ip netns del test_root_ns || true + ip netns add test_root_ns + + # See pkg/utils/testing.go + ip netns exec test_root_ns ip link add enp175s0f1 type dummy + ip netns exec test_root_ns ip link add enp175s6 type dummy + ip netns exec test_root_ns ip link add enp175s7 type dummy + + DEFAULT_CNI_DIR=$(mktemp -d) + export DEFAULT_CNI_DIR +} + + +test_macaddress() { + + make_container "container_1" + + export CNI_CONTAINERID=container_1 + export CNI_NETNS=/run/netns/container_1_netns + export CNI_IFNAME=net1 + + read -r -d '' CNI_INPUT <<- EOM + { + "type": "sriov", + "cniVersion": "0.3.1", + "name": "sriov-network", + "ipam": { + "type": "test-ipam-cni" + }, + "deviceID": "0000:af:06.0", + "mac": "60:00:00:00:00:E1", + "logLevel": "debug" + } +EOM + + export CNI_COMMAND=ADD + assert invoke_sriov_cni + assert 'ip netns exec container_1_netns ip link | grep -i 60:00:00:00:00:E1' + + export CNI_COMMAND=DEL + assert 'invoke_sriov_cni' + assert 'ip netns exec test_root_ns ip link show enp175s6' +} + + +test_vlan() { + + make_container "container_1" + + export CNI_CONTAINERID=container_1 + export CNI_NETNS=/run/netns/container_1_netns + export CNI_IFNAME=net1 + + read -r -d '' CNI_INPUT <<- EOM + { + "type": "sriov", + "cniVersion": "0.3.1", + "name": "sriov-network", + "vlan": 1234, + "ipam": { + "type": "test-ipam-cni" + }, + "deviceID": "0000:af:06.0", + "mac": "60:00:00:00:00:E1", + "logLevel": "debug" + } +EOM + + export CNI_COMMAND=ADD + assert invoke_sriov_cni + assert_file_contains "${DEFAULT_CNI_DIR}/enp175s0f1.calls" "LinkSetVfVlanQosProto enp175s0f1 0 1234 0 33024" + + export CNI_COMMAND=DEL + assert invoke_sriov_cni + assert 'ip netns exec test_root_ns ip link show enp175s6' + assert_file_contains "${DEFAULT_CNI_DIR}/enp175s0f1.calls" "LinkSetVfVlanQosProto enp175s0f1 0 0 0 33024" +} + +invoke_sriov_cni() { + echo "$CNI_INPUT" | ip netns exec test_root_ns go run "${this_folder}/sriov-mocked.go" +} + +# Create a container and its related network namespace. The first parameter is +# the name of the container, and as a convention, the netns name is `_netns` +make_container() { + container_name=$1 + delete_container "$container_name" + + ip netns add "${container_name}_netns" + assert "podman run -d --network ns:/run/netns/${container_name}_netns --name ${container_name} ${test_image} sleep inf" +} + +delete_container() { + container_name=$1 + ip netns del "${container_name}_netns" 2>/dev/null + + podman kill "${container_name}" >/dev/null 2>/dev/null + podman rm -f "${container_name}" >/dev/null 2>/dev/null +} + +assert_file_contains() { + file=$1 + substr=$2 + if ! grep -q "$substr" "$file"; then + fail "File [$file] does not contains [$substr], contents: \n $(cat "$file")" + fi +}