[NodeKiller] Add summary

Author: jprzychodzen

clusterloader2/pkg/chaos/monkey.go

@@ -17,6 +17,9 @@ limitations under the License.
 package chaos
 
 import (
+	"fmt"
+	"strings"
+
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/perf-tests/clusterloader2/api"
 )
@@ -47,3 +50,13 @@ func (m *Monkey) Init(config api.ChaosMonkeyConfig, stopCh <-chan struct{}) erro
 
 	return nil
 }
+
+// Summary logs Monkey execution
+func (m *Monkey) Summary() string {
+	var sb strings.Builder
+	if m.nodeKiller != nil {
+		sb.WriteString(fmt.Sprintf("Summary of Chaos Monkey execution\n"))
+		sb.WriteString(m.nodeKiller.Summary())
+	}
+	return sb.String()
+}

clusterloader2/pkg/chaos/nodes.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"math"
 	"math/rand"
+	"strings"
 	"sync"
 	"time"
 
@@ -47,14 +48,44 @@ type NodeKiller struct {
 	provider string
 	// killedNodes stores names of the nodes that have been killed by NodeKiller.
 	killedNodes sets.String
+	recorder    *eventRecorder
+}
+
+type nodeAction string
+
+const (
+	stopServices nodeAction = "stopService"
+	rebootNode              = "rebootNode"
+)
+
+type event struct {
+	time     time.Time
+	action   nodeAction
+	nodeName string
+}
+
+type eventRecorder struct {
+	events []event
+	mux    sync.Mutex
+}
+
+func newEventRecorder() *eventRecorder {
+	return &eventRecorder{[]event{}, sync.Mutex{}}
+}
+
+func (r *eventRecorder) record(a nodeAction, nodeName string) {
+	e := event{time.Now(), a, nodeName}
+	r.mux.Lock()
+	r.events = append(r.events, e)
+	r.mux.Unlock()
 }
 
 // NewNodeKiller creates new NodeKiller.
 func NewNodeKiller(config api.NodeFailureConfig, client clientset.Interface, provider string) (*NodeKiller, error) {
 	if provider != "gce" && provider != "gke" {
 		return nil, fmt.Errorf("provider %q is not supported by NodeKiller", provider)
 	}
-	return &NodeKiller{config, client, provider, sets.NewString()}, nil
+	return &NodeKiller{config, client, provider, sets.NewString(), newEventRecorder()}, nil
 }
 
 // Run starts NodeKiller until stopCh is closed.
@@ -121,6 +152,7 @@ func (k *NodeKiller) kill(nodes []v1.Node) {
 			defer wg.Done()
 
 			klog.Infof("%s: Stopping docker and kubelet on %q to simulate failure", k, node.Name)
+			k.addStopServicesEvent(node.Name)
 			err := util.SSH("sudo systemctl stop docker kubelet", &node, nil)
 			if err != nil {
 				klog.Errorf("%s: ERROR while stopping node %q: %v", k, node.Name, err)
@@ -141,6 +173,7 @@ func (k *NodeKiller) kill(nodes []v1.Node) {
 			// '> /dev/null 2> /dev/null < /dev/null' - File descriptor redirect, all three I/O to avoid ssh hanging,
 			//                                          see https://web.archive.org/web/20090429074212/http://www.openssh.com/faq.html#3.10
 			// '&' - Execute command in background, end without waiting for result
+			k.addRebootEvent(node.Name)
 			err = util.SSH("nohup sudo shutdown -r +1s > /dev/null 2> /dev/null < /dev/null &", &node, nil)
 			if err != nil {
 				klog.Errorf("%s: Error while rebooting node %q: %v", k, node.Name, err)
@@ -151,6 +184,24 @@ func (k *NodeKiller) kill(nodes []v1.Node) {
 	wg.Wait()
 }
 
+func (k *NodeKiller) addStopServicesEvent(nodeName string) {
+	k.recorder.record(stopServices, nodeName)
+}
+
+func (k *NodeKiller) addRebootEvent(nodeName string) {
+	k.recorder.record(rebootNode, nodeName)
+}
+
+// Summary logs NodeKiller execution
+func (k *NodeKiller) Summary() string {
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf("%s: Recorded following events\n", k))
+	for _, e := range k.recorder.events {
+		sb.WriteString(fmt.Sprintf("%s: At %v %v happend for node %s\n", k, e.time.Format(time.UnixDate), e.action, e.nodeName))
+	}
+	return sb.String()
+}
+
 func (k *NodeKiller) String() string {
 	return "NodeKiller"
 }

clusterloader2/pkg/test/simple_test_executor.go

@@ -102,6 +102,7 @@ func (ste *simpleTestExecutor) ExecuteTest(ctx Context, conf *api.Config) *error
 			}
 		}
 	}
+	klog.Infof(ctx.GetChaosMonkey().Summary())
 	return errList
 }