Skip to content

Commit 7e711d0

Browse files
mye956mye956
committed
Adding unit tests for fault request ordering
1 parent 75ee48f commit 7e711d0

File tree

1 file changed

+308
-4
lines changed

1 file changed

+308
-4
lines changed

‎ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go‎

Lines changed: 308 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"io"
2626
"net/http"
2727
"net/http/httptest"
28+
"strings"
2829
"testing"
2930
"time"
3031

@@ -59,6 +60,9 @@ const (
5960
tcLatencyFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","parent":"1:1","options":{"limit":1000,"delay":{"delay":123456789,"jitter":4567,"correlation":0},"ecn":false,"gap":0}}]`
6061
tcLossFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","dev":"eth0","parent":"1:1","options":{"limit":1000,"loss-random":{"loss":0.06,"correlation":0},"ecn":false,"gap":0}}]`
6162
tcCommandEmptyOutput = `[]`
63+
startEndpoint = "/api/%s/fault/v1/%s/start"
64+
stopEndpoint = "/api/%s/fault/v1/%s/stop"
65+
path = "/some/path"
6266
)
6367

6468
var (
@@ -76,7 +80,7 @@ var (
7680

7781
happyNetworkNamespaces = []*state.NetworkNamespace{
7882
{
79-
Path: "/some/path",
83+
Path: path,
8084
NetworkInterfaces: happyNetworkInterfaces,
8185
},
8286
}
@@ -2027,9 +2031,13 @@ func generateStopNetworkPacketLossTestCases() []networkFaultInjectionTestCase {
20272031
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
20282032
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
20292033
mockCMD := mock_execwrapper.NewMockCmd(ctrl)
2030-
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel)
2031-
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD)
2032-
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultExistsCommandOutput), nil)
2034+
gomock.InOrder(
2035+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel),
2036+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD),
2037+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLossFaultExistsCommandOutput), nil),
2038+
)
2039+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(2).Return(mockCMD)
2040+
mockCMD.EXPECT().CombinedOutput().Times(2).Return([]byte(""), nil)
20332041
},
20342042
},
20352043
{
@@ -2175,3 +2183,299 @@ func TestCheckNetworkPacketLoss(t *testing.T) {
21752183
tcs := generateCheckNetworkPacketLossTestCases()
21762184
testNetworkFaultInjectionCommon(t, tcs, NetworkFaultPath(types.PacketLossFaultType, types.CheckNetworkFaultPostfix))
21772185
}
2186+
2187+
func TestNetworkFaultRequestOrdering(t *testing.T) {
2188+
tcs := []struct {
2189+
name string
2190+
faultType string
2191+
requestBody interface{}
2192+
setAgentStateExpectations func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient)
2193+
setExecExpectations func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller, firstStartExecCmd, firstStopExecCmd []interface{})
2194+
}{
2195+
{
2196+
name: types.BlackHolePortFaultType + "request ordering",
2197+
faultType: types.BlackHolePortFaultType,
2198+
requestBody: happyBlackHolePortReqBody,
2199+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2200+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).
2201+
Return(happyTaskResponse, nil).
2202+
Times(2)
2203+
},
2204+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller, firstStartExecCmd, firstStopExecCmd []interface{}) {
2205+
startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2206+
stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2207+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
2208+
// We want to ensure that the start fault request executes and finishes first before the stop fault request.
2209+
// We can enforce the ordering of exec mock calls.
2210+
gomock.InOrder(
2211+
// Exec mocks for start black hole port request
2212+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) {
2213+
// Sleep for 2 seconds to mock that the request is taking some time
2214+
time.Sleep(2 * time.Second)
2215+
}).Times(1).Return(startCtx, startCancel),
2216+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2217+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(iptablesChainNotFoundError), errors.New("exit status 1")),
2218+
exec.EXPECT().ConvertToExitError(gomock.Any()).Times(1).Return(nil, true),
2219+
exec.EXPECT().GetExitCode(gomock.Any()).Times(1).Return(1),
2220+
// Ensuring that the start request is running here by also passing in the expected parameters for the first CommandContext call
2221+
exec.EXPECT().CommandContext(gomock.Any(), firstStartExecCmd[0], firstStartExecCmd[1:]...).Times(1).Return(cmdExec),
2222+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2223+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2224+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2225+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2226+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2227+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2228+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2229+
2230+
// Exec mocks for stop black hole port request
2231+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel),
2232+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2233+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2234+
// Ensuring that the stop request is running here by also passing in the expected parameters for the first CommandContext call
2235+
exec.EXPECT().CommandContext(gomock.Any(), firstStopExecCmd[0], firstStopExecCmd[1:]...).Times(1).Return(cmdExec),
2236+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2237+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2238+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2239+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2240+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2241+
)
2242+
},
2243+
},
2244+
{
2245+
name: types.LatencyFaultType + "request ordering",
2246+
faultType: types.LatencyFaultType,
2247+
requestBody: happyNetworkLatencyReqBody,
2248+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2249+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).
2250+
Return(happyTaskResponse, nil).
2251+
Times(2)
2252+
},
2253+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller, firstStartExecCmd, firstStopExecCmd []interface{}) {
2254+
startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2255+
stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2256+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
2257+
// We want to ensure that the start fault request executes and finishes first before the stop fault request.
2258+
// We can enforce the ordering of exec mock calls.
2259+
gomock.InOrder(
2260+
// Exec mocks for start latency request
2261+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) {
2262+
// Sleep for 2 seconds to mock that the request is taking some time
2263+
time.Sleep(2 * time.Second)
2264+
}).Times(1).Return(startCtx, startCancel),
2265+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2266+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2267+
// Ensuring that the start request is running here by also passing in the expected parameters for the first CommandContext call
2268+
exec.EXPECT().CommandContext(gomock.Any(), firstStartExecCmd[0], firstStartExecCmd[1:]...).Times(1).Return(cmdExec),
2269+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2270+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2271+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2272+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2273+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2274+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2275+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2276+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2277+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2278+
2279+
// Exec mocks for stop latency request
2280+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel),
2281+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2282+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultExistsCommandOutput), nil),
2283+
// Ensuring that the stop request is running here by also passing in the expected parameters for the first CommandContext call
2284+
exec.EXPECT().CommandContext(gomock.Any(), firstStopExecCmd[0], firstStopExecCmd[1:]...).Times(1).Return(cmdExec),
2285+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2286+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2287+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2288+
)
2289+
},
2290+
},
2291+
{
2292+
name: types.PacketLossFaultType + "request ordering",
2293+
faultType: types.PacketLossFaultType,
2294+
requestBody: happyNetworkPacketLossReqBody,
2295+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2296+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).
2297+
Return(happyTaskResponse, nil).
2298+
Times(2)
2299+
},
2300+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller, firstStartExecCmd, firstStopExecCmd []interface{}) {
2301+
startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2302+
stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2303+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
2304+
// We want to ensure that the start fault request executes and finishes first before the stop fault request.
2305+
// We can enforce the ordering of exec mock calls.
2306+
gomock.InOrder(
2307+
// Exec mocks for start packet loss request
2308+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) {
2309+
// Sleep for 2 seconds to mock that the request is taking some time
2310+
time.Sleep(2 * time.Second)
2311+
}).Times(1).Return(startCtx, startCancel),
2312+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2313+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2314+
// Ensuring that the start request is running here by also passing in the expected parameters for the first CommandContext call
2315+
exec.EXPECT().CommandContext(gomock.Any(), firstStartExecCmd[0], firstStartExecCmd[1:]...).Times(1).Return(cmdExec),
2316+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2317+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2318+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2319+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2320+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2321+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2322+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2323+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2324+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2325+
2326+
// Exec mocks for stop packet loss request
2327+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel),
2328+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2329+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLossFaultExistsCommandOutput), nil),
2330+
// Ensuring that the stop request is running here by also passing in the expected parameters for the first CommandContext call
2331+
exec.EXPECT().CommandContext(gomock.Any(), firstStopExecCmd[0], firstStopExecCmd[1:]...).Times(1).Return(cmdExec),
2332+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2333+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2334+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2335+
)
2336+
},
2337+
},
2338+
}
2339+
2340+
for _, tc := range tcs {
2341+
t.Run(tc.name, func(t *testing.T) {
2342+
// Mocks
2343+
ctrl := gomock.NewController(t)
2344+
defer ctrl.Finish()
2345+
2346+
agentState := mock_state.NewMockAgentState(ctrl)
2347+
metricsFactory := mock_metrics.NewMockEntryFactory(ctrl)
2348+
2349+
router := mux.NewRouter()
2350+
mockExec := mock_execwrapper.NewMockExec(ctrl)
2351+
handler := New(agentState, metricsFactory, mockExec)
2352+
networkConfigClient := netconfig.NewNetworkConfigClient()
2353+
2354+
var startHandleMethod, stopHandleMethod func(http.ResponseWriter, *http.Request)
2355+
var firstStartExecCmd, firstStopExecCmd []string
2356+
nsenterPrefix := fmt.Sprintf(nsenterCommandString, path)
2357+
switch tc.faultType {
2358+
case types.BlackHolePortFaultType:
2359+
chain := fmt.Sprintf("%s-%s-%d", trafficType, protocol, port)
2360+
2361+
newChainCmdString := nsenterPrefix + fmt.Sprintf(iptablesNewChainCmd, requestTimeoutSeconds, chain)
2362+
firstStartExecCmd = strings.Split(newChainCmdString, " ")
2363+
2364+
clearChainCmdString := nsenterPrefix + fmt.Sprintf(iptablesClearChainCmd, requestTimeoutSeconds, chain)
2365+
firstStopExecCmd = strings.Split(clearChainCmdString, " ")
2366+
2367+
startHandleMethod = handler.StartNetworkBlackholePort()
2368+
stopHandleMethod = handler.StopNetworkBlackHolePort()
2369+
case types.LatencyFaultType:
2370+
tcAddQdiscRootCommandComposed := nsenterPrefix + fmt.Sprintf(tcAddQdiscRootCommandString, deviceName)
2371+
firstStartExecCmd = strings.Split(tcAddQdiscRootCommandComposed, " ")
2372+
2373+
tcDeleteQdiscParentCommandComposed := nsenterPrefix + fmt.Sprintf(tcDeleteQdiscParentCommandString, deviceName)
2374+
firstStopExecCmd = strings.Split(tcDeleteQdiscParentCommandComposed, " ")
2375+
2376+
startHandleMethod = handler.StartNetworkLatency()
2377+
stopHandleMethod = handler.StopNetworkLatency()
2378+
case types.PacketLossFaultType:
2379+
tcAddQdiscRootCommandComposed := nsenterPrefix + fmt.Sprintf(tcAddQdiscRootCommandString, deviceName)
2380+
firstStartExecCmd = strings.Split(tcAddQdiscRootCommandComposed, " ")
2381+
2382+
tcDeleteQdiscParentCommandComposed := nsenterPrefix + fmt.Sprintf(tcDeleteQdiscParentCommandString, deviceName)
2383+
firstStopExecCmd = strings.Split(tcDeleteQdiscParentCommandComposed, " ")
2384+
2385+
startHandleMethod = handler.StartNetworkPacketLoss()
2386+
stopHandleMethod = handler.StopNetworkPacketLoss()
2387+
default:
2388+
t.Error("Unrecognized network fault type")
2389+
}
2390+
2391+
tc.setAgentStateExpectations(agentState, networkConfigClient)
2392+
tc.setExecExpectations(mockExec, ctrl, convertToInterfaceList(firstStartExecCmd), convertToInterfaceList(firstStopExecCmd))
2393+
2394+
router.HandleFunc(
2395+
NetworkFaultPath(tc.faultType, types.StartNetworkFaultPostfix),
2396+
startHandleMethod,
2397+
).Methods(http.MethodPost)
2398+
2399+
router.HandleFunc(
2400+
NetworkFaultPath(tc.faultType, types.StopNetworkFaultPostfix),
2401+
stopHandleMethod,
2402+
).Methods(http.MethodPost)
2403+
2404+
var requestBody io.Reader
2405+
reqBodyBytes, err := json.Marshal(tc.requestBody)
2406+
require.NoError(t, err)
2407+
requestBody = bytes.NewReader(reqBodyBytes)
2408+
startReq, err := http.NewRequest(http.MethodPost, fmt.Sprintf(startEndpoint, endpointId, tc.faultType), requestBody)
2409+
require.NoError(t, err)
2410+
2411+
ch1 := make(chan struct {
2412+
int
2413+
error
2414+
})
2415+
2416+
reqBodyBytes, err = json.Marshal(tc.requestBody)
2417+
require.NoError(t, err)
2418+
requestBody = bytes.NewReader(reqBodyBytes)
2419+
stopReq, err := http.NewRequest(http.MethodPost, fmt.Sprintf(stopEndpoint, endpointId, tc.faultType), requestBody)
2420+
require.NoError(t, err)
2421+
2422+
ch2 := make(chan struct {
2423+
int
2424+
error
2425+
})
2426+
2427+
// Make an asynchronous Start request first
2428+
go makeAsyncRequest(router, startReq, ch1)
2429+
2430+
// Waiting a bit before sending the stop request
2431+
time.Sleep(1 * time.Second)
2432+
2433+
// Make an asynchronous Stop request second
2434+
go makeAsyncRequest(router, stopReq, ch2)
2435+
2436+
// Waiting to get the status code of the start request
2437+
resp1 := <-ch1
2438+
require.NoError(t, resp1.error)
2439+
assert.Equal(t, http.StatusOK, resp1.int)
2440+
2441+
// Waiting to get the status code of the stop request
2442+
resp2 := <-ch2
2443+
require.NoError(t, resp2.error)
2444+
assert.Equal(t, http.StatusOK, resp2.int)
2445+
})
2446+
}
2447+
}
2448+
2449+
// Helper function for making asynchronous mock HTTP requests
2450+
func makeAsyncRequest(router *mux.Router, req *http.Request, ch chan<- struct {
2451+
int
2452+
error
2453+
}) {
2454+
defer close(ch)
2455+
2456+
// Makes a mock HTTP request
2457+
recorder := httptest.NewRecorder()
2458+
router.ServeHTTP(recorder, req)
2459+
2460+
var actualResponseBody types.NetworkFaultInjectionResponse
2461+
err := json.Unmarshal(recorder.Body.Bytes(), &actualResponseBody)
2462+
if err != nil {
2463+
ch <- struct {
2464+
int
2465+
error
2466+
}{-1, err}
2467+
} else {
2468+
ch <- struct {
2469+
int
2470+
error
2471+
}{recorder.Code, nil}
2472+
}
2473+
}
2474+
2475+
func convertToInterfaceList(strings []string) []interface{} {
2476+
interfaces := make([]interface{}, len(strings))
2477+
for i, s := range strings {
2478+
interfaces[i] = s
2479+
}
2480+
return interfaces
2481+
}

0 commit comments

Comments
 (0)