Skip to content

Commit 01dea21

Browse files
mye956mye956
committed
Adding unit tests for fault request ordering
1 parent 7c25e82 commit 01dea21

File tree

1 file changed

+306
-4
lines changed

1 file changed

+306
-4
lines changed

‎ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go‎

Lines changed: 306 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"io"
2626
"net/http"
2727
"net/http/httptest"
28+
"strings"
2829
"testing"
2930
"time"
3031

@@ -59,6 +60,9 @@ const (
5960
tcLatencyFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","parent":"1:1","options":{"limit":1000,"delay":{"delay":123456789,"jitter":4567,"correlation":0},"ecn":false,"gap":0}}]`
6061
tcLossFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","dev":"eth0","parent":"1:1","options":{"limit":1000,"loss-random":{"loss":0.06,"correlation":0},"ecn":false,"gap":0}}]`
6162
tcCommandEmptyOutput = `[]`
63+
startEndpoint = "/api/%s/fault/v1/%s/start"
64+
stopEndpoint = "/api/%s/fault/v1/%s/stop"
65+
path = "/some/path"
6266
)
6367

6468
var (
@@ -76,7 +80,7 @@ var (
7680

7781
happyNetworkNamespaces = []*state.NetworkNamespace{
7882
{
79-
Path: "/some/path",
83+
Path: path,
8084
NetworkInterfaces: happyNetworkInterfaces,
8185
},
8286
}
@@ -2027,9 +2031,13 @@ func generateStopNetworkPacketLossTestCases() []networkFaultInjectionTestCase {
20272031
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
20282032
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
20292033
mockCMD := mock_execwrapper.NewMockCmd(ctrl)
2030-
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel)
2031-
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD)
2032-
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultExistsCommandOutput), nil)
2034+
gomock.InOrder(
2035+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel),
2036+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD),
2037+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLossFaultExistsCommandOutput), nil),
2038+
)
2039+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(2).Return(mockCMD)
2040+
mockCMD.EXPECT().CombinedOutput().Times(2).Return([]byte(""), nil)
20332041
},
20342042
},
20352043
{
@@ -2175,3 +2183,297 @@ func TestCheckNetworkPacketLoss(t *testing.T) {
21752183
tcs := generateCheckNetworkPacketLossTestCases()
21762184
testNetworkFaultInjectionCommon(t, tcs, NetworkFaultPath(types.PacketLossFaultType, types.CheckNetworkFaultPostfix))
21772185
}
2186+
2187+
func TestNetworkFaultRequestOrdering(t *testing.T) {
2188+
tcs := []struct {
2189+
name string
2190+
faultType string
2191+
requestBody interface{}
2192+
setAgentStateExpectations func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient)
2193+
setExecExpectations func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller, firstStartExecCmd, firstStopExecCmd []interface{})
2194+
}{
2195+
{
2196+
name: types.BlackHolePortFaultType + "request ordering",
2197+
faultType: types.BlackHolePortFaultType,
2198+
requestBody: happyBlackHolePortReqBody,
2199+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2200+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).
2201+
Return(happyTaskResponse, nil).
2202+
Times(2)
2203+
},
2204+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller, firstStartExecCmd, firstStopExecCmd []interface{}) {
2205+
startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2206+
stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2207+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
2208+
// We want to ensure that the start fault request executes and finishes first before the stop fault request.
2209+
// We can enforce the ordering of exec mock calls.
2210+
gomock.InOrder(
2211+
// Exec mocks for start black hole port request
2212+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) {
2213+
// Sleep for 2 seconds to mock that the request is taking some time
2214+
time.Sleep(2 * time.Second)
2215+
}).Times(1).Return(startCtx, startCancel),
2216+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2217+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(iptablesChainNotFoundError), errors.New("exit status 1")),
2218+
exec.EXPECT().ConvertToExitError(gomock.Any()).Times(1).Return(nil, true),
2219+
exec.EXPECT().GetExitCode(gomock.Any()).Times(1).Return(1),
2220+
// Ensuring that the start request is running here by also passing in the expected parameters for the first CommandContext call
2221+
exec.EXPECT().CommandContext(gomock.Any(), firstStartExecCmd[0], firstStartExecCmd[1:]...).Times(1).Return(cmdExec),
2222+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2223+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2224+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2225+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2226+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2227+
2228+
// Exec mocks for stop black hole port request
2229+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel),
2230+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2231+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2232+
// Ensuring that the stop request is running here by also passing in the expected parameters for the first CommandContext call
2233+
exec.EXPECT().CommandContext(gomock.Any(), firstStopExecCmd[0], firstStopExecCmd[1:]...).Times(1).Return(cmdExec),
2234+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2235+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2236+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2237+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2238+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2239+
)
2240+
},
2241+
},
2242+
{
2243+
name: types.LatencyFaultType + "request ordering",
2244+
faultType: types.LatencyFaultType,
2245+
requestBody: happyNetworkLatencyReqBody,
2246+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2247+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).
2248+
Return(happyTaskResponse, nil).
2249+
Times(2)
2250+
},
2251+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller, firstStartExecCmd, firstStopExecCmd []interface{}) {
2252+
startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2253+
stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2254+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
2255+
// We want to ensure that the start fault request executes and finishes first before the stop fault request.
2256+
// We can enforce the ordering of exec mock calls.
2257+
gomock.InOrder(
2258+
// Exec mocks for start latency request
2259+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) {
2260+
// Sleep for 2 seconds to mock that the request is taking some time
2261+
time.Sleep(2 * time.Second)
2262+
}).Times(1).Return(startCtx, startCancel),
2263+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2264+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2265+
// Ensuring that the start request is running here by also passing in the expected parameters for the first CommandContext call
2266+
exec.EXPECT().CommandContext(gomock.Any(), firstStartExecCmd[0], firstStartExecCmd[1:]...).Times(1).Return(cmdExec),
2267+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2268+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2269+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2270+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2271+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2272+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2273+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2274+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2275+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2276+
2277+
// Exec mocks for stop latency request
2278+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel),
2279+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2280+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultExistsCommandOutput), nil),
2281+
// Ensuring that the stop request is running here by also passing in the expected parameters for the first CommandContext call
2282+
exec.EXPECT().CommandContext(gomock.Any(), firstStopExecCmd[0], firstStopExecCmd[1:]...).Times(1).Return(cmdExec),
2283+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2284+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2285+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2286+
)
2287+
},
2288+
},
2289+
{
2290+
name: types.PacketLossFaultType + "request ordering",
2291+
faultType: types.PacketLossFaultType,
2292+
requestBody: happyNetworkPacketLossReqBody,
2293+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2294+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).
2295+
Return(happyTaskResponse, nil).
2296+
Times(2)
2297+
},
2298+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller, firstStartExecCmd, firstStopExecCmd []interface{}) {
2299+
startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2300+
stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2301+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
2302+
// We want to ensure that the start fault request executes and finishes first before the stop fault request.
2303+
// We can enforce the ordering of exec mock calls.
2304+
gomock.InOrder(
2305+
// Exec mocks for start packet loss request
2306+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) {
2307+
// Sleep for 2 seconds to mock that the request is taking some time
2308+
time.Sleep(2 * time.Second)
2309+
}).Times(1).Return(startCtx, startCancel),
2310+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2311+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2312+
// Ensuring that the start request is running here by also passing in the expected parameters for the first CommandContext call
2313+
exec.EXPECT().CommandContext(gomock.Any(), firstStartExecCmd[0], firstStartExecCmd[1:]...).Times(1).Return(cmdExec),
2314+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2315+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2316+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2317+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2318+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2319+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2320+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2321+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2322+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2323+
2324+
// Exec mocks for stop packet loss request
2325+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel),
2326+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2327+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLossFaultExistsCommandOutput), nil),
2328+
// Ensuring that the stop request is running here by also passing in the expected parameters for the first CommandContext call
2329+
exec.EXPECT().CommandContext(gomock.Any(), firstStopExecCmd[0], firstStopExecCmd[1:]...).Times(1).Return(cmdExec),
2330+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2331+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2332+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2333+
)
2334+
},
2335+
},
2336+
}
2337+
2338+
for _, tc := range tcs {
2339+
t.Run(tc.name, func(t *testing.T) {
2340+
// Mocks
2341+
ctrl := gomock.NewController(t)
2342+
defer ctrl.Finish()
2343+
2344+
agentState := mock_state.NewMockAgentState(ctrl)
2345+
metricsFactory := mock_metrics.NewMockEntryFactory(ctrl)
2346+
2347+
router := mux.NewRouter()
2348+
mockExec := mock_execwrapper.NewMockExec(ctrl)
2349+
handler := New(agentState, metricsFactory, mockExec)
2350+
networkConfigClient := netconfig.NewNetworkConfigClient()
2351+
2352+
var startHandleMethod, stopHandleMethod func(http.ResponseWriter, *http.Request)
2353+
var firstStartExecCmd, firstStopExecCmd []string
2354+
nsenterPrefix := fmt.Sprintf(nsenterCommandString, path)
2355+
switch tc.faultType {
2356+
case types.BlackHolePortFaultType:
2357+
chain := fmt.Sprintf("%s-%s-%d", trafficType, protocol, port)
2358+
2359+
newChainCmdString := nsenterPrefix + fmt.Sprintf(iptablesNewChainCmd, requestTimeoutSeconds, chain)
2360+
firstStartExecCmd = strings.Split(newChainCmdString, " ")
2361+
2362+
clearChainCmdString := nsenterPrefix + fmt.Sprintf(iptablesClearChainCmd, requestTimeoutSeconds, chain)
2363+
firstStopExecCmd = strings.Split(clearChainCmdString, " ")
2364+
2365+
startHandleMethod = handler.StartNetworkBlackholePort()
2366+
stopHandleMethod = handler.StopNetworkBlackHolePort()
2367+
case types.LatencyFaultType:
2368+
tcAddQdiscRootCommandComposed := nsenterPrefix + fmt.Sprintf(tcAddQdiscRootCommandString, deviceName)
2369+
firstStartExecCmd = strings.Split(tcAddQdiscRootCommandComposed, " ")
2370+
2371+
tcDeleteQdiscParentCommandComposed := nsenterPrefix + fmt.Sprintf(tcDeleteQdiscParentCommandString, deviceName)
2372+
firstStopExecCmd = strings.Split(tcDeleteQdiscParentCommandComposed, " ")
2373+
2374+
startHandleMethod = handler.StartNetworkLatency()
2375+
stopHandleMethod = handler.StopNetworkLatency()
2376+
case types.PacketLossFaultType:
2377+
tcAddQdiscRootCommandComposed := nsenterPrefix + fmt.Sprintf(tcAddQdiscRootCommandString, deviceName)
2378+
firstStartExecCmd = strings.Split(tcAddQdiscRootCommandComposed, " ")
2379+
2380+
tcDeleteQdiscParentCommandComposed := nsenterPrefix + fmt.Sprintf(tcDeleteQdiscParentCommandString, deviceName)
2381+
firstStopExecCmd = strings.Split(tcDeleteQdiscParentCommandComposed, " ")
2382+
2383+
startHandleMethod = handler.StartNetworkPacketLoss()
2384+
stopHandleMethod = handler.StopNetworkPacketLoss()
2385+
default:
2386+
t.Error("Unrecognized network fault type")
2387+
}
2388+
2389+
tc.setAgentStateExpectations(agentState, networkConfigClient)
2390+
tc.setExecExpectations(mockExec, ctrl, convertToInterfaceList(firstStartExecCmd), convertToInterfaceList(firstStopExecCmd))
2391+
2392+
router.HandleFunc(
2393+
NetworkFaultPath(tc.faultType, types.StartNetworkFaultPostfix),
2394+
startHandleMethod,
2395+
).Methods(http.MethodPost)
2396+
2397+
router.HandleFunc(
2398+
NetworkFaultPath(tc.faultType, types.StopNetworkFaultPostfix),
2399+
stopHandleMethod,
2400+
).Methods(http.MethodPost)
2401+
2402+
var requestBody io.Reader
2403+
reqBodyBytes, err := json.Marshal(tc.requestBody)
2404+
require.NoError(t, err)
2405+
requestBody = bytes.NewReader(reqBodyBytes)
2406+
startReq, err := http.NewRequest(http.MethodPost, fmt.Sprintf(startEndpoint, endpointId, tc.faultType), requestBody)
2407+
require.NoError(t, err)
2408+
2409+
ch1 := make(chan struct {
2410+
int
2411+
error
2412+
})
2413+
2414+
reqBodyBytes, err = json.Marshal(tc.requestBody)
2415+
require.NoError(t, err)
2416+
requestBody = bytes.NewReader(reqBodyBytes)
2417+
stopReq, err := http.NewRequest(http.MethodPost, fmt.Sprintf(stopEndpoint, endpointId, tc.faultType), requestBody)
2418+
require.NoError(t, err)
2419+
2420+
ch2 := make(chan struct {
2421+
int
2422+
error
2423+
})
2424+
2425+
// Make an asynchronous Start request first
2426+
go makeAsyncRequest(router, startReq, ch1)
2427+
2428+
// Waiting a bit before sending the stop request
2429+
time.Sleep(1 * time.Second)
2430+
2431+
// Make an asynchronous Stop request second
2432+
go makeAsyncRequest(router, stopReq, ch2)
2433+
2434+
// Waiting to get the status code of the start request
2435+
resp1 := <-ch1
2436+
require.NoError(t, resp1.error)
2437+
assert.Equal(t, http.StatusOK, resp1.int)
2438+
2439+
// Waiting to get the status code of the stop request
2440+
resp2 := <-ch2
2441+
require.NoError(t, resp2.error)
2442+
assert.Equal(t, http.StatusOK, resp2.int)
2443+
})
2444+
}
2445+
}
2446+
2447+
// Helper function for making asynchronous mock HTTP requests
2448+
func makeAsyncRequest(router *mux.Router, req *http.Request, ch chan<- struct {
2449+
int
2450+
error
2451+
}) {
2452+
defer close(ch)
2453+
2454+
// Makes a mock HTTP request
2455+
recorder := httptest.NewRecorder()
2456+
router.ServeHTTP(recorder, req)
2457+
2458+
var actualResponseBody types.NetworkFaultInjectionResponse
2459+
err := json.Unmarshal(recorder.Body.Bytes(), &actualResponseBody)
2460+
if err != nil {
2461+
ch <- struct {
2462+
int
2463+
error
2464+
}{-1, err}
2465+
} else {
2466+
ch <- struct {
2467+
int
2468+
error
2469+
}{recorder.Code, nil}
2470+
}
2471+
}
2472+
2473+
func convertToInterfaceList(strings []string) []interface{} {
2474+
interfaces := make([]interface{}, len(strings))
2475+
for i, s := range strings {
2476+
interfaces[i] = s
2477+
}
2478+
return interfaces
2479+
}

0 commit comments

Comments
 (0)