1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package networkchaos
17
18 import (
19 "context"
20 "net/http"
21 "time"
22
23 . "github.com/onsi/ginkgo/v2"
24 corev1 "k8s.io/api/core/v1"
25 "k8s.io/apimachinery/pkg/util/wait"
26 "k8s.io/kubernetes/test/e2e/framework"
27 "k8s.io/utils/pointer"
28 "sigs.k8s.io/controller-runtime/pkg/client"
29
30 "github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
31 "github.com/chaos-mesh/chaos-mesh/e2e-test/e2e/util"
32 )
33
34 func TestcaseNetworkDelay(
35 ns string,
36 cli client.Client,
37 networkPeers []*corev1.Pod,
38 ports []uint16,
39 c http.Client,
40 ) {
41 ctx, cancel := context.WithCancel(context.Background())
42 defer cancel()
43
44 By("prepare experiment playground")
45 for index := range networkPeers {
46 err := util.WaitE2EHelperReady(c, ports[index])
47
48 framework.ExpectNoError(err, "wait e2e helper ready error")
49 }
50
51 result := probeNetworkCondition(c, networkPeers, ports, false)
52 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
53 framework.ExpectEqual(len(result[networkConditionSlow]), 0)
54
55 var (
56 testDelayTcParam = v1alpha1.TcParameter{
57 Delay: &v1alpha1.DelaySpec{
58 Latency: "200ms",
59 Correlation: "25",
60 Jitter: "0ms",
61 },
62 }
63 testDelayTcParamEvenMoreComplicate = v1alpha1.TcParameter{
64 Delay: &v1alpha1.DelaySpec{
65 Latency: "200ms",
66 Correlation: "25",
67 Jitter: "0ms",
68 },
69 Loss: &v1alpha1.LossSpec{
70 Loss: "25",
71 Correlation: "25",
72 },
73 Duplicate: &v1alpha1.DuplicateSpec{
74 Duplicate: "25",
75 Correlation: "25",
76 },
77 Corrupt: &v1alpha1.CorruptSpec{
78 Corrupt: "25",
79 Correlation: "25",
80 },
81 }
82 testDelayDuration = pointer.String("9m")
83 )
84
85 By("normal delay chaos")
86 networkDelay := makeNetworkDelayChaos(
87 ns, "network-chaos-1",
88 map[string]string{"app": "network-peer-0"},
89 nil,
90 v1alpha1.OneMode,
91 v1alpha1.OneMode,
92 v1alpha1.To,
93 testDelayTcParam,
94 testDelayDuration,
95 )
96 By("Injecting delay for 0")
97 err := cli.Create(ctx, networkDelay.DeepCopy())
98 framework.ExpectNoError(err, "create network chaos error")
99
100 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
101 result = probeNetworkCondition(c, networkPeers, ports, false)
102 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 3 {
103 return false, nil
104 }
105 return true, nil
106 })
107
108 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
109 framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 1}, {0, 2}, {0, 3}})
110
111 By("recover")
112 err = cli.Delete(ctx, networkDelay.DeepCopy())
113 framework.ExpectNoError(err, "delete network chaos error")
114
115 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
116 result = probeNetworkCondition(c, networkPeers, ports, false)
117 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
118 return false, nil
119 }
120 return true, nil
121 })
122
123 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
124 framework.ExpectEqual(len(result[networkConditionSlow]), 0)
125
126 networkDelayWithTarget := makeNetworkDelayChaos(
127 ns, "network-chaos-1",
128 map[string]string{"app": "network-peer-0"},
129 map[string]string{"app": "network-peer-1"},
130 v1alpha1.OneMode,
131 v1alpha1.OneMode,
132 v1alpha1.To,
133 testDelayTcParam,
134 testDelayDuration,
135 )
136
137 By("Injecting delay for 0 -> 1")
138 err = cli.Create(ctx, networkDelayWithTarget.DeepCopy())
139 framework.ExpectNoError(err, "create network chaos error")
140
141 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
142 result = probeNetworkCondition(c, networkPeers, ports, false)
143 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 1 {
144 return false, nil
145 }
146 return true, nil
147 })
148
149 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
150 framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 1}})
151
152 err = cli.Delete(ctx, networkDelayWithTarget.DeepCopy())
153 framework.ExpectNoError(err, "delete network chaos error")
154
155 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
156 result = probeNetworkCondition(c, networkPeers, ports, false)
157 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
158 return false, nil
159 }
160 return true, nil
161 })
162
163 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
164 framework.ExpectEqual(len(result[networkConditionSlow]), 0)
165
166 evenNetworkDelay := makeNetworkDelayChaos(
167 ns, "network-chaos-2",
168 map[string]string{"app": "network-peer-0"},
169 map[string]string{"partition": "0"},
170 v1alpha1.OneMode,
171 v1alpha1.AllMode,
172 v1alpha1.To,
173 testDelayTcParam,
174 testDelayDuration,
175 )
176 By("Injecting delay for 0 -> even partition")
177 err = cli.Create(ctx, evenNetworkDelay.DeepCopy())
178 framework.ExpectNoError(err, "create network chaos error")
179
180 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
181 result = probeNetworkCondition(c, networkPeers, ports, false)
182 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 1 {
183 return false, nil
184 }
185 return true, nil
186 })
187 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
188 framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 2}})
189
190 By("Injecting delay for 0 -> 1")
191 err = cli.Create(ctx, networkDelayWithTarget.DeepCopy())
192 framework.ExpectNoError(err, "create network chaos error")
193
194 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
195 result = probeNetworkCondition(c, networkPeers, ports, false)
196 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 2 {
197 return false, nil
198 }
199 return true, nil
200 })
201 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
202 framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 1}, {0, 2}})
203
204 err = cli.Delete(ctx, networkDelayWithTarget.DeepCopy())
205 framework.ExpectNoError(err, "delete network chaos error")
206
207 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
208 result = probeNetworkCondition(c, networkPeers, ports, false)
209 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 1 {
210 return false, nil
211 }
212 return true, nil
213 })
214 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
215 framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 2}})
216
217 err = cli.Delete(ctx, evenNetworkDelay.DeepCopy())
218 framework.ExpectNoError(err, "delete network chaos error")
219
220 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
221 result = probeNetworkCondition(c, networkPeers, ports, false)
222 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
223 return false, nil
224 }
225 return true, nil
226 })
227 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
228 framework.ExpectEqual(len(result[networkConditionSlow]), 0)
229
230 complicateNetem := makeNetworkDelayChaos(
231 ns, "network-chaos-3",
232 map[string]string{"app": "network-peer-0"},
233 nil,
234 v1alpha1.OneMode,
235 v1alpha1.OneMode,
236 v1alpha1.To,
237 testDelayTcParamEvenMoreComplicate,
238 testDelayDuration,
239 )
240 By("Injecting complicate chaos for 0")
241 err = cli.Create(ctx, complicateNetem.DeepCopy())
242 framework.ExpectNoError(err, "create network chaos error")
243 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
244 result = probeNetworkCondition(c, networkPeers, ports, false)
245 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 3 {
246 return false, nil
247 }
248 return true, nil
249 })
250 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
251 framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 1}, {0, 2}, {0, 3}})
252
253 By("recover")
254 err = cli.Delete(ctx, complicateNetem.DeepCopy())
255 framework.ExpectNoError(err, "delete network chaos error")
256
257 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
258 result = probeNetworkCondition(c, networkPeers, ports, false)
259 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
260 return false, nil
261 }
262 return true, nil
263 })
264 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
265 framework.ExpectEqual(len(result[networkConditionSlow]), 0)
266
267 bothDirectionNetem := makeNetworkDelayChaos(
268 ns, "network-chaos-4",
269 map[string]string{"app": "network-peer-0"},
270 map[string]string{"partition": "0"},
271 v1alpha1.OneMode,
272 v1alpha1.AllMode,
273 v1alpha1.Both,
274 testDelayTcParam,
275 testDelayDuration,
276 )
277 By("Injecting both direction chaos for 0")
278 err = cli.Create(ctx, bothDirectionNetem.DeepCopy())
279 framework.ExpectNoError(err, "create network chaos error")
280 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
281 result = probeNetworkCondition(c, networkPeers, ports, true)
282 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 2 {
283 return false, nil
284 }
285 return true, nil
286 })
287 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
288 framework.ExpectEqual(result[networkConditionSlow], [][]int{{0, 2}, {2, 0}})
289
290 By("recover")
291 err = cli.Delete(ctx, bothDirectionNetem.DeepCopy())
292 framework.ExpectNoError(err, "delete network chaos error")
293
294 wait.Poll(time.Second, 15*time.Second, func() (done bool, err error) {
295 result = probeNetworkCondition(c, networkPeers, ports, true)
296 if len(result[networkConditionBlocked]) != 0 || len(result[networkConditionSlow]) != 0 {
297 return false, nil
298 }
299 return true, nil
300 })
301 framework.ExpectEqual(len(result[networkConditionBlocked]), 0)
302 framework.ExpectEqual(len(result[networkConditionSlow]), 0)
303 }
304