Skip to content

Commit a1ffc50

Browse files
jpayne3506jpayne3506
authored
ci: Add Windows CNIv2 to CNI Load Test Pipeline (Azure#2279)
* add: env var to skip tests from make test-load * add: windows nodepool to existing make calls * add: Windows CNIv2 * refactor: change cniv2 file name * style: pipeline yaml layout * fix: label selector * ci: increase timeout for restartNode job * ci: ensure kubeproxy is running on windows * chore: clean up comments * ci: capture windows node count
1 parent b22ac38 commit a1ffc50

File tree

17 files changed

+383
-76
lines changed

17 files changed

+383
-76
lines changed

‎.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ stages:
4646
name: "$(BUILD_POOL_NAME_DEFAULT)"
4747
dependsOn:
4848
- create_${{ parameters.name }}
49+
- publish
4950
- setup
5051
displayName: "Cilium Test - ${{ parameters.name }}"
5152
jobs:

‎.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ parameters:
88

99
jobs:
1010
- job: CNI_${{ parameters.os }}
11+
condition: and( not(canceled()), not(failed()) )
1112
displayName: CNI k8s E2E ${{ parameters.os }}
1213
dependsOn: ${{ parameters.dependsOn }}
1314
pool:

‎.pipelines/cni/load-test-templates/create-cluster-template.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ parameters:
55
vmSize: ""
66
vmSizeWin: ""
77
osSKU: Ubuntu
8+
os: linux
9+
nodeCountWin: 2
810

911
steps:
1012
- task: AzureCLI@1
@@ -18,10 +20,10 @@ steps:
1820
make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }}
1921
make -C ./hack/aks ${{ parameters.clusterType }} \
2022
AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) \
21-
CLUSTER=${{ parameters.clusterName }} NODE_COUNT=${{ parameters.nodeCount }} \
23+
CLUSTER=${{ parameters.clusterName }} NODE_COUNT=${{ parameters.nodeCount }} NODE_COUNT_WIN=${{ parameters.nodeCountWin }} \
2224
VM_SIZE=${{ parameters.vmSize }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} \
2325
WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} \
24-
OS_SKU=${{parameters.osSKU}}
26+
OS_SKU=${{parameters.osSKU}} OS=${{parameters.os}}
2527
echo "Cluster successfully created"
2628
2729
name: "CreateAksCluster"

‎.pipelines/cni/load-test-templates/restart-cns-template.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,17 @@ steps:
2020
echo "Ensure there are pods scheduled on each node"
2121
cd test/integration/load
2222
scale=$(( ${{ parameters.scaleup }} * ${{ parameters.nodeCount }} ))
23-
REPLICAS=$scale go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load
23+
REPLICAS=$scale OS_TYPE=${{ parameters.os }} go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load
2424
cd ../../../
2525
2626
echo "Validate pod IP assignment before CNS restart"
27-
make test-validate-state CNI_TYPE=${{ parameters.cni }}
27+
make test-validate-state OS_TYPE=${{ parameters.os }} CNI_TYPE=${{ parameters.cni }}
2828
echo "restart CNS"
2929
kubectl rollout restart ds azure-cns -n kube-system
3030
kubectl rollout status ds azure-cns -n kube-system
3131
kubectl get pod -owide -A
3232
echo "Validate pod IP assignment after CNS restart"
33-
make test-validate-state CNI_TYPE=${{ parameters.cni }}
33+
make test-validate-state OS_TYPE=${{ parameters.os }} CNI_TYPE=${{ parameters.cni }}
3434
name: "restartCNS"
3535
displayName: "Restart CNS and Validate pods"
3636
retryCountOnTaskFailure: 3

‎.pipelines/cni/load-test-templates/restart-node-template.yaml‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ steps:
1212
scriptType: "bash"
1313
addSpnToEnvironment: true
1414
inlineScript: |
15-
echo "Scale up the pods and immediated restart the nodes"
1615
clusterName=${{ parameters.clusterName }}
1716
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
1817
make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
@@ -21,12 +20,13 @@ steps:
2120
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(LOCATION) --query "[].name" -o tsv); do
2221
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(LOCATION) VMSS_NAME=$val
2322
done
23+
kubectl get pods -n kube-system -owide
2424
2525
if ! [ ${{ parameters.cni }} = 'cniv1' ]; then
2626
echo "Ensure CNS daemonsets are ready and available"
2727
labels="app=azure-cns app=azure-cns-win"
2828
for label in $labels; do
29-
kubectl get pods -n kube-system -l $label
29+
kubectl get pods -n kube-system -l k8s-$label -owide
3030
kubectl rollout status ds -n kube-system -l $label
3131
done
3232
fi
@@ -36,6 +36,20 @@ steps:
3636
name: "RestartNodes"
3737
displayName: "Restart Nodes"
3838

39+
- ${{ if contains(parameters.os, 'windows') }}:
40+
- script: |
41+
kubectl apply -f test/integration/manifests/load/privileged-daemonset-windows.yaml
42+
kubectl rollout status -n kube-system ds privileged-daemonset
43+
44+
kubectl get pod -n kube-system -l app=privileged-daemonset,os=windows -owide
45+
pods=`kubectl get pod -n kube-system -l app=privileged-daemonset,os=windows --no-headers | awk '{print $1}'`
46+
for pod in $pods; do
47+
kubectl exec -i -n kube-system $pod -- powershell "Restart-service kubeproxy"
48+
done
49+
name: kubeproxy
50+
displayName: Restart Kubeproxy on Windows nodes
51+
retryCountOnTaskFailure: 3
52+
3953
- template: ../../templates/log-template.yaml
4054
parameters:
4155
clusterName: ${{ parameters.clusterName }}

‎.pipelines/cni/pipeline.yaml‎

Lines changed: 103 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ stages:
142142
platforms: linux/amd64 linux/arm64 windows/amd64
143143
cni:
144144
name: cni
145+
os_versions: ltsc2022
145146
platforms: linux/amd64 linux/arm64 windows/amd64
146147
npm:
147148
name: npm
@@ -154,65 +155,53 @@ stages:
154155
os_versions: $(os_versions)
155156
platforms: $(platforms)
156157

157-
- template: cilium/cilium-overlay-load-test-template.yaml
158-
parameters:
159-
name: cilium_overlay
160-
clusterType: overlay-byocni-nokubeproxy-up
161-
clusterName: "cilium-over"
162-
nodeCount: ${NODE_COUNT_CILIUM}
163-
vmSize: ${VM_SIZE_CILIUM}
164-
165-
- template: cilium/cilium-overlay-load-test-template.yaml
166-
parameters:
167-
name: cilium_overlay_hubble
168-
clusterType: overlay-byocni-nokubeproxy-up
169-
clusterName: "cil-over-hub"
170-
hubbleEnabled: true
171-
nodeCount: ${NODE_COUNT_CILIUM}
172-
vmSize: ${VM_SIZE_CILIUM}
173-
174-
- template: cilium/cilium-overlay-load-test-template.yaml
158+
## Windows E2E
159+
- template: singletenancy/cniv1-template.yaml
175160
parameters:
176-
name: cilium_overlay_mariner
177-
clusterType: overlay-byocni-nokubeproxy-up
178-
clusterName: "cil-over-mar"
179-
nodeCount: ${NODE_COUNT_CILIUM}
180-
vmSize: ${VM_SIZE_CILIUM}
161+
name: win22_cniv1
162+
clusterType: windows-cniv1-up
163+
clusterName: "win22-cniv1"
164+
nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
165+
nodeCountWin: ${NODE_COUNT_WIN}
166+
vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
167+
vmSizeWin: ${VM_SIZE_WIN}
168+
os: windows
169+
os_version: 'ltsc2022'
181170
arch: amd64
182-
osSKU: AzureLinux
183-
184-
- template: cilium/cilium-overlay-load-test-template.yaml
185-
parameters:
186-
name: cilium_overlay_arm
187-
clusterType: overlay-byocni-nokubeproxy-up
188-
clusterName: "cil-over-arm"
189-
nodeCount: ${NODE_COUNT_CILIUM}
190-
vmSize: Standard_D8ps_v5
191-
arch: arm64
171+
scaleup: ${SCALEUP_WIN}
172+
iterations: ${ITERATIONS_WIN}
192173

193-
- template: cilium/cilium-overlay-load-test-template.yaml
174+
- template: singletenancy/cniv2-template.yaml
194175
parameters:
195-
name: cilium_overlay_rdma
196-
clusterType: overlay-byocni-nokubeproxy-up
197-
clusterName: "cil-over-rdma"
198-
nodeCount: 2
199-
vmSize: Standard_HC44-16rs
176+
name: windows_podsubnet
177+
clusterType: swift-byocni-up
178+
clusterName: win-podsub
179+
nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
180+
nodeCountWin: ${NODE_COUNT_WIN}
181+
vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
182+
vmSizeWin: ${VM_SIZE_WIN}
200183
arch: amd64
184+
os: windows
185+
os_version: 'ltsc2022'
186+
scaleup: ${SCALEUP_WIN}
187+
iterations: ${ITERATIONS_WIN}
201188

202-
- template: singletenancy/cniv1-template.yaml
189+
- template: singletenancy/cniv2-template.yaml
203190
parameters:
204-
name: win22_cniv1
205-
clusterType: windows-cniv1-up
206-
clusterName: "win22-cniv1"
207-
nodeCount: ${NODE_COUNT_WIN}
191+
name: windows_overlay
192+
clusterType: overlay-byocni-up
193+
clusterName: win-over
194+
nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
195+
nodeCountWin: ${NODE_COUNT_WIN}
208196
vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
209197
vmSizeWin: ${VM_SIZE_WIN}
198+
arch: amd64
210199
os: windows
211200
os_version: 'ltsc2022'
212-
arch: amd64
213201
scaleup: ${SCALEUP_WIN}
214202
iterations: ${ITERATIONS_WIN}
215203

204+
## Linux E2E
216205
- template: singletenancy/cniv1-template.yaml
217206
parameters:
218207
name: linux_cniv1
@@ -225,25 +214,29 @@ stages:
225214
scaleup: ${SCALEUP_LINUX}
226215
iterations: ${ITERATIONS_LINUX}
227216

228-
- template: singletenancy/linux-cniv2-template.yaml
217+
- template: singletenancy/cniv2-template.yaml
229218
parameters:
230219
name: linux_podsubnet
231220
clusterType: swift-byocni-up
232221
clusterName: "linux-podsub"
233222
nodeCount: ${NODE_COUNT_LINUX}
234223
vmSize: ${VM_SIZE}
235224
arch: amd64
225+
scaleup: ${SCALEUP_LINUX}
226+
iterations: ${ITERATIONS_LINUX}
236227

237-
- template: singletenancy/linux-cniv2-template.yaml
228+
- template: singletenancy/cniv2-template.yaml
238229
parameters:
239230
name: linux_overlay
240231
clusterType: overlay-byocni-up
241232
clusterName: "linux-over"
242233
nodeCount: ${NODE_COUNT_LINUX}
243234
vmSize: ${VM_SIZE}
244235
arch: amd64
236+
scaleup: ${SCALEUP_LINUX}
237+
iterations: ${ITERATIONS_LINUX}
245238

246-
- template: singletenancy/linux-cniv2-template.yaml
239+
- template: singletenancy/cniv2-template.yaml
247240
parameters:
248241
name: mariner_linux_overlay
249242
clusterType: overlay-byocni-up
@@ -252,24 +245,76 @@ stages:
252245
vmSize: ${VM_SIZE}
253246
arch: amd64
254247
osSKU: AzureLinux
248+
scaleup: ${SCALEUP_LINUX}
249+
iterations: ${ITERATIONS_LINUX}
255250

256-
- template: singletenancy/linux-cniv2-template.yaml
251+
- template: singletenancy/cniv2-template.yaml
257252
parameters:
258253
name: arm_linux_overlay
259254
clusterType: overlay-byocni-up
260255
clusterName: "arm-over"
261256
nodeCount: ${NODE_COUNT_LINUX}
262257
vmSize: Standard_D8ps_v5
263258
arch: arm64
259+
scaleup: ${SCALEUP_LINUX}
260+
iterations: ${ITERATIONS_LINUX}
264261

265-
- template: singletenancy/linux-cniv2-template.yaml
262+
- template: singletenancy/cniv2-template.yaml
266263
parameters:
267264
name: rdma_linux_overlay
268265
clusterType: overlay-byocni-up
269266
clusterName: "rdma-over"
270267
nodeCount: 2
271268
vmSize: Standard_HC44-16rs
272269
arch: amd64
270+
scaleup: ${SCALEUP_LINUX}
271+
iterations: ${ITERATIONS_LINUX}
272+
273+
## Cilium E2E
274+
- template: cilium/cilium-overlay-load-test-template.yaml
275+
parameters:
276+
name: cilium_overlay
277+
clusterType: overlay-byocni-nokubeproxy-up
278+
clusterName: "cilium-over"
279+
nodeCount: ${NODE_COUNT_CILIUM}
280+
vmSize: ${VM_SIZE_CILIUM}
281+
282+
- template: cilium/cilium-overlay-load-test-template.yaml
283+
parameters:
284+
name: cilium_overlay_hubble
285+
clusterType: overlay-byocni-nokubeproxy-up
286+
clusterName: "cil-over-hub"
287+
hubbleEnabled: true
288+
nodeCount: ${NODE_COUNT_CILIUM}
289+
vmSize: ${VM_SIZE_CILIUM}
290+
291+
- template: cilium/cilium-overlay-load-test-template.yaml
292+
parameters:
293+
name: cilium_overlay_mariner
294+
clusterType: overlay-byocni-nokubeproxy-up
295+
clusterName: "cil-over-mar"
296+
nodeCount: ${NODE_COUNT_CILIUM}
297+
vmSize: ${VM_SIZE_CILIUM}
298+
arch: amd64
299+
osSKU: AzureLinux
300+
301+
- template: cilium/cilium-overlay-load-test-template.yaml
302+
parameters:
303+
name: cilium_overlay_arm
304+
clusterType: overlay-byocni-nokubeproxy-up
305+
clusterName: "cil-over-arm"
306+
nodeCount: ${NODE_COUNT_CILIUM}
307+
vmSize: Standard_D8ps_v5
308+
arch: arm64
309+
310+
- template: cilium/cilium-overlay-load-test-template.yaml
311+
parameters:
312+
name: cilium_overlay_rdma
313+
clusterType: overlay-byocni-nokubeproxy-up
314+
clusterName: "cil-over-rdma"
315+
nodeCount: 2
316+
vmSize: Standard_HC44-16rs
317+
arch: amd64
273318

274319
- stage: delete_resources
275320
displayName: "Delete Resources"
@@ -289,6 +334,8 @@ stages:
289334
- mariner_linux_overlay
290335
- arm_linux_overlay
291336
- rdma_linux_overlay
337+
- windows_podsubnet_HNS
338+
- windows_overlay_HNS
292339
- setup
293340
variables:
294341
commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
@@ -335,6 +382,12 @@ stages:
335382
rdma_linux_overlay:
336383
name: rdma_linux_overlay
337384
clusterName: "rdma-over"
385+
win-cniv2-podsubnet:
386+
name: windows_podsubnet
387+
clusterName: win-podsub
388+
win-cniv2-overlay:
389+
name: windows_overlay
390+
clusterName: win-over
338391
steps:
339392
- task: AzureCLI@1
340393
inputs:

‎.pipelines/cni/singletenancy/cniv1-template.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ parameters:
44
clusterType: ""
55
clusterName: ""
66
nodeCount: ""
7+
nodeCountWin: ""
78
vmSize: ""
89
vmSizeWin: ""
910
os: ""
@@ -74,10 +75,12 @@ stages:
7475
clusterType: ${{ parameters.clusterType }}
7576
clusterName: ${{ parameters.clusterName }}-$(commitID)
7677
nodeCount: ${{ parameters.nodeCount }}
78+
nodeCountWin: ${{ parameters.nodeCountWin }}
7779
vmSize: ${{ parameters.vmSize }}
7880
vmSizeWin: ${{ parameters.vmSizeWin }}
7981
region: $(LOCATION)
8082
osSKU: ${{ parameters.osSKU }}
83+
os: ${{ parameters.os }}
8184

8285
# Conditions for below E2E test scenarios confirm that:
8386
# Pipeline has not been canceled and that the previous job has reports anything other than failure(Succeeded, SuccededWithIssues, Skipped). Previous job is declared by dependsOn:
@@ -131,8 +134,8 @@ stages:
131134
parameters:
132135
clusterName: ${{ parameters.clusterName }}-$(commitID)
133136
os: ${{ parameters.os }}
134-
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
135137
os_version: ${{ parameters.os_version }}
138+
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
136139
tag: $(npmVersion)
137140
dependsOn: update_cni
138141
- job: deploy_pods

0 commit comments

Comments
 (0)