Skip to content

Commit 0032697

Browse files
emosbaughemosbaugh
authored
Ensure k8s version matches the target version after upgrades (#894)
1 parent 9ed26c2 commit 0032697

File tree

9 files changed

+120
-21
lines changed

9 files changed

+120
-21
lines changed

‎.github/actions/e2e/action.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ inputs:
3131
dr-aws-secret-access-key:
3232
description: 'Disaster Recovery AWS Secret Access Key'
3333
required: true
34+
k0s-version:
35+
description: 'k0s version to expect after upgrades in e2e tests'
36+
required: true
3437

3538
runs:
3639
using: composite
@@ -104,6 +107,7 @@ runs:
104107
export DR_AWS_S3_PREFIX_AIRGAP=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
105108
export DR_AWS_ACCESS_KEY_ID=${{ inputs.dr-aws-access-key-id }}
106109
export DR_AWS_SECRET_ACCESS_KEY=${{ inputs.dr-aws-secret-access-key }}
110+
export K0S_VERSION=${{ inputs.k0s-version }}
107111
make e2e-test TEST_NAME=${{ inputs.test-name }}
108112
- name: Upload Host Support Bundle
109113
uses: actions/upload-artifact@v4

‎.github/workflows/ci.yaml

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,10 @@ jobs:
176176
output/bin/embedded-cluster-original
177177
output/bin/embedded-cluster-previous-k0s
178178
output/bin/embedded-cluster-release-builder
179+
- name: Export K0s Version
180+
run: |
181+
make print-K0S_VERSION
182+
echo "k0s_version=$(make print-K0S_VERSION)" >> "$GITHUB_OUTPUT"
179183
180184
check-images:
181185
name: Check Images
@@ -248,6 +252,12 @@ jobs:
248252
# re-promote a release containing an old version of embedded-cluster to test upgrades
249253
replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "appver-${SHORT_SHA}-pre-minio-removal"
250254
255+
# install the previous k0s version to ensure an upgrade occurs
256+
sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
257+
replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}-previous-k0s"
258+
# return the cluster config to the current version
259+
sed -i "s/${SHORT_SHA}-previous-k0s/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
260+
251261
replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}"
252262
replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}-noop"
253263
replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "appver-${SHORT_SHA}-upgrade"
@@ -260,12 +270,14 @@ jobs:
260270
run: |
261271
export SHORT_SHA=dev-${GITHUB_SHA::7}
262272
echo "${SHORT_SHA}"
263-
replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
264273
265274
# airgap tests install the previous k0s version to ensure an upgrade occurs
266275
sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
267-
268276
replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}-previous-k0s"
277+
# return the cluster config to the current version
278+
sed -i "s/${SHORT_SHA}-previous-k0s/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
279+
280+
replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
269281
replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI-airgap --version "appver-${SHORT_SHA}-upgrade"
270282
271283
- name: Create download link message text
@@ -316,6 +328,7 @@ jobs:
316328
- TestCommandsRequireSudo
317329
- TestInstallWithoutEmbed
318330
- TestInstallFromReplicatedApp
331+
- TestUpgradeFromReplicatedApp
319332
- TestResetAndReinstall
320333
- TestResetAndReinstallAirgap
321334
- TestCollectSupportBundle
@@ -367,6 +380,7 @@ jobs:
367380
license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}
368381
dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
369382
dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
383+
k0s-version: ${{ steps.build.outputs.k0s_version }}
370384

371385
# this job will validate that all the tests passed
372386
# it is used for the github branch protection rule

‎e2e/install_test.go

Lines changed: 68 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func TestSingleNodeInstallation(t *testing.T) {
4848
}
4949

5050
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
51-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
51+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
5252
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
5353
t.Fatalf("fail to check postupgrade state: %v", err)
5454
}
@@ -104,7 +104,7 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) {
104104
}
105105

106106
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
107-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
107+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
108108
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
109109
t.Fatalf("fail to check postupgrade state: %v", err)
110110
}
@@ -164,7 +164,7 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {
164164
}
165165

166166
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
167-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
167+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
168168
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
169169
t.Fatalf("fail to check postupgrade state: %v", err)
170170
}
@@ -224,7 +224,7 @@ func TestSingleNodeInstallationDebian11(t *testing.T) {
224224
}
225225

226226
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
227-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
227+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
228228
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
229229
t.Fatalf("fail to check postupgrade state: %v", err)
230230
}
@@ -280,7 +280,7 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) {
280280
}
281281

282282
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
283-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
283+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
284284
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
285285
t.Fatalf("fail to check postupgrade state: %v", err)
286286
}
@@ -499,7 +499,61 @@ func TestInstallFromReplicatedApp(t *testing.T) {
499499
}
500500

501501
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
502-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
502+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
503+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
504+
t.Fatalf("fail to check postupgrade state: %v", err)
505+
}
506+
507+
t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
508+
}
509+
510+
func TestUpgradeFromReplicatedApp(t *testing.T) {
511+
t.Parallel()
512+
513+
RequireEnvVars(t, []string{"SHORT_SHA"})
514+
515+
tc := cluster.NewTestCluster(&cluster.Input{
516+
T: t,
517+
Nodes: 1,
518+
Image: "debian/12",
519+
})
520+
defer cleanupCluster(t, tc)
521+
522+
t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339))
523+
line := []string{"vandoor-prepare.sh", fmt.Sprintf("%s-previous-k0s", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"}
524+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
525+
t.Fatalf("fail to download embedded-cluster on node 0 %s: %v", tc.Nodes[0], err)
526+
}
527+
528+
t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
529+
line = []string{"single-node-install.sh", "ui"}
530+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
531+
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
532+
}
533+
534+
if err := setupPlaywright(t, tc); err != nil {
535+
t.Fatalf("fail to setup playwright: %v", err)
536+
}
537+
if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
538+
t.Fatalf("fail to run playwright test deploy-app: %v", err)
539+
}
540+
541+
t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
542+
line = []string{"check-installation-state.sh", fmt.Sprintf("%s-previous-k0s", os.Getenv("SHORT_SHA"))}
543+
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
544+
t.Fatalf("fail to check installation state: %v", err)
545+
}
546+
547+
appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
548+
testArgs := []string{appUpgradeVersion}
549+
550+
t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
551+
if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
552+
t.Fatalf("fail to run playwright test deploy-app: %v", err)
553+
}
554+
555+
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
556+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
503557
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
504558
t.Fatalf("fail to check postupgrade state: %v", err)
505559
}
@@ -653,7 +707,7 @@ func TestOldVersionUpgrade(t *testing.T) {
653707
}
654708

655709
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
656-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
710+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
657711
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
658712
t.Fatalf("fail to check postupgrade state: %v", err)
659713
}
@@ -763,7 +817,7 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) {
763817
}
764818

765819
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
766-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
820+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
767821
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
768822
t.Fatalf("fail to check postupgrade state: %v", err)
769823
}
@@ -875,7 +929,7 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
875929
}
876930

877931
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
878-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
932+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
879933
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
880934
t.Fatalf("fail to check postupgrade state: %v", err)
881935
}
@@ -1055,7 +1109,7 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
10551109
}
10561110

10571111
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
1058-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
1112+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
10591113
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
10601114
t.Fatalf("fail to check postupgrade state: %v", err)
10611115
}
@@ -1219,7 +1273,7 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) {
12191273
}
12201274

12211275
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
1222-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
1276+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
12231277
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
12241278
t.Fatalf("fail to check postupgrade state: %v", err)
12251279
}
@@ -1334,7 +1388,7 @@ func TestMultiNodeHAInstallation(t *testing.T) {
13341388
}
13351389

13361390
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
1337-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
1391+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
13381392
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
13391393
t.Fatalf("fail to check postupgrade state: %v", err)
13401394
}
@@ -1546,7 +1600,7 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
15461600
}
15471601

15481602
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
1549-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
1603+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
15501604
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
15511605
t.Fatalf("fail to check postupgrade state: %v", err)
15521606
}
@@ -1606,7 +1660,7 @@ func TestInstallSnapshotFromReplicatedApp(t *testing.T) {
16061660
}
16071661

16081662
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
1609-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
1663+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
16101664
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
16111665
t.Fatalf("fail to check postupgrade state: %v", err)
16121666
}

‎e2e/restore_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
390390
}
391391

392392
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
393-
line = []string{"check-postupgrade-state.sh"}
393+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
394394
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
395395
t.Fatalf("fail to check postupgrade state: %v", err)
396396
}
@@ -591,7 +591,7 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) {
591591
}
592592

593593
t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
594-
line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
594+
line = []string{"check-postupgrade-state.sh", k8sVersion()}
595595
if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
596596
t.Fatalf("fail to check postupgrade state: %v", err)
597597
}

‎e2e/scripts/check-postupgrade-state.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ function check_nginx_version {
1212
}
1313

1414
main() {
15+
local k8s_version="$1"
16+
1517
echo "ensure that installation is installed"
1618
wait_for_installation
1719

@@ -102,6 +104,12 @@ main() {
102104
echo "no charts had an order of '110'"
103105
exit 1
104106
fi
107+
108+
echo "ensure that all nodes are running k8s $k8s_version"
109+
if ! ensure_nodes_match_kube_version "$k8s_version"; then
110+
echo "not all nodes are running k8s $k8s_version"
111+
exit 1
112+
fi
105113
}
106114

107115
export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"

‎e2e/scripts/common.sh

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,16 @@ ensure_node_config() {
203203
fi
204204
}
205205

206+
ensure_nodes_match_kube_version() {
207+
local version="$1"
208+
if kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}' | grep -v "$version"; then
209+
echo "Node kubelet version does not match expected version $version"
210+
kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}'
211+
kubectl get nodes
212+
return 1
213+
fi
214+
}
215+
206216
check_pod_install_order() {
207217
local ingress_install_time=
208218
ingress_install_time=$(kubectl get pods --no-headers=true -n ingress-nginx -o jsonpath='{.items[*].metadata.creationTimestamp}' | sort | head -n 1)

‎e2e/utils.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,3 +159,12 @@ func injectString(original, injection, after string) string {
159159
// Construct the new string by adding the injection between the parts
160160
return parts[0] + after + " " + injection + parts[1]
161161
}
162+
163+
func k8sVersion() string {
164+
// split the version string (like 'v1.29.6+k0s.0') into the k8s version and the k0s revision
165+
verParts := strings.Split(os.Getenv("K0S_VERSION"), "+")
166+
if len(verParts) < 2 {
167+
panic(fmt.Sprintf("failed to parse k8s version %q", os.Getenv("K0S_VERSION")))
168+
}
169+
return verParts[0]
170+
}

‎pkg/addons/embeddedclusteroperator/static/metadata.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
# $ make buildtools
66
# $ output/bin/buildtools update addon <addon name>
77
#
8-
version: 0.42.0
8+
version: 0.42.1
99
location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/embedded-cluster-operator
1010
images:
11-
replicated/embedded-cluster-operator-image: 0.42.0@sha256:a0eab9ea8bb058f392c75d34b6fa88ae66b3f5fc6ab8953686379cacb13bb11f
11+
replicated/embedded-cluster-operator-image: 0.42.1@sha256:bc293473dd59a03f82e1328bed58c6ba24b8fbdaacc4559b8c338c933239a042
1212
utils: latest@sha256:88c0e2ba4daa9682ca85de32e11aad545d338cdaf91edf6ab52eb0c2d6b825a3

‎pkg/addons/embeddedclusteroperator/static/values.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@ global:
1717
replicated.com/disaster-recovery-chart: embedded-cluster-operator
1818
image:
1919
repository: proxy.replicated.com/anonymous/replicated/embedded-cluster-operator-image
20-
tag: 0.42.0@sha256:a0eab9ea8bb058f392c75d34b6fa88ae66b3f5fc6ab8953686379cacb13bb11f
20+
tag: 0.42.1@sha256:bc293473dd59a03f82e1328bed58c6ba24b8fbdaacc4559b8c338c933239a042
2121
utilsImage: 'proxy.replicated.com/anonymous/replicated/ec-utils@sha256:88c0e2ba4daa9682ca85de32e11aad545d338cdaf91edf6ab52eb0c2d6b825a3'

0 commit comments

Comments
 (0)