Skip to content

Commit 300dbca

Browse files
sgalsalehsgalsaleh
authored
Fix flakiness in playwright regression tests (#5174)
1 parent 28e7e90 commit 300dbca

File tree

7 files changed

+59
-46
lines changed

7 files changed

+59
-46
lines changed

‎e2e/playwright/regression/@existing-online-install-admin/test.spec.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import {
2222
validateCurrentDeployLogs,
2323
validateConfigView,
2424
validateVersionHistoryRows,
25-
deployVersion,
25+
deployNewVersion,
2626
validateCurrentLicense,
2727
updateOnlineLicense,
2828
validateUpdatedLicense,
@@ -79,14 +79,14 @@ test('type=existing cluster, env=online, phase=new install, rbac=cluster admin',
7979
await validateCurrentDeployLogs(page, expect);
8080
await validateConfigView(page, expect);
8181
await validateVersionHistoryRows(page, expect, true);
82-
await deployVersion(page, expect, 0, 1, 'Config Change', false);
82+
await deployNewVersion(page, expect, 1, 'Config Change', false);
8383

8484
await validateCurrentLicense(page, expect, constants.CUSTOMER_NAME, constants.CHANNEL_NAME, constants.IS_AIRGAP_SUPPORTED, constants.IS_EC);
8585
const newIntEntitlement = await updateOnlineLicense(page, constants.CUSTOMER_ID, constants.CUSTOMER_NAME, constants.CHANNEL_ID, constants.IS_AIRGAP_SUPPORTED, constants.IS_EC);
8686
await validateUpdatedLicense(page, expect, newIntEntitlement);
8787

8888
await validateVersionDiff(page, expect, 2, 1);
89-
await deployVersion(page, expect, 0, 2, 'License Change', false);
89+
await deployNewVersion(page, expect, 2, 'License Change', false);
9090

9191
await validateSnapshotsAWSConfig(page, expect);
9292
await validateAutomaticFullSnapshots(page, expect);
@@ -102,7 +102,7 @@ test('type=existing cluster, env=online, phase=new install, rbac=cluster admin',
102102

103103
await validateViewFiles(page, expect, constants.CHANNEL_ID, constants.CHANNEL_NAME, constants.CUSTOMER_NAME, constants.LICENSE_ID, constants.IS_AIRGAPPED, registryInfo);
104104
await updateRegistrySettings(page, expect, registryInfo, false);
105-
await validateCheckForUpdates(page, expect, constants.CHANNEL_ID, constants.VENDOR_UPDATE_RELEASE_SEQUENCE, 4, false);
105+
await validateCheckForUpdates(page, expect, constants.CHANNEL_ID, constants.VENDOR_UPDATE_RELEASE_SEQUENCE, false);
106106
await validateDuplicateLicenseUpload(page, expect);
107107
await logout(page, expect);
108108
});

‎e2e/playwright/regression/@existing-online-install-minimal/test.spec.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import {
2222
validateCurrentDeployLogs,
2323
validateConfigView,
2424
validateVersionHistoryRows,
25-
deployVersion,
25+
deployNewVersion,
2626
validateCurrentLicense,
2727
updateOnlineLicense,
2828
validateUpdatedLicense,
@@ -75,14 +75,14 @@ test('type=existing cluster, env=online, phase=new install, rbac=minimal rbac',
7575
await validateCurrentDeployLogs(page, expect);
7676
await validateConfigView(page, expect);
7777
await validateVersionHistoryRows(page, expect, true);
78-
await deployVersion(page, expect, 0, 1, 'Config Change', true);
78+
await deployNewVersion(page, expect, 1, 'Config Change', true);
7979

8080
await validateCurrentLicense(page, expect, constants.CUSTOMER_NAME, constants.CHANNEL_NAME, constants.IS_AIRGAP_SUPPORTED, constants.IS_EC);
8181
const newIntEntitlement = await updateOnlineLicense(page, constants.CUSTOMER_ID, constants.CUSTOMER_NAME, constants.CHANNEL_ID, constants.IS_AIRGAP_SUPPORTED, constants.IS_EC);
8282
await validateUpdatedLicense(page, expect, newIntEntitlement);
8383

8484
await validateVersionDiff(page, expect, 2, 1);
85-
await deployVersion(page, expect, 0, 2, 'License Change', true);
85+
await deployNewVersion(page, expect, 2, 'License Change', true);
8686

8787
await validateSnapshotsAWSConfig(page, expect);
8888
await validateAutomaticFullSnapshots(page, expect);
@@ -98,7 +98,7 @@ test('type=existing cluster, env=online, phase=new install, rbac=minimal rbac',
9898

9999
await validateViewFiles(page, expect, constants.CHANNEL_ID, constants.CHANNEL_NAME, constants.CUSTOMER_NAME, constants.LICENSE_ID, constants.IS_AIRGAPPED, registryInfo);
100100
await updateRegistrySettings(page, expect, registryInfo, true);
101-
await validateCheckForUpdates(page, expect, constants.CHANNEL_ID, constants.VENDOR_UPDATE_RELEASE_SEQUENCE, 4, true);
101+
await validateCheckForUpdates(page, expect, constants.CHANNEL_ID, constants.VENDOR_UPDATE_RELEASE_SEQUENCE, true);
102102
await validateDuplicateLicenseUpload(page, expect);
103103
await logout(page, expect);
104104
});

‎e2e/playwright/regression/shared/registry-settings.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
import { Page, Expect, Locator } from '@playwright/test';
1+
import { Page, Expect } from '@playwright/test';
22

33
import { RegistryInfo } from './cli';
44
import { APP_SLUG } from './constants';
5-
import { deployVersion } from './version-history';
5+
import { deployNewVersion } from './version-history';
66
import { validateRegistryChangeKustomization } from './view-files';
77

88
export const updateRegistrySettings = async (page: Page, expect: Expect, registryInfo: RegistryInfo, isMinimalRBAC: boolean) => {
@@ -36,7 +36,7 @@ export const updateRegistrySettings = async (page: Page, expect: Expect, registr
3636
await expect(card.getByTestId("airgap-registry-settings-progress")).not.toBeVisible({ timeout: 240000 });
3737

3838
await page.reload();
39-
await deployVersion(page, expect, 0, 3, 'Registry Change', isMinimalRBAC);
39+
await deployNewVersion(page, expect, 3, 'Registry Change', isMinimalRBAC);
4040

4141
await validateRegistryChangeKustomization(page, expect, registryInfo);
4242
};

‎e2e/playwright/regression/shared/version-history.ts

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Page, Expect } from '@playwright/test';
1+
import { Page, Expect, Locator } from '@playwright/test';
22
import * as uuid from "uuid";
33

44
import { promoteRelease } from './api';
@@ -117,7 +117,7 @@ export const validateVersionHistoryRows = async (page: Page, expect: Expect, isO
117117
await expect(secondRow.getByRole('button', { name: 'Redeploy', exact: true })).toBeVisible();
118118
};
119119

120-
export const deployVersion = async (page: Page, expect: Expect, rowIndex: number, sequence: number, source: string, isMinimalRBAC: boolean, skipNavigation: boolean = false) => {
120+
export const deployNewVersion = async (page: Page, expect: Expect, sequence: number | undefined, source: string, isMinimalRBAC: boolean, skipNavigation: boolean = false) => {
121121
if (!skipNavigation) {
122122
await page.locator('.NavItem').getByText('Application', { exact: true }).click();
123123
await page.getByRole('link', { name: 'Version history', exact: true }).click();
@@ -126,10 +126,14 @@ export const deployVersion = async (page: Page, expect: Expect, rowIndex: number
126126
const allVersionsCard = page.getByTestId('all-versions-card');
127127
await expect(allVersionsCard).toBeVisible({ timeout: 15000 });
128128

129-
const versionRow = allVersionsCard.getByTestId(`version-history-row-${rowIndex}`);
129+
const versionRow = allVersionsCard.getByTestId('version-history-row-0');
130130
await expect(versionRow).toBeVisible();
131-
await expect(versionRow).toContainText(`Sequence ${sequence}`);
132131
await expect(versionRow).toContainText(source);
132+
if (sequence) {
133+
await expect(versionRow).toContainText(`Sequence ${sequence}`);
134+
} else {
135+
sequence = await sequenceFromVersionRow(versionRow);
136+
}
133137

134138
const preflightChecksLoader = versionRow.getByTestId('preflight-checks-loader');
135139
await expect(preflightChecksLoader).not.toBeVisible({ timeout: 180000 });
@@ -154,10 +158,10 @@ export const deployVersion = async (page: Page, expect: Expect, rowIndex: number
154158
await expect(versionRow).toContainText('Currently deployed version', { timeout: 45000 });
155159
await expect(versionRow.getByRole('button', { name: 'Redeploy', exact: true })).toBeVisible();
156160

157-
const previousVersionRow = allVersionsCard.getByTestId(`version-history-row-${rowIndex + 1}`);
158-
await expect(previousVersionRow).toContainText(`Sequence ${sequence - 1}`);
161+
const previousVersionRow = allVersionsCard.getByTestId('version-history-row-1');
159162
await expect(previousVersionRow).toContainText('Previously deployed');
160163
await expect(previousVersionRow.getByRole('button', { name: 'Rollback', exact: true })).toBeVisible();
164+
await expect(previousVersionRow).toContainText(`Sequence ${sequence - 1}`);
161165

162166
const currentVersionCard = page.getByTestId("current-version-card");
163167
await expect(currentVersionCard).toContainText(`Sequence ${sequence}`);
@@ -278,7 +282,7 @@ export const validateCurrentlyDeployedVersionInfo = async (page: Page, expect: E
278282
}
279283
};
280284

281-
export const validateCheckForUpdates = async (page: Page, expect: Expect, channelId: string, vendorReleaseSequence: number, expectedSequence: number, isMinimalRBAC: boolean) => {
285+
export const validateCheckForUpdates = async (page: Page, expect: Expect, channelId: string, vendorReleaseSequence: number, isMinimalRBAC: boolean) => {
282286
await page.getByRole('link', { name: 'Version history', exact: true }).click();
283287

284288
const newVersionLabel = `1.0.0+${uuid.v4()}`;
@@ -305,5 +309,21 @@ export const validateCheckForUpdates = async (page: Page, expect: Expect, channe
305309
await releaseNotesModal.getByRole("button", { name: "Close" }).click();
306310
await expect(releaseNotesModal).not.toBeVisible();
307311

308-
await deployVersion(page, expect, 0, expectedSequence, 'Upstream Update', isMinimalRBAC, true);
312+
// multiple instances of the test can run in parallel, each of which will promote a new release
313+
// making it difficult to predict what the new sequence will be.
314+
await deployNewVersion(page, expect, undefined, 'Upstream Update', isMinimalRBAC, true);
315+
};
316+
317+
export const sequenceFromVersionRow = async (versionRow: Locator): Promise<number> => {
318+
const versionSequence = versionRow.getByTestId("version-sequence");
319+
const versionSequenceText = await versionSequence.textContent();
320+
const versionSequenceNumber = versionSequenceText.match(/\d+/);
321+
if (!versionSequenceNumber || versionSequenceNumber.length !== 1) {
322+
throw new Error(`version sequence number not found in text "${versionSequenceText}"`);
323+
}
324+
const versionSequenceNumberInt = parseInt(versionSequenceNumber[0]);
325+
if (isNaN(versionSequenceNumberInt)) {
326+
throw new Error(`version sequence number is not a number "${versionSequenceNumber}"`);
327+
}
328+
return versionSequenceNumberInt;
309329
};

‎e2e/playwright/regression/shared/view-files.ts

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -117,10 +117,6 @@ export const validateViewFiles = async (
117117
};
118118

119119
export const validateRegistryChangeKustomization = async (page: Page, expect: Expect, registryInfo: RegistryInfo) => {
120-
// resize page so that the entire editor content is visible for text detection
121-
await page.setViewportSize({ width: 3840, height: 2160 });
122-
123-
await page.locator('.NavItem').getByText('Application', { exact: true }).click();
124120
await page.getByRole('link', { name: 'View files', exact: true }).click();
125121

126122
const viewFilesPage = page.getByTestId('view-files-page');
@@ -137,9 +133,6 @@ export const validateRegistryChangeKustomization = async (page: Page, expect: Ex
137133
const editor = viewFilesPage.getByTestId('file-editor');
138134
const kustomizationYAML = await getEditorText(editor);
139135
expect(kustomizationYAML).toContain(`newName: ${registryInfo.ip}/${APP_SLUG}/`);
140-
141-
// resize page back to default size
142-
await page.setViewportSize({ width: 1280, height: 720 });
143136
};
144137

145138
const selectFile = async (page: Page, fileTree: Locator, file: string) => {

‎e2e/playwright/tests/@min-kots-version-online/test.spec.ts

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ const validateOnlineUpdateRestrictive = async (page: Page, expect: Expect) => {
4747
await page.getByTestId("console-subnav").getByRole("link", { name: "Version history" }).click();
4848

4949
const availableUpdateCard = page.getByTestId("available-updates-card");
50-
let card = availableUpdateCard.getByTestId("version-history-row-0");
50+
const card = availableUpdateCard.getByTestId("version-history-row-0");
5151
await expect(card.getByTestId("version-label")).toContainText(constants.VENDOR_RESTRICTIVE_RELEASE_SEMVER);
5252
await expect(card.getByTestId("version-action-button")).toContainText("Download");
5353
await expect(card.getByTestId("version-status")).toContainText("Pending download");
@@ -57,32 +57,32 @@ const validateOnlineUpdateRestrictive = async (page: Page, expect: Expect) => {
5757
await expect(errorMessage).toContainText("Upgrade KOTS");
5858
await expect(errorMessage).toContainText(constants.RESTRICTIVE_MIN_KOTS_VERSION);
5959

60-
var allVersionsCard = page.getByTestId("all-versions-card");
61-
card = allVersionsCard.getByTestId("version-history-row-0");
62-
await expect(card.getByTestId("version-label")).toContainText(constants.VENDOR_RESTRICTIVE_RELEASE_SEMVER);
60+
const allVersionsCard = page.getByTestId("all-versions-card");
61+
const versionRow = allVersionsCard.getByTestId("version-history-row-0");
62+
await expect(versionRow.getByTestId("version-label")).toContainText(constants.VENDOR_RESTRICTIVE_RELEASE_SEMVER);
6363

6464
// Click the download button and validate that you cannot download it
65-
await card.getByTestId("version-action-button").click();
65+
await versionRow.getByTestId("version-action-button").click();
6666
await page.waitForTimeout(1 * 1000); // 1 second
67-
await expect(card.getByTestId("version-downloading-status")).not.toContainText("Downloading");
67+
await expect(versionRow.getByTestId("version-downloading-status")).not.toContainText("Downloading");
6868

69-
errorMessage = card.getByTestId("version-downloading-status");
69+
errorMessage = versionRow.getByTestId("version-downloading-status");
7070
await expect(errorMessage).toContainText("requires", { timeout: 5 * 1000 }); // 5 seconds
7171
await expect(errorMessage).toContainText("Upgrade KOTS");
7272
await expect(errorMessage).toContainText(constants.RESTRICTIVE_MIN_KOTS_VERSION);
7373

7474
// Click the diff button and validate that you cannot select this version to diff because it was
7575
// unable to download
7676
await page.getByTestId("select-releases-to-diff-button").click();
77-
await expect(card).not.toBeVisible();
77+
await expect(versionRow).not.toBeVisible();
7878

7979
// Click the cancel button and validate that you can see the version card again
8080
await page.getByTestId("cancel-diff-button").click();
81-
await expect(card).toBeVisible();
81+
await expect(versionRow).toBeVisible();
8282

8383
// A license sync may happen on installation and there will be a "License changed" version in the
8484
// list
85-
const versionSequence = card.getByTestId("version-sequence");
85+
const versionSequence = versionRow.getByTestId("version-sequence");
8686
const versionSequenceText = await versionSequence.textContent();
8787
const versionSequenceNumber = versionSequenceText.match(/\d+/);
8888
if (!versionSequenceNumber || versionSequenceNumber.length !== 1) {

‎e2e/playwright/tests/@range-kots-version-online/test.spec.ts

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ const validateOnlineUpdateRestrictive = async (page: Page, expect: Expect) => {
5252
await page.getByTestId("console-subnav").getByRole("link", { name: "Version history" }).click();
5353

5454
const availableUpdateCard = page.getByTestId("available-updates-card");
55-
let card = availableUpdateCard.getByTestId("version-history-row-0");
55+
const card = availableUpdateCard.getByTestId("version-history-row-0");
5656
await expect(card.getByTestId("version-label")).toContainText(constants.VENDOR_RESTRICTIVE_RELEASE_SEMVER);
5757
await expect(card.getByTestId("version-action-button")).toContainText("Download");
5858
await expect(card.getByTestId("version-status")).toContainText("Pending download");
@@ -62,32 +62,32 @@ const validateOnlineUpdateRestrictive = async (page: Page, expect: Expect) => {
6262
await expect(errorMessage).toContainText("Upgrade KOTS");
6363
await expect(errorMessage).toContainText(constants.RESTRICTIVE_TARGET_KOTS_VERSION);
6464

65-
var allVersionsCard = page.getByTestId("all-versions-card");
66-
card = allVersionsCard.getByTestId("version-history-row-0");
67-
await expect(card.getByTestId("version-label")).toContainText(constants.VENDOR_RESTRICTIVE_RELEASE_SEMVER);
65+
const allVersionsCard = page.getByTestId("all-versions-card");
66+
const versionRow = allVersionsCard.getByTestId("version-history-row-0");
67+
await expect(versionRow.getByTestId("version-label")).toContainText(constants.VENDOR_RESTRICTIVE_RELEASE_SEMVER);
6868

6969
// Click the download button and validate that you cannot download it
70-
await card.getByTestId("version-action-button").click();
70+
await versionRow.getByTestId("version-action-button").click();
7171
await page.waitForTimeout(1 * 1000); // 1 second
72-
await expect(card.getByTestId("version-downloading-status")).not.toContainText("Downloading");
72+
await expect(versionRow.getByTestId("version-downloading-status")).not.toContainText("Downloading");
7373

74-
errorMessage = card.getByTestId("version-downloading-status");
74+
errorMessage = versionRow.getByTestId("version-downloading-status");
7575
await expect(errorMessage).toContainText("requires", { timeout: 5 * 1000 }); // 5 seconds
7676
await expect(errorMessage).toContainText("Upgrade KOTS");
7777
await expect(errorMessage).toContainText(constants.RESTRICTIVE_TARGET_KOTS_VERSION);
7878

7979
// Click the diff button and validate that you cannot select this version to diff because it was
8080
// unable to download
8181
await page.getByTestId("select-releases-to-diff-button").click();
82-
await expect(card).not.toBeVisible();
82+
await expect(versionRow).not.toBeVisible();
8383

8484
// Click the cancel button and validate that you can see the version card again
8585
await page.getByTestId("cancel-diff-button").click();
86-
await expect(card).toBeVisible();
86+
await expect(versionRow).toBeVisible();
8787

8888
// A license sync may happen on installation and there will be a "License changed" version in the
8989
// list
90-
const versionSequence = card.getByTestId("version-sequence");
90+
const versionSequence = versionRow.getByTestId("version-sequence");
9191
const versionSequenceText = await versionSequence.textContent();
9292
const versionSequenceNumber = versionSequenceText.match(/\d+/);
9393
if (!versionSequenceNumber || versionSequenceNumber.length !== 1) {

0 commit comments

Comments
 (0)