Move config validation test to Playwright

.github/workflows/build-test.yaml

@@ -974,8 +974,6 @@ jobs:
           kots-namespace: 'config-validation'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
           aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
           aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
           replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'

e2e/inventory/inventory.go

@@ -22,7 +22,7 @@ const (
 func NewRegressionTest() Test {
 	return Test{
 		Name:            "Regression",
-		Label:           "type=existing cluster, env=online, phase=new install, rbac=minimal rbac",
+		TestimLabel:     "type=existing cluster, env=online, phase=new install, rbac=minimal rbac",
 		Namespace:       "qakotsregression",
 		UpstreamURI:     "qakotsregression/type-existing-cluster-env-on-2",
 		Browser:         "firefox",
@@ -36,7 +36,7 @@ func NewRegressionTest() Test {
 func NewSmokeTest() Test {
 	return Test{
 		Name:           "Smoke Test",
-		Suite:          "smoke-test",
+		TestimSuite:    "smoke-test",
 		Namespace:      "smoke-test",
 		UpstreamURI:    "qakotstestim/github-actions-qa",
 		NeedsSnapshots: true,
@@ -46,25 +46,26 @@ func NewSmokeTest() Test {
 func NewAirgapSmokeTest() Test {
 	return Test{
 		Name:        "airgap-smoke-test",
-		Suite:       "airgap-smoke-test",
+		TestimSuite: "airgap-smoke-test",
 		Namespace:   "airgap-smoke-test",
 		UpstreamURI: "airgap-smoke-test/automated",
 	}
 }
 
 func NewConfigValidation() Test {
 	return Test{
+		ID:          "config-validation",
 		Name:        "Config Validation",
-		Suite:       "config-validation",
 		Namespace:   "config-validation",
+		AppSlug:     "config-validation-panda",
 		UpstreamURI: "config-validation-panda/automated",
 	}
 }
 
 func NewBackupAndRestore() Test {
 	return Test{
 		Name:           "Backup and Restore",
-		Suite:          "backup-and-restore",
+		TestimSuite:    "backup-and-restore",
 		Namespace:      "backup-and-restore",
 		UpstreamURI:    "backup-and-restore/automated",
 		NeedsSnapshots: true,
@@ -73,17 +74,18 @@ func NewBackupAndRestore() Test {
 
 func NewNoRequiredConfig() Test {
 	return Test{
+		ID:          "no-required-config",
 		Name:        "No Required Config",
-		Suite:       "no-required-config",
 		Namespace:   "no-required-config",
+		AppSlug:     "no-required-config",
 		UpstreamURI: "no-required-config/automated",
 	}
 }
 
 func NewVersionHistoryPagination() Test {
 	return Test{
 		Name:        "Version History Pagination",
-		Suite:       "version-history-pagination",
+		TestimSuite: "version-history-pagination",
 		Namespace:   "version-history-pagination",
 		UpstreamURI: "version-history-pagination/automated",
 	}
@@ -92,7 +94,7 @@ func NewVersionHistoryPagination() Test {
 func NewChangeLicense() Test {
 	return Test{
 		Name:        "Change License",
-		Suite:       "change-license",
+		TestimSuite: "change-license",
 		Namespace:   "change-license",
 		UpstreamURI: "change-license/automated",
 	}
@@ -101,7 +103,7 @@ func NewChangeLicense() Test {
 func NewHelmManagedMode() Test {
 	return Test{
 		Name:          "Helm Managed",
-		Suite:         "helm-managed",
+		TestimSuite:   "helm-managed",
 		Namespace:     "helm-managed",
 		UpstreamURI:   "helm-managed/automated",
 		IsHelmManaged: true,
@@ -112,7 +114,7 @@ func NewHelmManagedMode() Test {
 func NewMultiAppBackupAndRestoreTest() Test {
 	return Test{
 		Name:           "multi-app-backup-and-restore",
-		Suite:          "multi-app-backup-and-restore",
+		TestimSuite:    "multi-app-backup-and-restore",
 		Namespace:      "multi-app-backup-and-restore",
 		UpstreamURI:    "multi-app-backup-and-restore/automated",
 		NeedsSnapshots: true,
@@ -122,7 +124,7 @@ func NewMultiAppBackupAndRestoreTest() Test {
 func MultiAppTest() Test {
 	return Test{
 		Name:        "multi-app-install",
-		Suite:       "multi-app-install",
+		TestimSuite: "multi-app-install",
 		Namespace:   "multi-app-install",
 		UpstreamURI: "multi-app-install/automated",
 	}
@@ -131,7 +133,7 @@ func MultiAppTest() Test {
 func NewMinKotsVersion() Test {
 	return Test{
 		Name:                   "Min KOTS Version",
-		Suite:                  "min-kots-version",
+		TestimSuite:            "min-kots-version",
 		Namespace:              "min-kots-version",
 		UpstreamURI:            "min-kots-version/automated",
 		SkipCompatibilityCheck: true,
@@ -141,7 +143,7 @@ func NewMinKotsVersion() Test {
 func NewTargetKotsVersion() Test {
 	return Test{
 		Name:                   "Target KOTS Version",
-		Suite:                  "target-kots-version",
+		TestimSuite:            "target-kots-version",
 		Namespace:              "target-kots-version",
 		UpstreamURI:            "target-kots-version/automated",
 		SkipCompatibilityCheck: true,
@@ -151,7 +153,7 @@ func NewTargetKotsVersion() Test {
 func NewRangeKotsVersion() Test {
 	return Test{
 		Name:                   "Range KOTS Version",
-		Suite:                  "range-kots-version",
+		TestimSuite:            "range-kots-version",
 		Namespace:              "range-kots-version",
 		UpstreamURI:            "range-kots-version/automated",
 		SkipCompatibilityCheck: true,
@@ -161,7 +163,7 @@ func NewRangeKotsVersion() Test {
 func NewSupportBundle() Test {
 	return Test{
 		Name:        "Support Bundle",
-		Suite:       "support-bundle",
+		TestimSuite: "support-bundle",
 		Namespace:   "support-bundle",
 		UpstreamURI: "support-bundle-halibut/automated",
 	}
@@ -170,7 +172,7 @@ func NewSupportBundle() Test {
 func NewGitOps() Test {
 	return Test{
 		Name:        "GitOps",
-		Suite:       "gitops",
+		TestimSuite: "gitops",
 		Namespace:   "gitops",
 		UpstreamURI: "gitops-bobcat/automated",
 	}

e2e/inventory/test.go

@@ -5,10 +5,12 @@ import "github.com/replicatedhq/kots/e2e/kubectl"
 type TestimParams map[string]interface{}
 
 type Test struct {
+	ID                     string // must match directory name in e2e/playwright/tests
 	Name                   string // must match test-focus in .github/workflows/build-test.yaml
-	Suite                  string
-	Label                  string
+	TestimSuite            string
+	TestimLabel            string
 	Namespace              string
+	AppSlug                string
 	UpstreamURI            string
 	Browser                string
 	UseMinimalRBAC         bool

e2e/playwright/playwright.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"time"
 
 	//lint:ignore ST1001 since Ginkgo and Gomega are DSLs this makes the tests more natural to read
@@ -29,15 +30,18 @@ func NewClient() *Client {
 }
 
 func (t *Client) HasTest(test inventory.Test) bool {
-	_, err := os.Stat(fmt.Sprintf("/playwright/tests/%s", test.Suite))
+	if test.ID == "" {
+		return false
+	}
+	_, err := os.Stat(fmt.Sprintf("/playwright/tests/%s", test.ID))
 	return err == nil
 }
 
 func (t *Client) NewRun(kubeconfig string, test inventory.Test, runOptions RunOptions) *Run {
 	args := []string{
 		"playwright",
 		"test",
-		test.Suite,
+		test.ID,
 	}
 
 	cmd := exec.Command("npx", args...)
@@ -46,6 +50,8 @@ func (t *Client) NewRun(kubeconfig string, test inventory.Test, runOptions RunOp
 	cmd.Env = append(cmd.Env, fmt.Sprintf("KUBECONFIG=%s", kubeconfig))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("PORT=%s", runOptions.Port))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("NAMESPACE=%s", test.Namespace))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("APP_SLUG=%s", test.AppSlug))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("TEST_PATH=%s", filepath.Join("tests", test.ID)))
 	cmd.Env = append(cmd.Env, "NODE_OPTIONS=--max-old-space-size=4096")
 	session, err := util.RunCommand(cmd)
 	Expect(err).WithOffset(1).Should(Succeed(), "Run playwright test failed")

e2e/playwright/tests/config-validation/invalid-jwt.txt

@@ -0,0 +1 @@
+1

e2e/playwright/tests/config-validation/license.yaml

@@ -0,0 +1,24 @@
+apiVersion: kots.io/v1beta1
+kind: License
+metadata:
+  name: github-action
+spec:
+  appSlug: config-validation-panda
+  channelID: 2OYmHaB9o0edOCJsaGFCTyz6JMb
+  channelName: Automated
+  customerName: github-action
+  endpoint: https://replicated.app
+  entitlements:
+    expires_at:
+      description: License Expiration
+      signature: {}
+      title: Expiration
+      value: ""
+      valueType: String
+  isGitOpsSupported: true
+  isNewKotsUiEnabled: true
+  isSnapshotSupported: true
+  licenseID: 2OYmj2DyiqbeR0rL84il0OFV9m0
+  licenseSequence: 1
+  licenseType: dev
+  signature: eyJsaWNlbnNlRGF0YSI6ImV5SmhjR2xXWlhKemFXOXVJam9pYTI5MGN5NXBieTkyTVdKbGRHRXhJaXdpYTJsdVpDSTZJa3hwWTJWdWMyVWlMQ0p0WlhSaFpHRjBZU0k2ZXlKdVlXMWxJam9pWjJsMGFIVmlMV0ZqZEdsdmJpSjlMQ0p6Y0dWaklqcDdJbXhwWTJWdWMyVkpSQ0k2SWpKUFdXMXFNa1I1YVhGaVpWSXdja3c0Tkdsc01FOUdWamx0TUNJc0lteHBZMlZ1YzJWVWVYQmxJam9pWkdWMklpd2lZM1Z6ZEc5dFpYSk9ZVzFsSWpvaVoybDBhSFZpTFdGamRHbHZiaUlzSW1Gd2NGTnNkV2NpT2lKamIyNW1hV2N0ZG1Gc2FXUmhkR2x2Ymkxd1lXNWtZU0lzSW1Ob1lXNXVaV3hKUkNJNklqSlBXVzFJWVVJNWJ6QmxaRTlEU25OaFIwWkRWSGw2TmtwTllpSXNJbU5vWVc1dVpXeE9ZVzFsSWpvaVFYVjBiMjFoZEdWa0lpd2liR2xqWlc1elpWTmxjWFZsYm1ObElqb3hMQ0psYm1Sd2IybHVkQ0k2SW1oMGRIQnpPaTh2Y21Wd2JHbGpZWFJsWkM1aGNIQWlMQ0psYm5ScGRHeGxiV1Z1ZEhNaU9uc2laWGh3YVhKbGMxOWhkQ0k2ZXlKMGFYUnNaU0k2SWtWNGNHbHlZWFJwYjI0aUxDSmtaWE5qY21sd2RHbHZiaUk2SWt4cFkyVnVjMlVnUlhod2FYSmhkR2x2YmlJc0luWmhiSFZsSWpvaUlpd2lkbUZzZFdWVWVYQmxJam9pVTNSeWFXNW5JaXdpYzJsbmJtRjBkWEpsSWpwN2ZYMTlMQ0pwYzBkcGRFOXdjMU4xY0hCdmNuUmxaQ0k2ZEhKMVpTd2lhWE5UYm1Gd2MyaHZkRk4xY0hCdmNuUmxaQ0k2ZEhKMVpTd2lhWE5PWlhkTGIzUnpWV2xGYm1GaWJHVmtJanAwY25WbGZYMD0iLCJpbm5lclNpZ25hdHVyZSI6ImV5SnNhV05sYm5ObFUybG5ibUYwZFhKbElqb2lVV2RSYURKWWJUTlBTSGRaYzFaUmVtVXhPVmR5THpFMmQyc3ZkRnB4YUhwQlpuQXhlWEJMV1hobE9EUndiVXg2Y1M4MWRpdERUMWxSY1VGSGVGWTVaRWx4ZDNOWFlsQklTWE5tZFd3d2RESjBNaTluVHpodFIwa3ZRU3RhVDNad1FrdDZkSFZDYkZVNVJ6ZHFlRGh5YjNvMlJtMVhOMjlLTWtNdlJqQlpXbU5ZVEhKa01XY3laVzAwUzJkM1ptZG1TemwwWmk5VmJraGtLMUozV1ZsSmVGQjBabkUxYWpoSGQwNWhXRzFIVDBKdVlTOTFZbXhqVkROd2JIbDJZVVE0ZGprNVZUTnhlVUZWV21wSlEyUmxXRmRRYUhwQlMwMWxWSFJwUTNZNFJuWktVa3B4YURoTlR6WlJWREZsU1hKUFNWWjBURkpTVkZNMVltaFRkREpyVjJaS1dqZFlXVzgwY2tKblJXaERUbWhyTWpSWVpFWXhTblpOWlRKMFpIQTVRVTFPZEVaaFJ6WTRjMjVEZEUxeFNqYzNlRGhYUWs0clJEbFZaMEkxY0VaaVFWcE5kV1ozTm1oR1dIUkRXalZVVTFwUE9GUjNQVDBpTENKd2RXSnNhV05MWlhraU9pSXRMUzB0TFVKRlIwbE9JRkJWUWt4SlF5QkxSVmt0TFMwdExWeHVUVWxKUWtscVFVNUNaMnR4YUd0cFJ6bDNNRUpCVVVWR1FVRlBRMEZST0VGTlNVbENRMmRMUTBGUlJVRnpkV2xyU2pWc1ZXbHBVVTFZZWpoa0wya3ZLMXh1VUhObVRVNXFla3AyTTBKemREZGhTMnRVYlVsTWRtZHFVemRVTXpKVGFHbEhkek5rT0ZwYVptSk5RMmh1UVZacVQyMDVVekJEWkZaR1QxSjBLM05LWTF4dVpFc3pTWFZ0WkhwbVYwVnNObVJ5UmxJdk5sSm9VakpDY1N0UGIyUXdXVUp6ZEcxbGJGTTRiblpQVEcxbkt5OUxWR3Q2ZUN0a1JYTTFRazlNUjJ4TVRGeHVNVnBHTUZadlEzTkJXV2RqY0hSb1EwdEhXa3BPU0VscVVXeGtMelpQYzBwRFNuUTRTa1E1UVVOMGRHRmpjSGhXVTNRclNuWlFUelY1V1Vkc1VIa3pPVnh1YlVKME1HSmhSWGxVT1RCdlpXTjVUVzlOTlhoUFowUlFjblpsZFRaamNXWXlWa0U1ZFV0M1VXeERiMGxhTTBOcGJGZFdiMVZJVlVZMFZGTlZTMUpRTlZ4dVJFeHNMemd4VWpOdWIzbHJXa1V2YUVKV05IVm9ibEFyYzBONlQwRmhVMkV6YUN0cUwzUlZTVlJEVEhOUVJtMHdNakp1YW1nMlpsVXlibTltWm1wTE5GeHVOMUZKUkVGUlFVSmNiaTB0TFMwdFJVNUVJRkJWUWt4SlF5QkxSVmt0TFMwdExWeHVJaXdpYTJWNVUybG5ibUYwZFhKbElqb2laWGxLZW1GWFpIVlpXRkl4WTIxVmFVOXBTbkJYVjBVMVZucG9lVlJHVW5OaGJVcHFXV3BXYlZkSVNuZGFNMVp2V2tkR05FMUVhek5YUlc5M1lrVlNVRk5GU1hsUFJuQjBWbXhDVDFRd1VYWk9WMFpaVm10a1JVMHdkRzVVYkVJMFZtdFdNRkpzUmt4V1UzUmFaVzFvZUdGdE9WSk9hbFo0VGxSYWEwMUZhRlphVkZwcVYwUnJkMkZHU2xsVmF6VjBaRVpaTlZONlRtRlZNRW94V1RGd2NsTnFVakJhUmxweVZVaFdORXN3YUZoWFYyUkRWa1JrTlZSRlRtOWtWMXAzVFRCNFZGZFlRbmRSVmtrelRXMVdhVmxVUW5oVVZGb3dWVmhHUWxOSVZYaE9Wa0pxVVd4Q2FGb3hUbXhTUXpoMlZVZFNVV1ZHU2pCVWFtTjJXV3MxTm1KRVNsUmFWMXBVVjIxR2NWVklXbEJWUkZwTVpVVmthV1JyY0RWT2JsSmFWbXRPU0ZScVJrcFNWR2h2U3pGc1VFMVVhRkZrTTNCTVZrWmFiazlHUmpaVmVrSlBZbXN4VmxNelRrMWpWbkF6VFZkak5WWXlXa0pTUnpneFUyNXNiRXd3VmtoU1dFSkNZVlJTVlZFeVkzbFdNRGxXUzNwR1NtSnRSalphVlRWVVUxUlJNbGRHY0VWUmEyaERWRE5GZG1Wck5XcGhWa0paWkVSU05sSklWbGxaYkd4RlYwWkZjbFl4YkZoaldFbDRXbXM1TmxJd1RsQmFiVXBDVFZob1RWSkZhRmhYYkZadllXNWplazE2UW5GT2EzaDVTekZTV1dJd1JUbFFVMGx6U1cxa2MySXlTbWhpUlhSc1pWVnNhMGxxYjJsWmJWSnNXbFJWTWs1VVdYZFpNbHBwVGtST2FrOVhTWGxQUjBwdFQxUm9iRmxYVG1oYWJVVXlUa1JaYVdaUlBUMGlmUT09In0=

e2e/playwright/tests/config-validation/test.spec.ts

@@ -0,0 +1,87 @@
+import { test, expect } from '@playwright/test';
+import { login, uploadLicense } from '../shared';
+
+const { execSync } = require("child_process");
+
+test('config validation', async ({ page }) => {
+  test.slow();
+  await login(page);
+  await uploadLicense(page, expect);
+  await expect(page.locator('h3')).toContainText('Config Regex Group Validation', { timeout: 15000 });
+  await page.getByRole('button', { name: 'Continue' }).click();
+  await expect(page.locator('#email_text-errblock')).toContainText('This item is required');
+  await page.locator('#email_text-group').getByRole('textbox').click();
+  await page.locator('#email_text-group').getByRole('textbox').fill('test');
+  await expect(page.locator('#email_text-group')).toContainText('A valid email address must be specified.');
+  await expect(page.locator('#app')).toContainText('Error detected. Please use config nav to the left to locate and fix issues.');
+  await page.locator('#email_text-group').getByRole('textbox').click();
+  await page.locator('#email_text-group').getByRole('textbox').fill('[email protected]');
+  await expect(page.getByText('A valid email address must be specified.')).not.toBeVisible();
+  await expect(page.getByText('Error detected. Please use config nav to the left to locate and fix issues.')).not.toBeVisible();
+  await page.locator('input[type="password"]').click();
+  await page.locator('input[type="password"]').fill('dd');
+  await expect(page.locator('#password-group')).toContainText('The password must be between 8 and 16 characters long and can contain a combination of uppercase letters, lowercase letters, digits, and special characters.');
+  await expect(page.locator('#app')).toContainText('Error detected. Please use config nav to the left to locate and fix issues.');
+  await page.locator('input[type="password"]').fill('password');
+  await expect(page.getByText('The password must be between 8 and 16 characters long and can contain a combination of uppercase letters, lowercase letters, digits, and special characters.')).not.toBeVisible();
+  await expect(page.getByText('Error detected. Please use config nav to the left to locate and fix issues.')).not.toBeVisible();
+  await page.locator('textarea').click();
+  await page.locator('textarea').fill('dd');
+  await expect(page.locator('#cve_text_area-group')).toContainText('A valid CVE number must be in the format CVE-YYYY-NNNN, where YYYY is the year and NNNN is a number between 0001 and 9999.');
+  await page.locator('textarea').fill('CVE-2023-1234');
+  await expect(page.getByText('A valid CVE number must be in the format CVE-YYYY-NNNN, where YYYY is the year and NNNN is a number between 0001 and 9999.')).not.toBeVisible();
+  await expect(page.getByText('Error detected. Please use config nav to the left to locate and fix issues.')).not.toBeVisible();
+  await page.setInputFiles('input[type="file"]', `${process.env.TEST_PATH}/invalid-jwt.txt`);
+  await expect(page.locator('#jwt_file-group')).toContainText('A valid JWT file must be in the format header.payload.signature.');
+  await expect(page.locator('#app')).toContainText('Error detected. Please use config nav to the left to locate and fix issues.');
+  await page.setInputFiles('input[type="file"]', `${process.env.TEST_PATH}/valid-jwt.txt`);
+  await expect(page.getByText('A valid JWT file must be in the format header.payload.signature.')).not.toBeVisible();
+  await expect(page.getByText('Error detected. Please use config nav to the left to locate and fix issues.')).not.toBeVisible();
+  await page.getByLabel('Customize domain name').check();
+  await page.locator('#domain_name-group').getByRole('textbox').click();
+  await page.locator('#domain_name-group').getByRole('textbox').fill('okay.domain.com');
+  await page.getByRole('button', { name: 'Continue' }).click();
+  await page.getByRole('link', { name: 'Config', exact: true }).click();
+  await page.locator('#email_text-group').getByRole('textbox').click();
+  await page.locator('#email_text-group').getByRole('textbox').fill('');
+  await page.getByRole('button', { name: 'Save config' }).click();
+  await expect(page.locator('#email_text-errblock')).toContainText('This item is required');
+  await page.locator('#email_text-group').getByRole('textbox').click();
+  await page.locator('#email_text-group').getByRole('textbox').fill('[email protected]');
+  await page.locator('input[type="password"]').click();
+  await page.locator('input[type="password"]').fill('dd');
+  await expect(page.locator('#password-group')).toContainText('The password must be between 8 and 16 characters long and can contain a combination of uppercase letters, lowercase letters, digits, and special characters.');
+  await expect(page.locator('#app')).toContainText('Error detected. Please use config nav to the left to locate and fix issues.');
+  await page.locator('input[type="password"]').fill('password');
+  await expect(page.getByText('The password must be between 8 and 16 characters long and can contain a combination of uppercase letters, lowercase letters, digits, and special characters.')).not.toBeVisible();
+  await expect(page.getByText('Error detected. Please use config nav to the left to locate and fix issues.')).not.toBeVisible();
+  await page.getByText('CVE-2023-').click();
+  await page.getByText('CVE-2023-').fill('CVE-2023-123');
+  await expect(page.locator('#cve_text_area-group')).toContainText('A valid CVE number must be in the format CVE-YYYY-NNNN, where YYYY is the year and NNNN is a number between 0001 and 9999.');
+  await expect(page.locator('#app')).toContainText('Error detected. Please use config nav to the left to locate and fix issues.');
+  await page.getByText('CVE-2023-').fill('CVE-2023-1234');
+  await expect(page.getByText('A valid CVE number must be in the format CVE-YYYY-NNNN, where YYYY is the year and NNNN is a number between 0001 and 9999.')).not.toBeVisible();
+  await expect(page.getByText('Error detected. Please use config nav to the left to locate and fix issues.')).not.toBeVisible();
+  await page.setInputFiles('input[type="file"]', `${process.env.TEST_PATH}/invalid-jwt.txt`);
+  await expect(page.locator('#jwt_file-group')).toContainText('A valid JWT file must be in the format header.payload.signature.');
+  await expect(page.locator('#app')).toContainText('Error detected. Please use config nav to the left to locate and fix issues.');
+  await page.setInputFiles('input[type="file"]', `${process.env.TEST_PATH}/valid-jwt.txt`);
+  await expect(page.getByText('A valid JWT file must be in the format header.payload.signature.')).not.toBeVisible();
+  await expect(page.getByText('Error detected. Please use config nav to the left to locate and fix issues.')).not.toBeVisible();
+  await page.getByRole('button', { name: 'Save config' }).click();
+  await expect(page.locator('.Modal-body')).toContainText('The config for Config Validation has been updated.', { timeout: 10000 });
+
+  // validate the cli
+  var invalidTestInput = [
+    { key: "email_text", invalidValue: "invalid_email", error: "A valid email address must be specified." },
+    { key: "cve_text_area", invalidValue: "CVE20221234", error: "A valid CVE number must be in the format CVEYYYYNNNN, where YYYY is the year and NNNN is a number between 0001 and 9999." },
+    { key: "password", invalidValue: "short", error: "The password must be between 8 and 16 characters long and can contain a combination of uppercase letters, lowercase letters, digits, and special characters." },
+  ];
+  for (const i of invalidTestInput) {
+    const setConfigEmailCmd = `kubectl kots set config ${process.env.APP_SLUG} -n=${process.env.NAMESPACE} --key=${i.key} --value=${i.invalidValue} --merge | grep -A1 "Errors:" | grep -v "Errors:"  | sed 's/^[[:space:]]*//;s/[-]*//g'`;
+    const setConfigResult = execSync(setConfigEmailCmd).toString().trim();
+    if (setConfigResult !== i.error) {
+      throw new Error(`Expected error message "${i.error}" but got "${setConfigResult}"`);
+    }
+  }
+});

e2e/playwright/tests/config-validation/valid-jwt.txt

@@ -0,0 +1 @@
+eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c